From e15eabe3fae72643d9705b79c9eb8f8d5d3d2c7a Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Mon, 2 Sep 2013 16:03:10 +0400 Subject: [PATCH 01/11] Ghostbusters --- modules/photo/include/opencv2/photo.hpp | 74 ++++- modules/photo/src/align.cpp | 344 ++++++++++++++++++++++-- modules/photo/src/calibrate.cpp | 10 +- modules/photo/src/hdr_common.cpp | 17 ++ modules/photo/src/hdr_common.hpp | 1 + modules/photo/src/merge.cpp | 30 +-- modules/photo/test/test_hdr.cpp | 26 +- 7 files changed, 427 insertions(+), 75 deletions(-) diff --git a/modules/photo/include/opencv2/photo.hpp b/modules/photo/include/opencv2/photo.hpp index f96aa4505f..ca9adb7d04 100644 --- a/modules/photo/include/opencv2/photo.hpp +++ b/modules/photo/include/opencv2/photo.hpp @@ -164,7 +164,7 @@ createTonemapMantiuk(float gamma = 1.0f, float scale = 0.7f, float saturation = class CV_EXPORTS_W ExposureAlign : public Algorithm { public: - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArrayOfArrays dst, + CV_WRAP virtual void process(InputArrayOfArrays src, std::vector& dst, const std::vector& times, InputArray response) = 0; }; @@ -173,22 +173,26 @@ public: class CV_EXPORTS_W AlignMTB : public ExposureAlign { public: - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArrayOfArrays dst, + CV_WRAP virtual void process(InputArrayOfArrays src, std::vector& dst, const std::vector& times, InputArray response) = 0; - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArrayOfArrays dst) = 0; + CV_WRAP virtual void process(InputArrayOfArrays src, std::vector& dst) = 0; CV_WRAP virtual void calculateShift(InputArray img0, InputArray img1, Point& shift) = 0; CV_WRAP virtual void shiftMat(InputArray src, OutputArray dst, const Point shift) = 0; + CV_WRAP virtual void computeBitmaps(Mat& img, Mat& tb, Mat& eb) = 0; CV_WRAP virtual int getMaxBits() const = 0; CV_WRAP virtual void setMaxBits(int max_bits) = 0; CV_WRAP virtual int getExcludeRange() const = 0; CV_WRAP virtual void setExcludeRange(int exclude_range) = 0; + + CV_WRAP virtual bool getCut() const = 0; + CV_WRAP virtual void setCut(bool value) = 0; }; -CV_EXPORTS_W Ptr createAlignMTB(int max_bits = 6, int exclude_range = 4); +CV_EXPORTS_W Ptr createAlignMTB(int max_bits = 6, int exclude_range = 4, bool cut = true); class CV_EXPORTS_W ExposureCalibrate : public Algorithm { @@ -206,9 +210,6 @@ public: CV_WRAP virtual int getSamples() const = 0; CV_WRAP virtual void setSamples(int samples) = 0; - - CV_WRAP virtual bool getTest() const = 0; - CV_WRAP virtual void setTest(bool val) = 0; }; CV_EXPORTS_W Ptr createCalibrateDebevec(int samples = 50, float lambda = 10.0f); @@ -278,6 +279,65 @@ public: CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, const std::vector& times) = 0; }; +CV_EXPORTS_W Ptr createMergeRobertson(); + +class CV_EXPORTS_W Ghostbuster : public Algorithm +{ +public: + CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) = 0; +}; + +// "Ghost Detection and Removal in High Dynamic Range Images", Sidibe et al., 2009 + +class CV_EXPORTS_W GhostbusterOrder : public Ghostbuster +{ +public: + CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) = 0; + CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst) = 0; + + CV_WRAP virtual int getUnderexp() = 0; + CV_WRAP virtual void setUnderexp(int value) = 0; + + CV_WRAP virtual int getOverexp() = 0; + CV_WRAP virtual void setOverexp(int value) = 0; +}; + +CV_EXPORTS_W Ptr createGhostbusterOrder(int underexp = 20, int overexp = 240); + +// "Fast and Robust High Dynamic Range Image Generation with Camera and Object Movement", Grosch, 2006 + +class CV_EXPORTS_W GhostbusterPredict : public Ghostbuster +{ +public: + CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) = 0; + CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times) = 0; + + CV_WRAP virtual int getThreshold() = 0; + CV_WRAP virtual void setThreshold(int value) = 0; + + CV_WRAP virtual int getUnderexp() = 0; + CV_WRAP virtual void setUnderexp(int value) = 0; + + CV_WRAP virtual int getOverexp() = 0; + CV_WRAP virtual void setOverexp(int value) = 0; +}; + +CV_EXPORTS_W Ptr createGhostbusterPredict(int thresh = 10, int underexp = 20, int overexp = 240); + +// "Bitmap Movement Detection: HDR for Dynamic Scenes", Pece, Kautz, 2010 + +class CV_EXPORTS_W GhostbusterBitmap : public Ghostbuster +{ +public: + CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) = 0; + CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst) = 0; + + CV_WRAP virtual int getExclude() = 0; + CV_WRAP virtual void setExclude(int value) = 0; +}; + +CV_EXPORTS_W Ptr createGhostbusterBitmap(int exclude = 4); + } // cv #endif diff --git a/modules/photo/src/align.cpp b/modules/photo/src/align.cpp index 505ff3e217..7eb9c6f090 100644 --- a/modules/photo/src/align.cpp +++ b/modules/photo/src/align.cpp @@ -50,25 +50,25 @@ namespace cv class AlignMTBImpl : public AlignMTB { public: - AlignMTBImpl(int max_bits, int exclude_range) : + AlignMTBImpl(int max_bits, int exclude_range, bool cut) : max_bits(max_bits), exclude_range(exclude_range), + cut(cut), name("AlignMTB") { } - void process(InputArrayOfArrays src, OutputArrayOfArrays dst, + void process(InputArrayOfArrays src, std::vector& dst, const std::vector& times, InputArray response) { process(src, dst); } - void process(InputArrayOfArrays _src, OutputArray _dst) + void process(InputArrayOfArrays _src, std::vector& dst) { - std::vector src, dst; + std::vector src; _src.getMatVector(src); - _dst.getMatVector(dst); - + checkImageDimensions(src); dst.resize(src.size()); @@ -76,17 +76,41 @@ public: dst[pivot] = src[pivot]; Mat gray_base; cvtColor(src[pivot], gray_base, COLOR_RGB2GRAY); + std::vector shifts; for(size_t i = 0; i < src.size(); i++) { if(i == pivot) { + shifts.push_back(Point(0, 0)); continue; } Mat gray; cvtColor(src[i], gray, COLOR_RGB2GRAY); Point shift; calculateShift(gray_base, gray, shift); + shifts.push_back(shift); shiftMat(src[i], dst[i], shift); } + if(cut) { + Point max(0, 0), min(0, 0); + for(size_t i = 0; i < shifts.size(); i++) { + if(shifts[i].x > max.x) { + max.x = shifts[i].x; + } + if(shifts[i].y > max.y) { + max.y = shifts[i].y; + } + if(shifts[i].x < min.x) { + min.x = shifts[i].x; + } + if(shifts[i].y < min.y) { + min.y = shifts[i].y; + } + } + Point size = dst[0].size(); + for(size_t i = 0; i < dst.size(); i++) { + dst[i] = dst[i](Rect(max, min + size)); + } + } } void calculateShift(InputArray _img0, InputArray _img1, Point& shift) @@ -109,8 +133,8 @@ public: shift *= 2; Mat tb1, tb2, eb1, eb2; - computeBitmaps(pyr0[level], tb1, eb1, exclude_range); - computeBitmaps(pyr1[level], tb2, eb2, exclude_range); + computeBitmaps(pyr0[level], tb1, eb1); + computeBitmaps(pyr1[level], tb2, eb2); int min_err = pyr0[level].total(); Point new_shift(shift); @@ -140,12 +164,13 @@ public: _dst.create(src.size(), src.type()); Mat dst = _dst.getMat(); - dst = Mat::zeros(src.size(), src.type()); + Mat res = Mat::zeros(src.size(), src.type()); int width = src.cols - abs(shift.x); int height = src.rows - abs(shift.y); Rect dst_rect(max(shift.x, 0), max(shift.y, 0), width, height); Rect src_rect(max(-shift.x, 0), max(-shift.y, 0), width, height); - src(src_rect).copyTo(dst(dst_rect)); + src(src_rect).copyTo(res(dst_rect)); + res.copyTo(dst); } int getMaxBits() const { return max_bits; } @@ -154,11 +179,15 @@ public: int getExcludeRange() const { return exclude_range; } void setExcludeRange(int val) { exclude_range = val; } + bool getCut() const { return cut; } + void setCut(bool val) { cut = val; } + void write(FileStorage& fs) const { fs << "name" << name << "max_bits" << max_bits - << "exclude_range" << exclude_range; + << "exclude_range" << exclude_range + << "cut" << static_cast(cut); } void read(const FileNode& fn) @@ -167,11 +196,21 @@ public: CV_Assert(n.isString() && String(n) == name); max_bits = fn["max_bits"]; exclude_range = fn["exclude_range"]; + int cut_val = fn["cut"]; + cut = static_cast(cut_val); + } + + void computeBitmaps(Mat& img, Mat& tb, Mat& eb) + { + int median = getMedian(img); + compare(img, median, tb, CMP_GT); + compare(abs(img - median), exclude_range, eb, CMP_GT); } protected: String name; int max_bits, exclude_range; + bool cut; void downsample(Mat& src, Mat& dst) { @@ -217,18 +256,289 @@ protected: } return median; } +}; - void computeBitmaps(Mat& img, Mat& tb, Mat& eb, int exclude_range) +Ptr createAlignMTB(int max_bits, int exclude_range, bool cut) +{ + return new AlignMTBImpl(max_bits, exclude_range, cut); +} + +class floatIndexCmp { +public: + floatIndexCmp(std::vector data) : + data(data) { - int median = getMedian(img); - compare(img, median, tb, CMP_GT); - compare(abs(img - median), exclude_range, eb, CMP_GT); } + + bool operator() (int i,int j) + { + return data[i] < data[j]; + } +protected: + std::vector data; +}; + +class GhostbusterOrderImpl : public GhostbusterOrder +{ +public: + GhostbusterOrderImpl(int underexp, int overexp) : + underexp(underexp), + overexp(overexp), + name("GhostbusterOrder") + { + } + + void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) + { + process(src, dst); + } + + void process(InputArrayOfArrays src, OutputArray dst) + { + std::vector unsorted_images; + src.getMatVector(unsorted_images); + checkImageDimensions(unsorted_images); + + std::vector images; + sortImages(unsorted_images, images); + + int channels = images[0].channels(); + dst.create(images[0].size(), CV_8U); + + Mat res = Mat::zeros(images[0].size(), CV_8U); + + std::vector splitted(channels); + split(images[0], splitted); + for(int i = 0; i < images.size() - 1; i++) { + + std::vector next_splitted(channels); + split(images[i + 1], next_splitted); + + for(int c = 0; c < channels; c++) { + Mat exposed = (splitted[c] >= underexp) & (splitted[c] <= overexp); + exposed &= (next_splitted[c] >= underexp) & (next_splitted[c] <= overexp); + Mat ghost = (splitted[c] > next_splitted[c]) & exposed; + res |= ghost; + } + splitted = next_splitted; + } + res.copyTo(dst.getMat()); + } + + int getUnderexp() {return underexp;} + void setUnderexp(int value) {underexp = value;} + + int getOverexp() {return overexp;} + void setOverexp(int value) {overexp = value;} + + void write(FileStorage& fs) const + { + fs << "name" << name + << "overexp" << overexp + << "underexp" << underexp; + } + + void read(const FileNode& fn) + { + FileNode n = fn["name"]; + CV_Assert(n.isString() && String(n) == name); + overexp = fn["overexp"]; + underexp = fn["underexp"]; + } + +protected: + int overexp, underexp; + String name; + + void sortImages(std::vector& images, std::vector& sorted) + { + std::vectorindices(images.size()); + std::vectormeans(images.size()); + for(size_t i = 0; i < images.size(); i++) { + indices[i] = i; + means[i] = mean(mean(images[i]))[0]; + } + sort(indices.begin(), indices.end(), floatIndexCmp(means)); + sorted.resize(images.size()); + for(size_t i = 0; i < images.size(); i++) { + sorted[i] = images[indices[i]]; + } + } +}; + +Ptr createGhostbusterOrder(int underexp, int overexp) +{ + return new GhostbusterOrderImpl(underexp, overexp); +} + +class GhostbusterPredictImpl : public GhostbusterPredict +{ +public: + GhostbusterPredictImpl(int thresh, int underexp, int overexp) : + thresh(thresh), + underexp(underexp), + overexp(overexp), + name("GhostbusterPredict") + { + } + + void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) + { + std::vector images; + src.getMatVector(images); + checkImageDimensions(images); + + int channels = images[0].channels(); + dst.create(images[0].size(), CV_8U); + + Mat res = Mat::zeros(images[0].size(), CV_8U); + + Mat radiance; + LUT(images[0], response, radiance); + std::vector splitted(channels); + split(radiance, splitted); + std::vector resp_split(channels); + split(response, resp_split); + for(int i = 0; i < images.size() - 1; i++) { + + std::vector next_splitted(channels); + LUT(images[i + 1], response, radiance); + split(radiance, next_splitted); + + for(int c = 0; c < channels; c++) { + + Mat predicted = splitted[c] / times[i] * times[i + 1]; + + Mat low = max(thresh, next_splitted[c]) - thresh; + Mat high = min(255 - thresh, next_splitted[c]) + thresh; + low.convertTo(low, CV_8U); + high.convertTo(high, CV_8U); + LUT(low, resp_split[c], low); + LUT(high, resp_split[c], high); + + Mat exposed = (splitted[c] >= underexp) & (splitted[c] <= overexp); + exposed &= (next_splitted[c] >= underexp) & (next_splitted[c] <= overexp); + + Mat ghost = (low < predicted) & (predicted < high); + ghost &= exposed; + res |= ghost; + } + splitted = next_splitted; + } + res.copyTo(dst.getMat()); + } + + virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times) + { + process(src, dst, times, linearResponse(3)); + } + + CV_WRAP virtual int getThreshold() {return thresh;} + CV_WRAP virtual void setThreshold(int value) {thresh = value;} + + int getUnderexp() {return underexp;} + void setUnderexp(int value) {underexp = value;} + + int getOverexp() {return overexp;} + void setOverexp(int value) {overexp = value;} + + void write(FileStorage& fs) const + { + fs << "name" << name + << "overexp" << overexp + << "underexp" << underexp + << "thresh" << thresh; + } + + void read(const FileNode& fn) + { + FileNode n = fn["name"]; + CV_Assert(n.isString() && String(n) == name); + overexp = fn["overexp"]; + underexp = fn["underexp"]; + thresh = fn["thresh"]; + } + +protected: + int thresh, underexp, overexp; + String name; +}; + +Ptr createGhostbusterPredict(int thresh, int underexp, int overexp) +{ + return new GhostbusterPredictImpl(thresh, underexp, overexp); +} + +class GhostbusterBitmapImpl : public GhostbusterBitmap +{ +public: + GhostbusterBitmapImpl(int exclude) : + exclude(exclude), + name("GhostbusterBitmap") + { + } + + void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) + { + process(src, dst); + } + + void process(InputArrayOfArrays src, OutputArray dst) + { + std::vector images; + src.getMatVector(images); + checkImageDimensions(images); + + int channels = images[0].channels(); + dst.create(images[0].size(), CV_8U); + + Mat res = Mat::zeros(images[0].size(), CV_8U); + + Ptr MTB = createAlignMTB(); + MTB->setExcludeRange(exclude); + + for(size_t i = 0; i < images.size(); i++) { + Mat gray; + if(channels == 1) { + gray = images[i]; + } else { + cvtColor(images[i], gray, COLOR_RGB2GRAY); + } + + Mat tb, eb; + MTB->computeBitmaps(gray, tb, eb); + tb &= eb & 1; + res += tb; + } + res = (res > 0) & (res < images.size()); + res.copyTo(dst.getMat()); + } + + int getExclude() {return exclude;} + void setExclude(int value) {exclude = value;} + + void write(FileStorage& fs) const + { + fs << "name" << name + << "exclude" << exclude; + } + + void read(const FileNode& fn) + { + FileNode n = fn["name"]; + CV_Assert(n.isString() && String(n) == name); + exclude = fn["exclude"]; + } + +protected: + int exclude; + String name; }; -CV_EXPORTS_W Ptr createAlignMTB(int max_bits, int exclude_range) +Ptr createGhostbusterBitmap(int exclude) { - return new AlignMTBImpl(max_bits, exclude_range); + return new GhostbusterBitmapImpl(exclude); } } + diff --git a/modules/photo/src/calibrate.cpp b/modules/photo/src/calibrate.cpp index ab21a8603c..95951a48d1 100644 --- a/modules/photo/src/calibrate.cpp +++ b/modules/photo/src/calibrate.cpp @@ -55,8 +55,7 @@ public: samples(samples), lambda(lambda), name("CalibrateDebevec"), - w(tringleWeights()), - test(false) + w(tringleWeights()) { } @@ -84,9 +83,6 @@ public: for(int i = 0; i < samples; i++) { int pos = 3 * (rand() % images[0].total()) + channel; - if(test) { - pos = 3 * i + channel; - } for(size_t j = 0; j < images.size(); j++) { int val = (images[j].ptr() + pos)[0]; @@ -113,9 +109,6 @@ public: exp(result, result); } - bool getTest() const { return test; } - void setTest(bool val) { test = val; } - int getSamples() const { return samples; } void setSamples(int val) { samples = val; } @@ -141,7 +134,6 @@ protected: String name; int samples; float lambda; - bool test; Mat w; }; diff --git a/modules/photo/src/hdr_common.cpp b/modules/photo/src/hdr_common.cpp index 80bd87f39f..4a0b3203b8 100644 --- a/modules/photo/src/hdr_common.cpp +++ b/modules/photo/src/hdr_common.cpp @@ -92,4 +92,21 @@ void mapLuminance(Mat src, Mat dst, Mat lum, Mat new_lum, float saturation) merge(channels, dst); } +Mat linearResponse(int channels) +{ + Mat single_response = Mat(256, 1, CV_32F); + for(int i = 1; i < 256; i++) { + single_response.at(i) = static_cast(i); + } + single_response.at(0) = static_cast(1); + + std::vector splitted(channels); + for(int c = 0; c < channels; c++) { + splitted[c] = single_response; + } + Mat result; + merge(splitted, result); + return result; +} + }; diff --git a/modules/photo/src/hdr_common.hpp b/modules/photo/src/hdr_common.hpp index 191ac63917..b00227f900 100644 --- a/modules/photo/src/hdr_common.hpp +++ b/modules/photo/src/hdr_common.hpp @@ -56,6 +56,7 @@ void mapLuminance(Mat src, Mat dst, Mat lum, Mat new_lum, float saturation); Mat RobertsonWeights(); +Mat linearResponse(int channels); }; #endif diff --git a/modules/photo/src/merge.cpp b/modules/photo/src/merge.cpp index 2cf06c1f39..0ee3518588 100644 --- a/modules/photo/src/merge.cpp +++ b/modules/photo/src/merge.cpp @@ -125,23 +125,6 @@ public: protected: String name; Mat weights; - - Mat linearResponse(int channels) - { - Mat single_response = Mat(256, 1, CV_32F); - for(int i = 1; i < 256; i++) { - single_response.at(i) = static_cast(i); - } - single_response.at(0) = static_cast(1); - - std::vector splitted(channels); - for(int c = 0; c < channels; c++) { - splitted[c] = single_response; - } - Mat result; - merge(splitted, result); - return result; - } }; Ptr createMergeDebevec() @@ -329,7 +312,7 @@ public: Mat response = input_response.getMat(); if(response.empty()) { - response = linearResponse(channels); + response = linearResponse(channels) / 128.0f; } CV_Assert(response.rows == 256 && response.cols == 1 && response.channels() == channels); @@ -355,17 +338,6 @@ public: protected: String name; Mat weight; - - Mat linearResponse(int channels) - { - Mat response = Mat::zeros(256, 1, CV_32FC3); - for(int i = 0; i < 256; i++) { - for(int c = 0; c < 3; c++) { - response.at(i)[c] = static_cast(i) / 128.0f; - } - } - return response; - } }; Ptr createMergeRobertson() diff --git a/modules/photo/test/test_hdr.cpp b/modules/photo/test/test_hdr.cpp index b0a7e49383..a3d69900bd 100644 --- a/modules/photo/test/test_hdr.cpp +++ b/modules/photo/test/test_hdr.cpp @@ -79,11 +79,11 @@ void loadExposureSeq(String path, vector& images, vector& times = DE void loadResponseCSV(String path, Mat& response) { response = Mat(256, 1, CV_32FC3); - ifstream resp_file(path.c_str()); + ifstream resp_file(path); for(int i = 0; i < 256; i++) { for(int c = 0; c < 3; c++) { resp_file >> response.at(i)[c]; - resp_file.ignore(1); + resp_file.ignore(1); } } resp_file.close(); @@ -101,31 +101,31 @@ TEST(Photo_Tonemap, regression) linear->process(img, result); loadImage(test_path + "linear.png", expected); result.convertTo(result, CV_8UC3, 255); - checkEqual(result, expected, 0); + checkEqual(result, expected, 3); Ptr drago = createTonemapDrago(gamma); drago->process(img, result); loadImage(test_path + "drago.png", expected); result.convertTo(result, CV_8UC3, 255); - checkEqual(result, expected, 0); + checkEqual(result, expected, 3); Ptr durand = createTonemapDurand(gamma); durand->process(img, result); loadImage(test_path + "durand.png", expected); result.convertTo(result, CV_8UC3, 255); - checkEqual(result, expected, 0); + checkEqual(result, expected, 3); Ptr reinhard_devlin = createTonemapReinhardDevlin(gamma); reinhard_devlin->process(img, result); loadImage(test_path + "reinharddevlin.png", expected); result.convertTo(result, CV_8UC3, 255); - checkEqual(result, expected, 0); + checkEqual(result, expected, 3); Ptr mantiuk = createTonemapMantiuk(gamma); mantiuk->process(img, result); loadImage(test_path + "mantiuk.png", expected); result.convertTo(result, CV_8UC3, 255); - checkEqual(result, expected, 0); + checkEqual(result, expected, 3); } TEST(Photo_AlignMTB, regression) @@ -169,7 +169,7 @@ TEST(Photo_MergeMertens, regression) loadImage(test_path + "merge/mertens.png", expected); merge->process(images, result); result.convertTo(result, CV_8UC3, 255); - checkEqual(expected, result, 0); + checkEqual(expected, result, 3); } TEST(Photo_MergeDebevec, regression) @@ -188,7 +188,7 @@ TEST(Photo_MergeDebevec, regression) loadImage(test_path + "merge/debevec.exr", expected); merge->process(images, result, times, response); imwrite("test.exr", result); - checkEqual(expected, result, 1e-3f); + checkEqual(expected, result, 1e-2f); } TEST(Photo_CalibrateDebevec, regression) @@ -197,11 +197,11 @@ TEST(Photo_CalibrateDebevec, regression) vector images; vector times; - Mat expected, response; + Mat response, expected; loadExposureSeq(test_path + "exposures/", images, times); - loadResponseCSV(test_path + "calibrate/debevec.csv", expected); + loadResponseCSV(test_path + "calibrate/debevec.csv", expected); Ptr calibrate = createCalibrateDebevec(); - calibrate->setTest(true); + srand(1); calibrate->process(images, response, times); - checkEqual(expected, response, 1e-3f); + checkEqual(expected, response, 1e-3f); } From 0f703b816220748f055eea77f4ee1386e421f345 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Thu, 5 Sep 2013 14:10:53 +0400 Subject: [PATCH 02/11] Calibrate Debevec changes --- modules/photo/include/opencv2/photo.hpp | 5 ++- modules/photo/src/calibrate.cpp | 45 ++++++++++++++++++++----- modules/photo/test/test_hdr.cpp | 3 +- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/modules/photo/include/opencv2/photo.hpp b/modules/photo/include/opencv2/photo.hpp index ca9adb7d04..70d472e865 100644 --- a/modules/photo/include/opencv2/photo.hpp +++ b/modules/photo/include/opencv2/photo.hpp @@ -210,9 +210,12 @@ public: CV_WRAP virtual int getSamples() const = 0; CV_WRAP virtual void setSamples(int samples) = 0; + + CV_WRAP virtual bool getRandom() const = 0; + CV_WRAP virtual void setRandom(bool random) = 0; }; -CV_EXPORTS_W Ptr createCalibrateDebevec(int samples = 50, float lambda = 10.0f); +CV_EXPORTS_W Ptr createCalibrateDebevec(int samples = 70, float lambda = 10.0f, bool random = false); // "Dynamic range improvement through multiple exposures", Robertson et al., 1999 diff --git a/modules/photo/src/calibrate.cpp b/modules/photo/src/calibrate.cpp index 95951a48d1..4701d4ae41 100644 --- a/modules/photo/src/calibrate.cpp +++ b/modules/photo/src/calibrate.cpp @@ -43,7 +43,9 @@ #include "precomp.hpp" #include "opencv2/photo.hpp" #include "opencv2/imgproc.hpp" +//#include "opencv2/highgui.hpp" #include "hdr_common.hpp" +#include namespace cv { @@ -51,11 +53,12 @@ namespace cv class CalibrateDebevecImpl : public CalibrateDebevec { public: - CalibrateDebevecImpl(int samples, float lambda) : + CalibrateDebevecImpl(int samples, float lambda, bool random) : samples(samples), lambda(lambda), name("CalibrateDebevec"), - w(tringleWeights()) + w(tringleWeights()), + random(random) { } @@ -74,18 +77,35 @@ public: dst.create(256, 1, CV_32FCC); Mat result = dst.getMat(); + std::vector sample_points; + if(random) { + for(int i = 0; i < samples; i++) { + sample_points.push_back(Point(rand() % images[0].cols, rand() % images[0].rows)); + } + } else { + int x_points = sqrt(static_cast(samples) * images[0].cols / images[0].rows); + int y_points = samples / x_points; + int step_x = images[0].cols / x_points; + int step_y = images[0].rows / y_points; + + for(int i = 0, x = step_x / 2; i < x_points; i++, x += step_x) { + for(int j = 0, y = step_y; j < y_points; j++, y += step_y) { + sample_points.push_back(Point(x, y)); + } + } + } + std::vector result_split(channels); for(int channel = 0; channel < channels; channel++) { - Mat A = Mat::zeros(samples * images.size() + 257, 256 + samples, CV_32F); + Mat A = Mat::zeros(sample_points.size() * images.size() + 257, 256 + sample_points.size(), CV_32F); Mat B = Mat::zeros(A.rows, 1, CV_32F); int eq = 0; - for(int i = 0; i < samples; i++) { + for(size_t i = 0; i < sample_points.size(); i++) { - int pos = 3 * (rand() % images[0].total()) + channel; for(size_t j = 0; j < images.size(); j++) { - int val = (images[j].ptr() + pos)[0]; + int val = images[j].ptr()[3*(sample_points[i].y * images[j].cols + sample_points[j].x) + channel]; A.at(eq, val) = w.at(val); A.at(eq, 256 + i) = -w.at(val); B.at(eq, 0) = w.at(val) * log(times[j]); @@ -115,11 +135,15 @@ public: float getLambda() const { return lambda; } void setLambda(float val) { lambda = val; } + bool getRandom() const { return random; } + void setRandom(bool val) { random = val; } + void write(FileStorage& fs) const { fs << "name" << name << "samples" << samples - << "lambda" << lambda; + << "lambda" << lambda + << "random" << static_cast(random); } void read(const FileNode& fn) @@ -128,18 +152,21 @@ public: CV_Assert(n.isString() && String(n) == name); samples = fn["samples"]; lambda = fn["lambda"]; + int random_val = fn["random"]; + random = static_cast(random_val); } protected: String name; int samples; float lambda; + bool random; Mat w; }; -Ptr createCalibrateDebevec(int samples, float lambda) +Ptr createCalibrateDebevec(int samples, float lambda, bool random) { - return new CalibrateDebevecImpl(samples, lambda); + return new CalibrateDebevecImpl(samples, lambda, random); } class CalibrateRobertsonImpl : public CalibrateRobertson diff --git a/modules/photo/test/test_hdr.cpp b/modules/photo/test/test_hdr.cpp index a3d69900bd..efc3774fa0 100644 --- a/modules/photo/test/test_hdr.cpp +++ b/modules/photo/test/test_hdr.cpp @@ -1,4 +1,4 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// + /*M/////////////////////////////////////////////////////////////////////////////////////// // // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. // @@ -201,7 +201,6 @@ TEST(Photo_CalibrateDebevec, regression) loadExposureSeq(test_path + "exposures/", images, times); loadResponseCSV(test_path + "calibrate/debevec.csv", expected); Ptr calibrate = createCalibrateDebevec(); - srand(1); calibrate->process(images, response, times); checkEqual(expected, response, 1e-3f); } From cb27740de48a89befb124e805440661c82940299 Mon Sep 17 00:00:00 2001 From: fm Date: Thu, 5 Sep 2013 14:26:21 +0400 Subject: [PATCH 03/11] Linux compilation --- modules/photo/test/test_hdr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/photo/test/test_hdr.cpp b/modules/photo/test/test_hdr.cpp index efc3774fa0..569f3a3005 100644 --- a/modules/photo/test/test_hdr.cpp +++ b/modules/photo/test/test_hdr.cpp @@ -79,7 +79,7 @@ void loadExposureSeq(String path, vector& images, vector& times = DE void loadResponseCSV(String path, Mat& response) { response = Mat(256, 1, CV_32FC3); - ifstream resp_file(path); + ifstream resp_file(path.c_str()); for(int i = 0; i < 256; i++) { for(int c = 0; c < 3; c++) { resp_file >> response.at(i)[c]; From 85eb52b8af54c199d890ab075a67d8741ff4fab6 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Thu, 5 Sep 2013 14:41:35 +0400 Subject: [PATCH 04/11] Debevec Calibrate --- modules/photo/test/test_hdr.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/photo/test/test_hdr.cpp b/modules/photo/test/test_hdr.cpp index efc3774fa0..af85317055 100644 --- a/modules/photo/test/test_hdr.cpp +++ b/modules/photo/test/test_hdr.cpp @@ -202,5 +202,9 @@ TEST(Photo_CalibrateDebevec, regression) loadResponseCSV(test_path + "calibrate/debevec.csv", expected); Ptr calibrate = createCalibrateDebevec(); calibrate->process(images, response, times); - checkEqual(expected, response, 1e-3f); + Mat diff = abs(response - expected); + diff = diff.mul(1.0f / response); + double max; + minMaxLoc(diff, NULL, &max); + ASSERT_FALSE(max > 0.1); } From f4792f8a44e50023ec020f9ec65397c34f2a37b0 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Fri, 6 Sep 2013 14:43:04 +0400 Subject: [PATCH 05/11] Doc update --- modules/photo/doc/hdr_imaging.rst | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/photo/doc/hdr_imaging.rst b/modules/photo/doc/hdr_imaging.rst index b8090013df..f07d9ae941 100644 --- a/modules/photo/doc/hdr_imaging.rst +++ b/modules/photo/doc/hdr_imaging.rst @@ -192,16 +192,30 @@ Helper function, that shift Mat filling new regions with zeros. :param dst: result image :param shift: shift value + +AlignMTB::computeBitmaps +--------------------------- +Computes median threshold and exclude bitmaps of given image. + +.. ocv:function:: void computeBitmaps(Mat& img, Mat& tb, Mat& eb) + + :param img: input image + + :param tb: median threshold bitmap + :param eb: exclude bitmap + createAlignMTB --------------------------- Creates AlignMTB object -.. ocv:function:: Ptr createAlignMTB(int max_bits = 6, int exclude_range = 4) +.. ocv:function:: Ptr createAlignMTB(int max_bits = 6, int exclude_range = 4, bool cut = true) :param max_bits: logarithm to the base 2 of maximal shift in each dimension. Values of 5 and 6 are usually good enough (31 and 63 pixels shift respectively). :param exclude_range: range for exclusion bitmap that is constructed to suppress noise around the median value. + + :param cut: if true cuts images, otherwise fills the new regions with zeros. ExposureCalibrate --------------------------- @@ -234,11 +248,13 @@ createCalibrateDebevec --------------------------- Creates CalibrateDebevec object -.. ocv:function:: Ptr createCalibrateDebevec(int samples = 50, float lambda = 10.0f) +.. ocv:function:: createCalibrateDebevec(int samples = 70, float lambda = 10.0f, bool random = false) :param samples: number of pixel locations to use :param lambda: smoothness term weight. Greater values produce smoother results, but can alter the response. + + :param random: if true sample pixel locations are chosen at random, otherwise the form a rectangular grid. ExposureMerge --------------------------- From deeaddb0a919254327f54a2fd819ef39049058a8 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Fri, 6 Sep 2013 14:49:19 +0400 Subject: [PATCH 06/11] Doc update --- modules/photo/doc/hdr_imaging.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/photo/doc/hdr_imaging.rst b/modules/photo/doc/hdr_imaging.rst index f07d9ae941..56b74a63e8 100644 --- a/modules/photo/doc/hdr_imaging.rst +++ b/modules/photo/doc/hdr_imaging.rst @@ -280,7 +280,7 @@ MergeDebevec --------------------------- .. ocv:class:: MergeDebevec : public ExposureMerge -The resulting HDR image is calculated as weighted average of he exposures considering exposure values and camera response. +The resulting HDR image is calculated as weighted average of the exposures considering exposure values and camera response. For more information see [DM97]_. From c118f3c529642ed7c87c07722af175fadda22838 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Fri, 6 Sep 2013 17:30:43 +0400 Subject: [PATCH 07/11] Robertson update --- modules/photo/include/opencv2/photo.hpp | 6 +- modules/photo/src/align.cpp | 14 +++-- modules/photo/src/calibrate.cpp | 74 ++++++++++--------------- modules/photo/src/hdr_common.cpp | 35 +++++------- modules/photo/src/merge.cpp | 8 +-- modules/photo/test/test_hdr.cpp | 32 ++++++++++- 6 files changed, 92 insertions(+), 77 deletions(-) diff --git a/modules/photo/include/opencv2/photo.hpp b/modules/photo/include/opencv2/photo.hpp index 70d472e865..ca3975f12a 100644 --- a/modules/photo/include/opencv2/photo.hpp +++ b/modules/photo/include/opencv2/photo.hpp @@ -80,6 +80,8 @@ CV_EXPORTS_W void fastNlMeansDenoisingColoredMulti( InputArrayOfArrays srcImgs, float h = 3, float hColor = 3, int templateWindowSize = 7, int searchWindowSize = 21); +enum { LDR_SIZE = 256 }; + class CV_EXPORTS_W Tonemap : public Algorithm { public: @@ -227,9 +229,11 @@ public: CV_WRAP virtual float getThreshold() const = 0; CV_WRAP virtual void setThreshold(float threshold) = 0; + + CV_WRAP virtual Mat getRadiance() const = 0; }; -CV_EXPORTS_W Ptr createCalibrateRobertson(int samples = 50, float lambda = 10.0f); +CV_EXPORTS_W Ptr createCalibrateRobertson(int max_iter = 30, float threshold = 0.01f); class CV_EXPORTS_W ExposureMerge : public Algorithm { diff --git a/modules/photo/src/align.cpp b/modules/photo/src/align.cpp index 7eb9c6f090..295fea3503 100644 --- a/modules/photo/src/align.cpp +++ b/modules/photo/src/align.cpp @@ -243,14 +243,14 @@ protected: { int channels = 0; Mat hist; - int hist_size = 256; - float range[] = {0, 256} ; + int hist_size = LDR_SIZE; + float range[] = {0, LDR_SIZE} ; const float* ranges[] = {range}; calcHist(&img, 1, &channels, Mat(), hist, 1, &hist_size, ranges); float *ptr = hist.ptr(); int median = 0, sum = 0; int thresh = img.total() / 2; - while(sum < thresh && median < 256) { + while(sum < thresh && median < LDR_SIZE) { sum += static_cast(ptr[median]); median++; } @@ -309,7 +309,7 @@ public: std::vector splitted(channels); split(images[0], splitted); - for(int i = 0; i < images.size() - 1; i++) { + for(size_t i = 0; i < images.size() - 1; i++) { std::vector next_splitted(channels); split(images[i + 1], next_splitted); @@ -399,7 +399,7 @@ public: split(radiance, splitted); std::vector resp_split(channels); split(response, resp_split); - for(int i = 0; i < images.size() - 1; i++) { + for(size_t i = 0; i < images.size() - 1; i++) { std::vector next_splitted(channels); LUT(images[i + 1], response, radiance); @@ -430,7 +430,9 @@ public: virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times) { - process(src, dst, times, linearResponse(3)); + Mat response = linearResponse(3); + response.at(0) = response.at(1); + process(src, dst, times, response); } CV_WRAP virtual int getThreshold() {return thresh;} diff --git a/modules/photo/src/calibrate.cpp b/modules/photo/src/calibrate.cpp index 4701d4ae41..0b08a501b2 100644 --- a/modules/photo/src/calibrate.cpp +++ b/modules/photo/src/calibrate.cpp @@ -45,7 +45,6 @@ #include "opencv2/imgproc.hpp" //#include "opencv2/highgui.hpp" #include "hdr_common.hpp" -#include namespace cv { @@ -74,7 +73,7 @@ public: int channels = images[0].channels(); int CV_32FCC = CV_MAKETYPE(CV_32F, channels); - dst.create(256, 1, CV_32FCC); + dst.create(LDR_SIZE, 1, CV_32FCC); Mat result = dst.getMat(); std::vector sample_points; @@ -97,7 +96,7 @@ public: std::vector result_split(channels); for(int channel = 0; channel < channels; channel++) { - Mat A = Mat::zeros(sample_points.size() * images.size() + 257, 256 + sample_points.size(), CV_32F); + Mat A = Mat::zeros(sample_points.size() * images.size() + LDR_SIZE + 1, LDR_SIZE + sample_points.size(), CV_32F); Mat B = Mat::zeros(A.rows, 1, CV_32F); int eq = 0; @@ -107,12 +106,12 @@ public: int val = images[j].ptr()[3*(sample_points[i].y * images[j].cols + sample_points[j].x) + channel]; A.at(eq, val) = w.at(val); - A.at(eq, 256 + i) = -w.at(val); + A.at(eq, LDR_SIZE + i) = -w.at(val); B.at(eq, 0) = w.at(val) * log(times[j]); eq++; } } - A.at(eq, 128) = 1; + A.at(eq, LDR_SIZE / 2) = 1; eq++; for(int i = 0; i < 254; i++) { @@ -123,7 +122,7 @@ public: } Mat solution; solve(A, B, solution, DECOMP_SVD); - solution.rowRange(0, 256).copyTo(result_split[channel]); + solution.rowRange(0, LDR_SIZE).copyTo(result_split[channel]); } merge(result_split, result); exp(result, result); @@ -192,20 +191,14 @@ public: int channels = images[0].channels(); int CV_32FCC = CV_MAKETYPE(CV_32F, channels); - dst.create(256, 1, CV_32FCC); + dst.create(LDR_SIZE, 1, CV_32FCC); Mat response = dst.getMat(); - - response = Mat::zeros(256, 1, CV_32FCC); - for(int i = 0; i < 256; i++) { - for(int c = 0; c < channels; c++) { - response.at(i)[c] = i / 128.0; - } - } + response = linearResponse(3) / (LDR_SIZE / 2.0f); - Mat card = Mat::zeros(256, 1, CV_32FCC); - for(int i = 0; i < images.size(); i++) { + Mat card = Mat::zeros(LDR_SIZE, 1, CV_32FCC); + for(size_t i = 0; i < images.size(); i++) { uchar *ptr = images[i].ptr(); - for(int pos = 0; pos < images[i].total(); pos++) { + for(size_t pos = 0; pos < images[i].total(); pos++) { for(int c = 0; c < channels; c++, ptr++) { card.at(*ptr)[c] += 1; } @@ -213,43 +206,34 @@ public: } card = 1.0 / card; + Ptr merge = createMergeRobertson(); for(int iter = 0; iter < max_iter; iter++) { - Scalar channel_err(0, 0, 0); - Mat radiance = Mat::zeros(images[0].size(), CV_32FCC); - Mat wsum = Mat::zeros(images[0].size(), CV_32FCC); - for(int i = 0; i < images.size(); i++) { - Mat im, w; - LUT(images[i], weight, w); - LUT(images[i], response, im); - - Mat err_mat; - pow(im - times[i] * radiance, 2.0f, err_mat); - err_mat = w.mul(err_mat); - channel_err += sum(err_mat); - - radiance += times[i] * w.mul(im); - wsum += pow(times[i], 2) * w; - } - float err = (channel_err[0] + channel_err[1] + channel_err[2]) / (channels * radiance.total()); - radiance = radiance.mul(1 / wsum); + radiance = Mat::zeros(images[0].size(), CV_32FCC); + merge->process(images, radiance, times, response); - float* rad_ptr = radiance.ptr(); - response = Mat::zeros(256, 1, CV_32FC3); - for(int i = 0; i < images.size(); i++) { + Mat new_response = Mat::zeros(LDR_SIZE, 1, CV_32FC3); + for(size_t i = 0; i < images.size(); i++) { uchar *ptr = images[i].ptr(); - for(int pos = 0; pos < images[i].total(); pos++) { + float* rad_ptr = radiance.ptr(); + for(size_t pos = 0; pos < images[i].total(); pos++) { for(int c = 0; c < channels; c++, ptr++, rad_ptr++) { - response.at(*ptr)[c] += times[i] * *rad_ptr; + new_response.at(*ptr)[c] += times[i] * *rad_ptr; } } } - response = response.mul(card); + new_response = new_response.mul(card); for(int c = 0; c < 3; c++) { - for(int i = 0; i < 256; i++) { - response.at(i)[c] /= response.at(128)[c]; + float middle = new_response.at(LDR_SIZE / 2)[c]; + for(int i = 0; i < LDR_SIZE; i++) { + new_response.at(i)[c] /= middle; } } + float diff = sum(sum(abs(new_response - response)))[0] / channels; + new_response.copyTo(response); + if(diff < threshold) { + break; + } } } @@ -259,6 +243,8 @@ public: float getThreshold() const { return threshold; } void setThreshold(float val) { threshold = val; } + Mat getRadiance() const { return radiance; } + void write(FileStorage& fs) const { fs << "name" << name @@ -278,7 +264,7 @@ protected: String name; int max_iter; float threshold; - Mat weight; + Mat weight, radiance; }; Ptr createCalibrateRobertson(int max_iter, float threshold) diff --git a/modules/photo/src/hdr_common.cpp b/modules/photo/src/hdr_common.cpp index 4a0b3203b8..27512587b2 100644 --- a/modules/photo/src/hdr_common.cpp +++ b/modules/photo/src/hdr_common.cpp @@ -61,21 +61,22 @@ void checkImageDimensions(const std::vector& images) Mat tringleWeights() { - Mat w(256, 1, CV_32F); - for(int i = 0; i < 256; i++) { - w.at(i) = i < 128 ? i + 1.0f : 256.0f - i; + Mat w(LDR_SIZE, 1, CV_32F); + int half = LDR_SIZE / 2; + for(int i = 0; i < LDR_SIZE; i++) { + w.at(i) = i < half ? i + 1.0f : LDR_SIZE - i; } return w; } Mat RobertsonWeights() { - Mat weight(256, 1, CV_32FC3); - for(int i = 0; i < 256; i++) { - float value = exp(-4.0f * pow(i - 127.5f, 2.0f) / pow(127.5f, 2.0f)); - for(int c = 0; c < 3; c++) { - weight.at(i)[c] = value; - } + Mat weight(LDR_SIZE, 1, CV_32FC3); + float q = (LDR_SIZE - 1) / 4.0f; + for(int i = 0; i < LDR_SIZE; i++) { + float value = i / q - 2.0f; + value = exp(-value * value); + weight.at(i) = Vec3f::all(value); } return weight; } @@ -94,19 +95,11 @@ void mapLuminance(Mat src, Mat dst, Mat lum, Mat new_lum, float saturation) Mat linearResponse(int channels) { - Mat single_response = Mat(256, 1, CV_32F); - for(int i = 1; i < 256; i++) { - single_response.at(i) = static_cast(i); + Mat response = Mat(LDR_SIZE, 1, CV_MAKETYPE(CV_32F, channels)); + for(int i = 0; i < LDR_SIZE; i++) { + response.at(i) = Vec3f::all(i); } - single_response.at(0) = static_cast(1); - - std::vector splitted(channels); - for(int c = 0; c < channels; c++) { - splitted[c] = single_response; - } - Mat result; - merge(splitted, result); - return result; + return response; } }; diff --git a/modules/photo/src/merge.cpp b/modules/photo/src/merge.cpp index 0ee3518588..c2470217c7 100644 --- a/modules/photo/src/merge.cpp +++ b/modules/photo/src/merge.cpp @@ -43,7 +43,6 @@ #include "opencv2/photo.hpp" #include "opencv2/imgproc.hpp" #include "hdr_common.hpp" -#include namespace cv { @@ -77,9 +76,10 @@ public: if(response.empty()) { response = linearResponse(channels); + response.at(0) = response.at(1); } log(response, response); - CV_Assert(response.rows == 256 && response.cols == 1 && + CV_Assert(response.rows == LDR_SIZE && response.cols == 1 && response.channels() == channels); Mat exp_values(times); @@ -312,9 +312,9 @@ public: Mat response = input_response.getMat(); if(response.empty()) { - response = linearResponse(channels) / 128.0f; + response = linearResponse(channels) / (LDR_SIZE / 2.0f); } - CV_Assert(response.rows == 256 && response.cols == 1 && + CV_Assert(response.rows == LDR_SIZE && response.cols == 1 && response.channels() == channels); result = Mat::zeros(images[0].size(), CV_32FCC); diff --git a/modules/photo/test/test_hdr.cpp b/modules/photo/test/test_hdr.cpp index c4c376fe92..a86f11977b 100644 --- a/modules/photo/test/test_hdr.cpp +++ b/modules/photo/test/test_hdr.cpp @@ -187,7 +187,22 @@ TEST(Photo_MergeDebevec, regression) Mat result, expected; loadImage(test_path + "merge/debevec.exr", expected); merge->process(images, result, times, response); - imwrite("test.exr", result); + checkEqual(expected, result, 1e-2f); +} + +TEST(Photo_MergeRobertson, regression) +{ + string test_path = string(cvtest::TS::ptr()->get_data_path()) + "hdr/"; + + vector images; + vector times; + loadExposureSeq(test_path + "exposures/", images, times); + + Ptr merge = createMergeRobertson(); + + Mat result, expected; + loadImage(test_path + "merge/robertson.exr", expected); + merge->process(images, result, times); checkEqual(expected, result, 1e-2f); } @@ -208,3 +223,18 @@ TEST(Photo_CalibrateDebevec, regression) minMaxLoc(diff, NULL, &max); ASSERT_FALSE(max > 0.1); } + +TEST(Photo_CalibrateRobertson, regression) +{ + string test_path = string(cvtest::TS::ptr()->get_data_path()) + "hdr/"; + + vector images; + vector times; + Mat response, expected; + loadExposureSeq(test_path + "exposures/", images, times); + loadResponseCSV(test_path + "calibrate/robertson.csv", expected); + + Ptr calibrate = createCalibrateRobertson(); + calibrate->process(images, response, times); + checkEqual(expected, response, 1e-3f); +} \ No newline at end of file From a7bcf63765995676a7bb999df49a3001ec245fb9 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Fri, 6 Sep 2013 18:21:59 +0400 Subject: [PATCH 08/11] Robertson docs --- modules/photo/doc/hdr_imaging.rst | 46 +++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/modules/photo/doc/hdr_imaging.rst b/modules/photo/doc/hdr_imaging.rst index 56b74a63e8..042f888337 100644 --- a/modules/photo/doc/hdr_imaging.rst +++ b/modules/photo/doc/hdr_imaging.rst @@ -204,7 +204,7 @@ Computes median threshold and exclude bitmaps of given image. :param tb: median threshold bitmap :param eb: exclude bitmap - + createAlignMTB --------------------------- Creates AlignMTB object @@ -214,8 +214,8 @@ Creates AlignMTB object :param max_bits: logarithm to the base 2 of maximal shift in each dimension. Values of 5 and 6 are usually good enough (31 and 63 pixels shift respectively). :param exclude_range: range for exclusion bitmap that is constructed to suppress noise around the median value. - - :param cut: if true cuts images, otherwise fills the new regions with zeros. + + :param cut: if true cuts images, otherwise fills the new regions with zeros. ExposureCalibrate --------------------------- @@ -253,8 +253,27 @@ Creates CalibrateDebevec object :param samples: number of pixel locations to use :param lambda: smoothness term weight. Greater values produce smoother results, but can alter the response. - - :param random: if true sample pixel locations are chosen at random, otherwise the form a rectangular grid. + + :param random: if true sample pixel locations are chosen at random, otherwise the form a rectangular grid. + +CalibrateRobertson +--------------------------- +.. ocv:class:: CalibrateRobertson : public ExposureCalibrate + +Inverse camera response function is extracted for each brightness value by minimizing an objective function as linear system. +This algorithm uses all image pixels. + +For more information see [RB99]_. + +createCalibrateRobertson +--------------------------- +Creates CalibrateRobertson object + +.. ocv:function:: createCalibrateRobertson(int max_iter = 30, float threshold = 0.01f) + + :param max_iter: maximal number of Gauss-Seidel solver iterations. + + :param threshold: target difference between results of two successive steps of the minimization. ExposureMerge --------------------------- @@ -312,7 +331,6 @@ Short version of process, that doesn't take extra arguments. :param dst: result image - createMergeMertens --------------------------- Creates MergeMertens object @@ -324,6 +342,20 @@ Creates MergeMertens object :param saturation_weight: saturation measure weight :param exposure_weight: well-exposedness measure weight + +MergeRobertson +--------------------------- +.. ocv:class:: MergeRobertson : public ExposureMerge + +The resulting HDR image is calculated as weighted average of the exposures considering exposure values and camera response. + +For more information see [RB99]_. + +createMergeRobertson +--------------------------- +Creates MergeRobertson object + +.. ocv:function:: Ptr createMergeRobertson() References ========== @@ -343,3 +375,5 @@ References .. [DM97] P. Debevec, J. Malik, "Recovering High Dynamic Range Radiance Maps from Photographs", Proceedings OF ACM SIGGRAPH, 1997, 369 - 378. .. [MK07] T. Mertens, J. Kautz, F. Van Reeth, "Exposure Fusion", Proceedings of the 15th Pacific Conference on Computer Graphics and Applications, 2007, 382 - 390. + +.. [RB99] M. Robertson , S. Borman , R. Stevenson , "Dynamic range improvement through multiple exposures ", Proceedings of the Int. Conf. on Image Processing , 1999, 159 - 163. From 76c6fa934894aa95d0eded78a413e7a4d63c72b1 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Tue, 17 Sep 2013 00:39:22 +0400 Subject: [PATCH 09/11] No EXR in tests --- modules/photo/test/test_hdr.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/photo/test/test_hdr.cpp b/modules/photo/test/test_hdr.cpp index a86f11977b..f166746d99 100644 --- a/modules/photo/test/test_hdr.cpp +++ b/modules/photo/test/test_hdr.cpp @@ -185,8 +185,12 @@ TEST(Photo_MergeDebevec, regression) Ptr merge = createMergeDebevec(); Mat result, expected; - loadImage(test_path + "merge/debevec.exr", expected); + loadImage(test_path + "merge/debevec.hdr", expected); merge->process(images, result, times, response); + Ptr map = createTonemap(); + map->process(result, result); + map->process(expected, expected); + checkEqual(expected, result, 1e-2f); } @@ -203,6 +207,9 @@ TEST(Photo_MergeRobertson, regression) Mat result, expected; loadImage(test_path + "merge/robertson.exr", expected); merge->process(images, result, times); + Ptr map = createTonemap(); + map->process(result, result); + map->process(expected, expected); checkEqual(expected, result, 1e-2f); } From 89508bf7ab8886047292ce9ce393bf8d2f5b7dd0 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Tue, 17 Sep 2013 00:46:15 +0400 Subject: [PATCH 10/11] NO exr now --- modules/photo/test/test_hdr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/photo/test/test_hdr.cpp b/modules/photo/test/test_hdr.cpp index f166746d99..3edc4b093d 100644 --- a/modules/photo/test/test_hdr.cpp +++ b/modules/photo/test/test_hdr.cpp @@ -205,7 +205,7 @@ TEST(Photo_MergeRobertson, regression) Ptr merge = createMergeRobertson(); Mat result, expected; - loadImage(test_path + "merge/robertson.exr", expected); + loadImage(test_path + "merge/robertson.hdr", expected); merge->process(images, result, times); Ptr map = createTonemap(); map->process(result, result); From b1af5cc4782112ff58f8ea0c03b751bb8c0d7308 Mon Sep 17 00:00:00 2001 From: Fedor Morozov Date: Wed, 18 Sep 2013 18:15:51 +0400 Subject: [PATCH 11/11] Remove ghost removal --- modules/photo/include/opencv2/photo.hpp | 57 ----- modules/photo/src/align.cpp | 279 ------------------------ 2 files changed, 336 deletions(-) diff --git a/modules/photo/include/opencv2/photo.hpp b/modules/photo/include/opencv2/photo.hpp index ca3975f12a..f48df802b0 100644 --- a/modules/photo/include/opencv2/photo.hpp +++ b/modules/photo/include/opencv2/photo.hpp @@ -288,63 +288,6 @@ public: CV_EXPORTS_W Ptr createMergeRobertson(); -class CV_EXPORTS_W Ghostbuster : public Algorithm -{ -public: - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) = 0; -}; - -// "Ghost Detection and Removal in High Dynamic Range Images", Sidibe et al., 2009 - -class CV_EXPORTS_W GhostbusterOrder : public Ghostbuster -{ -public: - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) = 0; - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst) = 0; - - CV_WRAP virtual int getUnderexp() = 0; - CV_WRAP virtual void setUnderexp(int value) = 0; - - CV_WRAP virtual int getOverexp() = 0; - CV_WRAP virtual void setOverexp(int value) = 0; -}; - -CV_EXPORTS_W Ptr createGhostbusterOrder(int underexp = 20, int overexp = 240); - -// "Fast and Robust High Dynamic Range Image Generation with Camera and Object Movement", Grosch, 2006 - -class CV_EXPORTS_W GhostbusterPredict : public Ghostbuster -{ -public: - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) = 0; - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times) = 0; - - CV_WRAP virtual int getThreshold() = 0; - CV_WRAP virtual void setThreshold(int value) = 0; - - CV_WRAP virtual int getUnderexp() = 0; - CV_WRAP virtual void setUnderexp(int value) = 0; - - CV_WRAP virtual int getOverexp() = 0; - CV_WRAP virtual void setOverexp(int value) = 0; -}; - -CV_EXPORTS_W Ptr createGhostbusterPredict(int thresh = 10, int underexp = 20, int overexp = 240); - -// "Bitmap Movement Detection: HDR for Dynamic Scenes", Pece, Kautz, 2010 - -class CV_EXPORTS_W GhostbusterBitmap : public Ghostbuster -{ -public: - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) = 0; - CV_WRAP virtual void process(InputArrayOfArrays src, OutputArray dst) = 0; - - CV_WRAP virtual int getExclude() = 0; - CV_WRAP virtual void setExclude(int value) = 0; -}; - -CV_EXPORTS_W Ptr createGhostbusterBitmap(int exclude = 4); - } // cv #endif diff --git a/modules/photo/src/align.cpp b/modules/photo/src/align.cpp index 295fea3503..35c949172f 100644 --- a/modules/photo/src/align.cpp +++ b/modules/photo/src/align.cpp @@ -263,284 +263,5 @@ Ptr createAlignMTB(int max_bits, int exclude_range, bool cut) return new AlignMTBImpl(max_bits, exclude_range, cut); } -class floatIndexCmp { -public: - floatIndexCmp(std::vector data) : - data(data) - { - } - - bool operator() (int i,int j) - { - return data[i] < data[j]; - } -protected: - std::vector data; -}; - -class GhostbusterOrderImpl : public GhostbusterOrder -{ -public: - GhostbusterOrderImpl(int underexp, int overexp) : - underexp(underexp), - overexp(overexp), - name("GhostbusterOrder") - { - } - - void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) - { - process(src, dst); - } - - void process(InputArrayOfArrays src, OutputArray dst) - { - std::vector unsorted_images; - src.getMatVector(unsorted_images); - checkImageDimensions(unsorted_images); - - std::vector images; - sortImages(unsorted_images, images); - - int channels = images[0].channels(); - dst.create(images[0].size(), CV_8U); - - Mat res = Mat::zeros(images[0].size(), CV_8U); - - std::vector splitted(channels); - split(images[0], splitted); - for(size_t i = 0; i < images.size() - 1; i++) { - - std::vector next_splitted(channels); - split(images[i + 1], next_splitted); - - for(int c = 0; c < channels; c++) { - Mat exposed = (splitted[c] >= underexp) & (splitted[c] <= overexp); - exposed &= (next_splitted[c] >= underexp) & (next_splitted[c] <= overexp); - Mat ghost = (splitted[c] > next_splitted[c]) & exposed; - res |= ghost; - } - splitted = next_splitted; - } - res.copyTo(dst.getMat()); - } - - int getUnderexp() {return underexp;} - void setUnderexp(int value) {underexp = value;} - - int getOverexp() {return overexp;} - void setOverexp(int value) {overexp = value;} - - void write(FileStorage& fs) const - { - fs << "name" << name - << "overexp" << overexp - << "underexp" << underexp; - } - - void read(const FileNode& fn) - { - FileNode n = fn["name"]; - CV_Assert(n.isString() && String(n) == name); - overexp = fn["overexp"]; - underexp = fn["underexp"]; - } - -protected: - int overexp, underexp; - String name; - - void sortImages(std::vector& images, std::vector& sorted) - { - std::vectorindices(images.size()); - std::vectormeans(images.size()); - for(size_t i = 0; i < images.size(); i++) { - indices[i] = i; - means[i] = mean(mean(images[i]))[0]; - } - sort(indices.begin(), indices.end(), floatIndexCmp(means)); - sorted.resize(images.size()); - for(size_t i = 0; i < images.size(); i++) { - sorted[i] = images[indices[i]]; - } - } -}; - -Ptr createGhostbusterOrder(int underexp, int overexp) -{ - return new GhostbusterOrderImpl(underexp, overexp); -} - -class GhostbusterPredictImpl : public GhostbusterPredict -{ -public: - GhostbusterPredictImpl(int thresh, int underexp, int overexp) : - thresh(thresh), - underexp(underexp), - overexp(overexp), - name("GhostbusterPredict") - { - } - - void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) - { - std::vector images; - src.getMatVector(images); - checkImageDimensions(images); - - int channels = images[0].channels(); - dst.create(images[0].size(), CV_8U); - - Mat res = Mat::zeros(images[0].size(), CV_8U); - - Mat radiance; - LUT(images[0], response, radiance); - std::vector splitted(channels); - split(radiance, splitted); - std::vector resp_split(channels); - split(response, resp_split); - for(size_t i = 0; i < images.size() - 1; i++) { - - std::vector next_splitted(channels); - LUT(images[i + 1], response, radiance); - split(radiance, next_splitted); - - for(int c = 0; c < channels; c++) { - - Mat predicted = splitted[c] / times[i] * times[i + 1]; - - Mat low = max(thresh, next_splitted[c]) - thresh; - Mat high = min(255 - thresh, next_splitted[c]) + thresh; - low.convertTo(low, CV_8U); - high.convertTo(high, CV_8U); - LUT(low, resp_split[c], low); - LUT(high, resp_split[c], high); - - Mat exposed = (splitted[c] >= underexp) & (splitted[c] <= overexp); - exposed &= (next_splitted[c] >= underexp) & (next_splitted[c] <= overexp); - - Mat ghost = (low < predicted) & (predicted < high); - ghost &= exposed; - res |= ghost; - } - splitted = next_splitted; - } - res.copyTo(dst.getMat()); - } - - virtual void process(InputArrayOfArrays src, OutputArray dst, std::vector& times) - { - Mat response = linearResponse(3); - response.at(0) = response.at(1); - process(src, dst, times, response); - } - - CV_WRAP virtual int getThreshold() {return thresh;} - CV_WRAP virtual void setThreshold(int value) {thresh = value;} - - int getUnderexp() {return underexp;} - void setUnderexp(int value) {underexp = value;} - - int getOverexp() {return overexp;} - void setOverexp(int value) {overexp = value;} - - void write(FileStorage& fs) const - { - fs << "name" << name - << "overexp" << overexp - << "underexp" << underexp - << "thresh" << thresh; - } - - void read(const FileNode& fn) - { - FileNode n = fn["name"]; - CV_Assert(n.isString() && String(n) == name); - overexp = fn["overexp"]; - underexp = fn["underexp"]; - thresh = fn["thresh"]; - } - -protected: - int thresh, underexp, overexp; - String name; -}; - -Ptr createGhostbusterPredict(int thresh, int underexp, int overexp) -{ - return new GhostbusterPredictImpl(thresh, underexp, overexp); -} - -class GhostbusterBitmapImpl : public GhostbusterBitmap -{ -public: - GhostbusterBitmapImpl(int exclude) : - exclude(exclude), - name("GhostbusterBitmap") - { - } - - void process(InputArrayOfArrays src, OutputArray dst, std::vector& times, Mat response) - { - process(src, dst); - } - - void process(InputArrayOfArrays src, OutputArray dst) - { - std::vector images; - src.getMatVector(images); - checkImageDimensions(images); - - int channels = images[0].channels(); - dst.create(images[0].size(), CV_8U); - - Mat res = Mat::zeros(images[0].size(), CV_8U); - - Ptr MTB = createAlignMTB(); - MTB->setExcludeRange(exclude); - - for(size_t i = 0; i < images.size(); i++) { - Mat gray; - if(channels == 1) { - gray = images[i]; - } else { - cvtColor(images[i], gray, COLOR_RGB2GRAY); - } - - Mat tb, eb; - MTB->computeBitmaps(gray, tb, eb); - tb &= eb & 1; - res += tb; - } - res = (res > 0) & (res < images.size()); - res.copyTo(dst.getMat()); - } - - int getExclude() {return exclude;} - void setExclude(int value) {exclude = value;} - - void write(FileStorage& fs) const - { - fs << "name" << name - << "exclude" << exclude; - } - - void read(const FileNode& fn) - { - FileNode n = fn["name"]; - CV_Assert(n.isString() && String(n) == name); - exclude = fn["exclude"]; - } - -protected: - int exclude; - String name; -}; - -Ptr createGhostbusterBitmap(int exclude) -{ - return new GhostbusterBitmapImpl(exclude); -} - }