Merge 37633553f4 into ce3c6681c9

modules/photometric_calib/CMakeLists.txt

@@ -0,0 +1,2 @@
+set(the_description "Photometric Calibration")
+ocv_define_module(photometric_calib opencv_core opencv_aruco opencv_calib3d opencv_highgui WRAP python)

modules/photometric_calib/README.md

@@ -0,0 +1,8 @@
+Photometric Calibration
+================================================
+
+Implementation of non-parametric photometric calibration algorithm proposed by J. Engel et al.:
+
+1. Camera Response Function Calibration
+2. Vignette Calibration
+3. Photometric Distortion Remover

modules/photometric_calib/doc/photometric_calib.bib

@@ -0,0 +1,9 @@
+@InProceedings{engel2016monodataset,
+  author = "J. Engel and V. Usenko and D. Cremers",
+  title = "A Photometrically Calibrated Benchmark For Monocular Visual Odometry",
+  booktitle = "arXiv:1607.02555",
+  arXiv = " arXiv:1607.02555",
+  year = "2016",
+  month = "July",
+  keywords={mono-ds,dso}
+}

modules/photometric_calib/include/opencv2/photometric_calib.hpp

@@ -0,0 +1,28 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef __OPENCV_PHOTOMETRIC_CALIB_HPP__
+#define __OPENCV_PHOTOMETRIC_CALIB_HPP__
+
+#include "opencv2/photometric_calib/Reader.hpp"
+#include "opencv2/photometric_calib/GammaRemover.hpp"
+#include "opencv2/photometric_calib/VignetteRemover.hpp"
+#include "opencv2/photometric_calib/ResponseCalib.hpp"
+#include "opencv2/photometric_calib/VignetteCalib.hpp"
+
+/**
+ * @defgroup photometric_calib Photometric Calibration
+ * The photometric_calib contains photomeric calibration algorithm proposed by Jakob Engel. \n
+ * The implementation is totally based on the paper \cite engel2016monodataset. \n
+ * Photometric calibration aimed at removing the camera response function and vitnetting artefact,
+ * by which the tracking and image alignment algorithms based on direct methods can be improved significantly. \n
+ * For details please refer to \cite engel2016monodataset.
+ */
+
+namespace cv {
+namespace photometric_calib {
+} // namespace photometric_calib
+} // namespace cv
+
+#endif

modules/photometric_calib/include/opencv2/photometric_calib/GammaRemover.hpp

@@ -0,0 +1,80 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef _OPENCV_GAMMAREMOVER_HPP
+#define _OPENCV_GAMMAREMOVER_HPP
+
+#include "opencv2/core.hpp"
+
+namespace cv {
+namespace photometric_calib {
+
+//! @addtogroup photometric_calib
+//! @{
+
+/*!
+ * @brief Class for removing the camera response function (mostly gamma function) when provided with pcalib file.
+ *
+ */
+
+class CV_EXPORTS GammaRemover
+{
+public:
+    /*!
+     * @brief Constructor
+     * @param gammaPath the path of pcalib file of which the format should be .yaml or .yml
+     * @param w_ the width of input image
+     * @param h_ the height of input image
+     */
+    GammaRemover(const std::string &gammaPath, int w_, int h_);
+
+    /*!
+     * @brief get irradiance image in the form of cv::Mat. Convenient for display, etc.
+     * @param inputIm
+     * @return
+     */
+    Mat getUnGammaImageMat(Mat inputIm);
+
+    /*!
+     * @brief get irradiance image in the form of std::vector<float>. Convenient for optimization or SLAM.
+     * @param inputIm
+     * @param outImVec
+     */
+    void getUnGammaImageVec(Mat inputIm, std::vector<float> &outImVec);
+
+    /*!
+     * @brief get gamma function.
+     * @return
+     */
+    inline float *getG()
+    {
+        if (!validGamma)
+        { return 0; }
+        else
+        { return G; }
+    };
+
+    /*!
+     * @brief get inverse gamma function
+     * @return
+     */
+    inline float *getGInv()
+    {
+        if (!validGamma)
+        { return 0; }
+        else
+        { return GInv; }
+    };
+
+private:
+    float G[256];
+    float GInv[256];
+    int w, h;
+    bool validGamma;
+};
+
+} // namespace photometric_calib
+} // namespace cv
+
+#endif //_OPENCV__GAMMAREMOVER_HPP

modules/photometric_calib/include/opencv2/photometric_calib/Reader.hpp

@@ -0,0 +1,99 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef _OPENCV_READER_HPP
+#define _OPENCV_READER_HPP
+
+#include "opencv2/core.hpp"
+
+#include <vector>
+#include <string>
+
+namespace cv {
+namespace photometric_calib {
+
+//! @addtogroup photometric_calib
+//! @{
+
+/*!
+ * @brief Class for reading the sequence used for photometric calibration. Both the folder path of the sequence
+ * and the path of time file should be provided. The images of the sequence should be of format CV_8U. The time
+ * file should be .yaml or .yml. In the time file, the timestamps and exposure duration of the corresponding images
+ * of the sequence should be provided.
+ *
+ * The image paths are stored in std::vector<String> images, timestamps are stored in std::vector<double> timeStamps,
+ * exposure duration is stored in std::vector<float> exposureTimes
+ */
+
+class CV_EXPORTS Reader
+{
+public:
+    /*!
+     * @brief Constructor
+     * @param folderPath the path of folder which contains the images
+     * @param imageExt the format of the input images, e.g., jpg or png.
+     * @param timesPath the path of time file
+     */
+    Reader(const std::string &folderPath, const std::string &imageExt, const std::string &timesPath);
+
+    /*!
+     * @return the amount of images loaded
+     */
+    unsigned long getNumImages() const;
+
+
+    /*!
+     * @brief Given the id of the image and return the image. id is in fact just the index of the image in the
+     * vector contains all the images.
+     * @param id
+     * @return Mat of the id^th image.
+     */
+    Mat getImage(unsigned long id) const;
+
+    /*!
+     * @brief Given the id of the image and return its timestamp value. id is in fact just the index of the image in the
+     * vector contains all the images.
+     * @param id
+     * @return timestamp of the id^th image.
+     */
+    double getTimestamp(unsigned long id) const;
+
+    /*!
+     * @brief Given the id of the image and return its exposure duration when is was taken.
+     * @param id
+     * @return exposure duration of the image.
+     */
+    float getExposureDuration(unsigned long id) const;
+
+    int getWidth() const;
+
+    int getHeight() const;
+
+    const std::string &getFolderPath() const;
+
+    const std::string &getTimeFilePath() const;
+
+private:
+    /*!
+     * @brief Load timestamps and exposure duration.
+     * @param timesFile
+     */
+    inline void loadTimestamps(const std::string &timesFile);
+
+    std::vector<String> images; //All the names/paths of images
+    std::vector<double> timeStamps; //All the Unix Time Stamps of images
+    std::vector<float> exposureDurations;//All the exposure duration for images
+
+    int _width, _height;//The image width and height. All the images should be of the same size.
+
+    std::string _folderPath;
+    std::string _timeFilePath;
+};
+
+//! @}
+
+} // namespace photometric_calib
+} // namespace cv
+
+#endif //_OPENCV_READER_HPP

modules/photometric_calib/include/opencv2/photometric_calib/ResponseCalib.hpp

@@ -0,0 +1,52 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef _OPENCV_RESPONSECALIB_HPP
+#define _OPENCV_RESPONSECALIB_HPP
+
+#include "opencv2/photometric_calib/Reader.hpp"
+
+namespace cv {
+namespace photometric_calib {
+
+class CV_EXPORTS ResponseCalib
+{
+public:
+    ResponseCalib(std::string folderPath, std::string timePath, std::string imageFormat);
+
+    ResponseCalib(std::string folderPath, std::string timePath, int leakPadding, int nIts, int skipFrames,
+                  std::string imageFormat);
+
+    void plotE(const double *E, int w, int h, const std::string &saveTo);
+
+    Vec2d rmse(const double *G, const double *E, const std::vector<double> &exposureVec,
+               const std::vector<unsigned char *> &dataVec, int wh);
+
+    void plotG(const double *G, const std::string &saveTo);
+
+    void calib(bool debug);
+
+    inline const std::string &getImageFolderPath() const
+    {
+        CV_Assert(imageReader);
+        return imageReader->getFolderPath();
+    }
+
+    inline const std::string &getTimeFilePath() const
+    {
+        CV_Assert(imageReader);
+        return imageReader->getTimeFilePath();
+    }
+
+private:
+    int _leakPadding;
+    int _nIts;
+    int _skipFrames;
+    Reader *imageReader;
+};
+
+} // namespace photometric_calib
+} // namespace cv
+
+#endif //_OPENCV_RESPONSECALIB_HPP

modules/photometric_calib/include/opencv2/photometric_calib/VignetteCalib.hpp

@@ -0,0 +1,73 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef _OPENCV_VIGNETTECALIB_HPP
+#define _OPENCV_VIGNETTECALIB_HPP
+
+#include "opencv2/core.hpp"
+#include "opencv2/photometric_calib/Reader.hpp"
+#include "opencv2/photometric_calib/GammaRemover.hpp"
+
+namespace cv {
+namespace photometric_calib {
+
+
+class CV_EXPORTS VignetteCalib
+{
+public:
+    VignetteCalib(std::string folderPath, std::string timePath, std::string cameraFile, std::string gammaFile,
+                  std::string imageFormat);
+
+    VignetteCalib(std::string folderPath, std::string timePath, std::string cameraFile, std::string gammaFile,
+                  std::string imageFormat, int imageSkip, int maxIterations, int outlierTh,
+                  int gridWidth, int gridHeight, float facW, float facH, int maxAbsGrad);
+
+    virtual ~VignetteCalib();
+
+    //EIGEN_ALWAYS_INLINE float getInterpolatedElement(const float* const mat, const float x, const float y, const int width)
+    float getInterpolatedElement(const float *const mat, const float x, const float y, const int width);
+
+    float calMeanExposureTime();
+
+    void displayImage(float *I, int w, int h, std::string name);
+
+    void displayImageV(float *I, int w, int h, std::string name);
+
+    bool preCalib(unsigned long id, float *&image, float *&plane2imgX, float *&plane2imgY, bool debug);
+
+    void calib(bool debug);
+
+    void calibFast(bool debug);
+
+private:
+    int _imageSkip;
+    int _maxIterations;
+    int _outlierTh;
+
+    // grid width for template image.
+    int _gridWidth;
+    int _gridHeight;
+
+    // width of grid relative to marker (fac times marker size)
+    float _facW;
+    float _facH;
+
+    // remove pixel with absolute gradient larger than this from the optimization.
+    int _maxAbsGrad;
+
+    Mat _cameraMatrix;
+    Mat _distCoeffs;
+    Matx33f _K_p2idx;
+    Matx33f _K_p2idx_inverse;
+
+    Reader *imageReader;
+    GammaRemover *gammaRemover;
+
+    float _meanExposure;
+};
+
+} // namespace photometric_calib
+} // namespace cv
+
+#endif //_OPENCV_VIGNETTECALIB_HPP

modules/photometric_calib/include/opencv2/photometric_calib/VignetteRemover.hpp

@@ -0,0 +1,59 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef _OPENCV_VIGNETTEREMOVER_HPP
+#define _OPENCV_VIGNETTEREMOVER_HPP
+
+#include "opencv2/core.hpp"
+
+namespace cv {
+namespace photometric_calib {
+
+//! @addtogroup photometric_calib
+//! @{
+
+/*!
+ * @brief Class for removing the vignetting artifact when provided with vignetting file.
+ *
+ */
+
+class CV_EXPORTS VignetteRemover
+{
+public:
+    /*!
+     * @brief Constructor
+     * @param vignettePath the path of vignetting file
+     * @param pcalibPath the path of pcalib file
+     * @param w_ the width of input image
+     * @param h_ the height of input image
+     */
+    VignetteRemover(const std::string &vignettePath, const std::string &pcalibPath, int w_, int h_);
+
+    ~VignetteRemover();
+
+    /*!
+     * @brief get vignetting-removed image in form of cv::Mat.
+     * @param oriImMat the image to be calibrated.
+     */
+    Mat getUnVignetteImageMat(Mat oriImMat);
+
+    /*!
+     * @brief get vignetting-removed image in form of std::vector<float>.
+     * @param oriImMat the image to be calibrated.
+     * @param outImVec the vignetting-removed image vector.
+     */
+    void getUnVignetteImageVec(Mat oriImMat, std::vector<float> &outImVec);
+
+private:
+    float *vignetteMap;
+    float *vignetteMapInv;
+    std::string _pcalibPath;
+    int w, h;
+    bool validVignette;
+};
+
+} // namespace photometric_calib
+} // namespace cv
+
+#endif //_OPENCV_VIGNETTEREMOVER_HPP

modules/photometric_calib/samples/response_calibration.cpp

@@ -0,0 +1,52 @@
+#include "opencv2/core.hpp"
+#include "opencv2/highgui.hpp"
+#include "opencv2/photometric_calib.hpp"
+
+using namespace std;
+using namespace cv;
+
+int main()
+{
+    // Please down load the sample dataset from:
+    // https://www.dropbox.com/s/5x48uhc7k2bgjcj/GSoC2017_PhotometricCalib_Sample_Data.zip?dl=0
+    // By unzipping the file, you would get a folder named /GSoC2017_PhotometricCalib_Sample_Data which contains 4 subfolders:
+    // response_calib, response remover, vignette_calib, vignette_remover
+    // in this sample, we will use the data in the folder response_calib and response_remover
+
+    // Prefix for the data, e.g. /Users/Yelen/GSoC2017_PhotometricCalib_Sample
+    string userPrefix = "/Users/Yelen/GSoC2017_PhotometricCalib_Sample_Data/";
+    // The path for the images used for response calibration
+    string imageFolderPath = userPrefix + "response_calib/images";
+    // The yaml file which contains the timestamps and exposure times for each image used for camera response calibration
+    string timePath = userPrefix + "response_calib/times.yaml";
+
+    // Construct a photometric_calib::ResponseCalib object by giving path of image, path of time file and specify the format of images
+    photometric_calib::ResponseCalib resCal(imageFolderPath, timePath, "jpg");
+
+    // Debug mode will generate some temporary data
+    bool debug = true;
+    // Calibration of camera response function begins
+    resCal.calib(debug);
+
+    // The result and some intermediate data are stored in the folder ./photoCalibResult in which
+    // pcalib.yaml is the camera response function file
+    // Since we are using debug mode, we can visualize the response function:
+    Mat invRes = imread("./photoCalibResult/G-10.png", IMREAD_UNCHANGED);
+    // As shown as Fig.3 in the paper from J.Engel, et al. in the paper A Photometrically Calibrated Benchmark For Monocular Visual Odometry
+    namedWindow( "Inverse Response Function", WINDOW_AUTOSIZE );
+    imshow("Inverse Response Function", invRes);
+
+    // To see the response-calibrated image, we can use GammaRemover
+    Mat oriImg = imread(imageFolderPath + "/00480.jpg", IMREAD_UNCHANGED);
+    photometric_calib::GammaRemover gammaRemover("./photoCalibResult/pcalib.yaml", oriImg.cols, oriImg.rows);
+    Mat caliImg = gammaRemover.getUnGammaImageMat(oriImg);
+
+    // Visualization
+    namedWindow( "Original Image", WINDOW_AUTOSIZE );
+    imshow("Original Image", oriImg);
+    namedWindow( "Gamma Removed Image", WINDOW_AUTOSIZE );
+    imshow("Gamma Removed Image", caliImg);
+    waitKey(0);
+
+    return 0;
+}

modules/photometric_calib/samples/vignette_calibration.cpp

@@ -0,0 +1,67 @@
+#include "opencv2/core.hpp"
+#include "opencv2/highgui.hpp"
+#include "opencv2/photometric_calib.hpp"
+
+using namespace std;
+using namespace cv;
+
+int main()
+{
+    // Please down load the sample dataset from:
+    // https://www.dropbox.com/s/5x48uhc7k2bgjcj/GSoC2017_PhotometricCalib_Sample_Data.zip?dl=0
+    // By unzipping the file, you would get a folder named /GSoC2017_PhotometricCalib_Sample_Data which contains 2 subfolders:
+    // response_calib, vignette_calib
+    // in this sample, we will use the data in the folder vignette_calib
+
+    // Prefix for the data, e.g. /Users/Yelen/GSoC2017_PhotometricCalib_Sample
+    string userPrefix = "/Users/Yelen/GSoC2017_PhotometricCalib_Sample_Data/";
+    // The path for the images used for response calibration
+    string imageFolderPath = userPrefix + "vignette_calib/images";
+    // The yaml file which contains the timestamps and exposure times for each image used for vignette calibration
+    string timePath = userPrefix + "vignette_calib/times.yaml";
+    // The yaml file which contains the camera intrinsics and extrinsics.
+    // Note that the images are already rectified, so the distortion parameters are 0s
+    string cameraPath = userPrefix + "vignette_calib/camera.yaml";
+    // The pcalib file. Vignette calibration can be performed only when provided with pcalib file.
+    // We use the identical pcalib.yaml file generated by response_calibration.cpp
+    // You can refer to the code in response_calibration.cpp for details
+    string gammaPath = userPrefix + "vignette_calib/pcalib.yaml";
+
+    // Construct a photometric_calib::VignetteCalib object by giving path of image, path of time file, camera parameter file, pcalib file and specify the format of images
+    photometric_calib::VignetteCalib vigCal(imageFolderPath, timePath, cameraPath, gammaPath, "jpg");
+
+    // Debug mode will visualize the optimization process and generate some temporary data
+    bool debug = true;
+    // Calibration of camera response function begins
+    vigCal.calib(debug);
+
+    // You can also use fast mode, but with much memory (potentially with 10GB+)
+    // vigCal.calibFast(debug);
+
+    // The result and some intermediate data are stored in the folder ./vignetteCalibResult in which
+    // vignette.png and vignetteSmoothed.png are the vignette images.
+    // In practice, vignetteSomoothed.png is used, since it doesn't have the black boarders.
+    Mat vigSmoothed = imread("./vignetteCalibResult/vignetteSmoothed.png", IMREAD_UNCHANGED);
+    // As shown as Fig.4 in the paper from J.Engel, et al. in the paper A Photometrically Calibrated Benchmark For Monocular Visual Odometry
+    namedWindow( "Vignette Smoothed", WINDOW_AUTOSIZE );
+    imshow("Vignette Smoothed", vigSmoothed);
+
+    // To see the vignette-calibrated image, we can use VignetteRemover
+    Mat oriImg = imread(imageFolderPath + "/00480.jpg", IMREAD_UNCHANGED);
+    photometric_calib::GammaRemover gammaRemover(gammaPath, oriImg.cols, oriImg.rows);
+    photometric_calib::VignetteRemover vignetteRemover("./vignetteCalibResult/vignetteSmoothed.png", gammaPath, oriImg.cols, oriImg.rows);
+    Mat resCaliImg = gammaRemover.getUnGammaImageMat(oriImg);
+    Mat vigCaliImg = vignetteRemover.getUnVignetteImageMat(oriImg);
+
+    // Visualization
+    namedWindow( "Original Image", WINDOW_AUTOSIZE );
+    imshow("Original Image", oriImg);
+    namedWindow( "Gamma Removed Image", WINDOW_AUTOSIZE );
+    imshow("Gamma Removed Image", resCaliImg);
+    namedWindow( "Vignette Removed Image", WINDOW_AUTOSIZE );
+    imshow("Vignette Removed Image", vigCaliImg);
+
+    waitKey(0);
+
+    return 0;
+}

modules/photometric_calib/src/GammaRemover.cpp

@@ -0,0 +1,84 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "precomp.hpp"
+#include "opencv2/photometric_calib/GammaRemover.hpp"
+
+namespace cv {
+namespace photometric_calib {
+
+GammaRemover::GammaRemover(const std::string &gammaPath, int w_, int h_)
+{
+    validGamma = false;
+    w = w_;
+    h = h_;
+
+    // check the extension of the time file.
+    CV_Assert(gammaPath.substr(gammaPath.find_last_of(".") + 1) == "yaml" ||
+              gammaPath.substr(gammaPath.find_last_of(".") + 1) == "yml");
+
+    FileStorage gammaFile;
+    gammaFile.open(gammaPath, FileStorage::READ);
+    CV_Assert(gammaFile.isOpened());
+
+    FileNode gammaNode = gammaFile["gamma"];
+    CV_Assert(gammaNode.type() == FileNode::SEQ);
+    FileNodeIterator itS = gammaNode.begin(), itE = gammaNode.end();
+    std::vector<float> GInvVec;
+    for (; itS != itE; ++itS)
+    {
+        GInvVec.push_back((float) *itS);
+    }
+    CV_Assert(GInvVec.size() == 256);
+
+    for (int i = 0; i < 256; i++) GInv[i] = GInvVec[i];
+    for (int i = 0; i < 255; i++)
+    {
+        CV_Assert(GInv[i + 1] > GInv[i]);
+    }
+    float min = GInv[0];
+    float max = GInv[255];
+    for (int i = 0; i < 256; i++) GInv[i] = (float) (255.0 * (GInv[i] - min) / (max - min));
+    for (int i = 1; i < 255; i++)
+    {
+        for (int s = 1; s < 255; s++)
+        {
+            if (GInv[s] <= i && GInv[s + 1] >= i)
+            {
+                G[i] = s + (i - GInv[s]) / (GInv[s + 1] - GInv[s]);
+                break;
+            }
+        }
+    }
+    G[0] = 0;
+    G[255] = 255;
+    gammaFile.release();
+    validGamma = true;
+}
+
+Mat GammaRemover::getUnGammaImageMat(Mat inputIm)
+{
+    CV_Assert(validGamma);
+    uchar *inputImArr = inputIm.data;
+    float *outImArr = new float[w * h];
+    for (int i = 0; i < w * h; ++i)
+    {
+        outImArr[i] = GInv[inputImArr[i]];
+    }
+    Mat _outIm(h, w, CV_32F, outImArr);
+    Mat outIm = _outIm * (1 / 255.0f);
+    delete[] outImArr;
+    return outIm;
+}
+
+void GammaRemover::getUnGammaImageVec(Mat inputIm, std::vector<float> &outImVec)
+{
+    CV_Assert(validGamma);
+    uchar *inputImArr = inputIm.data;
+    CV_Assert(outImVec.size() == (unsigned long) w * h);
+    for (int i = 0; i < w * h; i++) outImVec[i] = GInv[inputImArr[i]];
+}
+
+} // namespace photometric_calib
+} // namespace cv

modules/photometric_calib/src/Reader.cpp

@@ -0,0 +1,129 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "precomp.hpp"
+#include "opencv2/photometric_calib/Reader.hpp"
+
+namespace cv {
+namespace photometric_calib {
+
+unsigned long Reader::getNumImages() const
+{
+    return (unsigned long) images.size();
+}
+
+void Reader::loadTimestamps(const std::string &timesFile)
+{
+    // check the extension of the time file.
+    CV_Assert(timesFile.substr(timesFile.find_last_of(".") + 1) == "yaml" ||
+              timesFile.substr(timesFile.find_last_of(".") + 1) == "yml");
+
+    FileStorage timeFile;
+    timeFile.open(timesFile, FileStorage::READ);
+    timeStamps.clear();
+    exposureDurations.clear();
+
+    CV_Assert(timeFile.isOpened());
+
+    FileNode timeStampNode = timeFile["times"];
+    FileNode exposureTimeNode = timeFile["exposures"];
+
+    CV_Assert(timeStampNode.type() == FileNode::SEQ && exposureTimeNode.type() == FileNode::SEQ);
+
+    FileNodeIterator itTs = timeStampNode.begin(), itTsEnd = timeStampNode.end();
+    FileNodeIterator itEt = exposureTimeNode.begin(), itEtEnd = exposureTimeNode.end();
+
+    for (; itTs != itTsEnd; ++itTs)
+    {
+        timeStamps.push_back((double) *itTs);
+    }
+    for (; itEt != itEtEnd; ++itEt)
+    {
+        exposureDurations.push_back((float) *itEt);
+    }
+
+    timeFile.release();
+
+    CV_Assert(timeStamps.size() == getNumImages() && exposureDurations.size() == getNumImages());
+    _timeFilePath = timesFile;
+}
+
+Reader::Reader(const std::string &folderPath, const std::string &imageExt, const std::string &timesPath)
+{
+    String cvFolderPath(folderPath);
+
+#if defined WIN32 || defined _WIN32 || defined WINCE
+    *cvFolderPath.end() == '\\' ? cvFolderPath = cvFolderPath : cvFolderPath += '\\';
+#else
+    *cvFolderPath.end() == '/' ? cvFolderPath = cvFolderPath : cvFolderPath += '/';
+#endif
+
+    cvFolderPath += ("*." + imageExt);
+    glob(cvFolderPath, images);
+    CV_Assert(images.size() > 0);
+    std::sort(images.begin(), images.end());
+    loadTimestamps(timesPath);
+
+    _width = 0;
+    _height = 0;
+
+    // images should be of CV_8U and same size
+    for (size_t i = 0; i < images.size(); ++i)
+    {
+        Mat img = imread(images[i], IMREAD_GRAYSCALE);
+        CV_Assert(img.type() == CV_8U);
+        if (i == 0)
+        {
+            _width = img.cols;
+            _height = img.rows;
+        }
+        else
+        {
+            CV_Assert(_width == img.cols && _height == img.rows);
+        }
+    }
+
+    _folderPath = folderPath;
+}
+
+Mat Reader::getImage(unsigned long id) const
+{
+    CV_Assert(id < images.size());
+    return imread(images[id], IMREAD_GRAYSCALE);
+}
+
+double Reader::getTimestamp(unsigned long id) const
+{
+    CV_Assert(id < timeStamps.size());
+    return timeStamps[id];
+}
+
+float Reader::getExposureDuration(unsigned long id) const
+{
+    CV_Assert(id < exposureDurations.size());
+    return exposureDurations[id];
+}
+
+int Reader::getWidth() const
+{
+    return _width;
+}
+
+int Reader::getHeight() const
+{
+    return _height;
+}
+
+const std::string &Reader::getFolderPath() const
+{
+    return _folderPath;
+}
+
+const std::string &Reader::getTimeFilePath() const
+{
+    return _timeFilePath;
+}
+
+} // namespace photometric_calib
+} // namespace cv

modules/photometric_calib/src/ResponseCalib.cpp

@@ -0,0 +1,367 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "precomp.hpp"
+#include "opencv2/photometric_calib/ResponseCalib.hpp"
+
+#include <fstream>
+#include <iostream>
+#include <math.h>
+
+namespace cv {
+namespace photometric_calib {
+
+ResponseCalib::ResponseCalib(std::string folderPath, std::string timePath, std::string imageFormat) : _leakPadding(2),
+                                                                                                      _nIts(10),
+                                                                                                      _skipFrames(1)
+{
+    imageReader = new Reader(folderPath, imageFormat, timePath);
+}
+
+ResponseCalib::ResponseCalib(std::string folderPath, std::string timePath, int leakPadding, int nIts, int skipFrames,
+                             std::string imageFormat) :
+        _leakPadding(leakPadding), _nIts(nIts), _skipFrames(skipFrames)
+{
+    imageReader = new Reader(folderPath, imageFormat, timePath);
+}
+
+Vec2d ResponseCalib::rmse(const double *G, const double *E, const std::vector<double> &exposureVec,
+                          const std::vector<uchar *> &dataVec,
+                          int wh)
+{
+    long double e = 0;        // yeah - these will be sums of a LOT of values, so we need super high precision.
+    long double num = 0;
+
+    size_t n = dataVec.size();
+    for (size_t i = 0; i < n; i++)
+    {
+        for (int k = 0; k < wh; k++)
+        {
+            if (dataVec[i][k] == 255) continue;
+            double r = G[dataVec[i][k]] - exposureVec[i] * E[k];
+            if (!std::isfinite(r)) continue;
+            e += r * r * 1e-10;
+            num++;
+        }
+    }
+
+    //return Eigen::Vector2d(1e5*sqrtl((e/num)), (double)num);
+    return Vec2d((double) (1e5 * sqrt((e / num))), (double) num);
+}
+
+void ResponseCalib::plotE(const double *E, int w, int h, const std::string &saveTo)
+{
+
+    // try to find some good color scaling for plotting.
+    double offset = 20;
+    double min = 1e10, max = -1e10;
+
+    double Emin = 1e10, Emax = -1e10;
+
+    for (int i = 0; i < w * h; i++)
+    {
+        double le = log(E[i] + offset);
+        if (le < min) min = le;
+        if (le > max) max = le;
+
+        if (E[i] < Emin) Emin = E[i];
+        if (E[i] > Emax) Emax = E[i];
+    }
+
+    cv::Mat EImg = cv::Mat(h, w, CV_8UC3);
+    cv::Mat EImg16 = cv::Mat(h, w, CV_16U);
+
+    for (int i = 0; i < w * h; i++)
+    {
+        float val = (float) (3 * (exp((log(E[i] + offset) - min) / (max - min)) - 1) / 1.7183);
+
+        int icP = (int) val;
+        float ifP = val - icP;
+        icP = icP % 3;
+
+        Vec3b color;
+        if (icP == 0) color = cv::Vec3b(0, 0, (uchar) (255 * ifP));
+        if (icP == 1) color = cv::Vec3b(0, (uchar) (255 * ifP), 255);
+        if (icP == 2) color = cv::Vec3b((uchar) (255 * ifP), 255, 255);
+
+        EImg.at<cv::Vec3b>(i) = color;
+        EImg16.at<ushort>(i) = (ushort) (255 * 255 * (E[i] - Emin) / (Emax - Emin));
+    }
+
+    std::cout << "Irradiance " << Emin << " - " << Emax << std::endl;
+    cv::imshow("lnE", EImg);
+
+    if (!saveTo.empty())
+    {
+        imwrite(saveTo + ".png", EImg);
+        std::cout << "Saved: " << saveTo + ".png" << std::endl;
+        imwrite(saveTo + "-16.png", EImg16);
+        std::cout << "Saved: " << saveTo + "-16.png" << std::endl;
+    }
+}
+
+void ResponseCalib::plotG(const double *G, const std::string &saveTo)
+{
+    cv::Mat GImg = cv::Mat(256, 256, CV_32FC1);
+    GImg.setTo(0);
+
+    double min = 1e10, max = -1e10;
+
+    for (int i = 0; i < 256; i++)
+    {
+        if (G[i] < min) min = G[i];
+        if (G[i] > max) max = G[i];
+    }
+
+    for (int i = 0; i < 256; i++)
+    {
+        double val = 256 * (G[i] - min) / (max - min);
+        for (int k = 0; k < 256; k++)
+        {
+            if (val < k)
+            {
+                GImg.at<float>(255 - k, i) = (float) (k - val);
+            }
+        }
+    }
+
+    std::cout << "Inv. Response " << min << " - " << max << std::endl;
+    cv::imshow("G", GImg);
+    if (!saveTo.empty()) cv::imwrite(saveTo, GImg * 255);
+    std::cout << "Saved: " << saveTo << std::endl;
+}
+
+void ResponseCalib::calib(bool debug)
+{
+    int w = 0, h = 0;
+    size_t n = 0;
+
+    std::vector<double> exposureDurationVec;
+    std::vector<uchar *> dataVec;
+
+    std::cout << "Preprocessing for response calibration... " << std::endl;
+    for (unsigned long i = 0; i < imageReader->getNumImages(); i += _skipFrames)
+    {
+        cv::Mat img = imageReader->getImage(i);
+        if (img.rows == 0 || img.cols == 0) continue;
+        CV_Assert(img.type() == CV_8U);
+
+        if ((w != 0 && w != img.cols) || img.cols == 0)
+        {
+            std::cout << "Width mismatch!" << std::endl;
+            exit(1);
+        }
+        if ((h != 0 && h != img.rows) || img.rows == 0)
+        {
+            std::cout << "Height mismatch!" << std::endl;
+            exit(1);
+        }
+        w = img.cols;
+        h = img.rows;
+
+        uchar *data = new uchar[w * h];
+        memcpy(data, img.data, w * h);
+        dataVec.push_back(data);
+        exposureDurationVec.push_back((double) (imageReader->getExposureDuration(i)));
+
+        unsigned char *data2 = new unsigned char[w * h];
+        for (int j = 0; j < _leakPadding; ++j)
+        {
+            memcpy(data2, data, w * h);
+            for (int y = 1; y < h - 1; ++y)
+            {
+                for (int x = 1; x < w - 1; ++x)
+                {
+                    if (data[x + y * w] == 255)
+                    {
+                        data2[x + 1 + w * (y + 1)] = 255;
+                        data2[x + 1 + w * (y)] = 255;
+                        data2[x + 1 + w * (y - 1)] = 255;
+
+                        data2[x + w * (y + 1)] = 255;
+                        data2[x + w * (y)] = 255;
+                        data2[x + w * (y - 1)] = 255;
+
+                        data2[x - 1 + w * (y + 1)] = 255;
+                        data2[x - 1 + w * (y)] = 255;
+                        data2[x - 1 + w * (y - 1)] = 255;
+                    }
+                }
+            }
+            memcpy(data, data2, w * h);
+        }
+        delete[] data2;
+    }
+    n = dataVec.size();
+    std::cout << "Loaded " << n << " images!" << std::endl;
+    std::cout << "Response calibration begin!" << std::endl;
+
+    double *E = new double[w * h];        // scene irradiance
+    double *En = new double[w * h];        // scene irradiance
+    double *G = new double[256];        // inverse response function
+
+    // set starting scene irradiance to mean of all images.
+    memset(E, 0, sizeof(double) * w * h);
+    memset(En, 0, sizeof(double) * w * h);
+    memset(G, 0, sizeof(double) * 256);
+
+    for (size_t i = 0; i < n; i++)
+    {
+        for (int k = 0; k < w * h; k++)
+        {
+            //if(dataVec[i][k]==255) continue;
+            E[k] += dataVec[i][k];
+            En[k]++;
+        }
+    }
+    for (int k = 0; k < w * h; k++)
+    {
+        E[k] = E[k] / En[k];
+    }
+
+    // TODO: System independent folder creating
+    // Only on Linux for now.
+    if (-1 == system("rm -rf photoCalibResult"))
+    {
+        std::cout << "could not delete old photoCalibResult folder!" << std::endl;
+    }
+    if (-1 == system("mkdir photoCalibResult"))
+    {
+        std::cout << "could not create photoCalibResult folder!" << std::endl;
+    }
+
+    std::ofstream logFile;
+    logFile.open("photoCalibResult/log.txt", std::ios::trunc | std::ios::out);
+    logFile.precision(15);
+
+    std::cout << "Initial RMSE = " << rmse(G, E, exposureDurationVec, dataVec, w * h)[0] << "!" << std::endl;
+    if (debug)
+    {
+        plotE(E, w, h, "photoCalibResult/E-0");
+        cv::waitKey(100);
+    }
+
+    bool optE = true;
+    bool optG = true;
+
+    for (int it = 0; it < _nIts; it++)
+    {
+        std::cout << "Iteration " << it + 1 << "..." << std::endl;
+        if (optG)
+        {
+            // optimize log inverse response function.
+            double *GSum = new double[256];
+            double *GNum = new double[256];
+            memset(GSum, 0, 256 * sizeof(double));
+            memset(GNum, 0, 256 * sizeof(double));
+            for (size_t i = 0; i < n; i++)
+            {
+                for (int k = 0; k < w * h; k++)
+                {
+                    int b = dataVec[i][k];
+                    if (b == 255) continue;
+                    GNum[b]++;
+                    GSum[b] += E[k] * exposureDurationVec[i];
+                }
+            }
+            for (int i = 0; i < 256; i++)
+            {
+                G[i] = GSum[i] / GNum[i];
+                if (!std::isfinite(G[i]) && i > 1) G[i] = G[i - 1] + (G[i - 1] - G[i - 2]);
+            }
+            delete[] GSum;
+            delete[] GNum;
+            printf("optG RMSE = %f! \t", rmse(G, E, exposureDurationVec, dataVec, w * h)[0]);
+
+            if (debug)
+            {
+                char buf[1000];
+                snprintf(buf, 1000, "photoCalibResult/G-%02d.png", it + 1);
+                plotG(G, buf);
+            }
+        }
+
+        if (optE)
+        {
+            // optimize scene irradiance function.
+            double *ESum = new double[w * h];
+            double *ENum = new double[w * h];
+            memset(ESum, 0, w * h * sizeof(double));
+            memset(ENum, 0, w * h * sizeof(double));
+            for (size_t i = 0; i < n; i++)
+            {
+                for (int k = 0; k < w * h; k++)
+                {
+                    int b = dataVec[i][k];
+                    if (b == 255) continue;
+                    ENum[k] += exposureDurationVec[i] * exposureDurationVec[i];
+                    ESum[k] += (G[b]) * exposureDurationVec[i];
+                }
+            }
+            for (int i = 0; i < w * h; i++)
+            {
+                E[i] = ESum[i] / ENum[i];
+                if (E[i] < 0) E[i] = 0;
+            }
+
+            delete[] ENum;
+            delete[] ESum;
+            printf("OptE RMSE = %f!  \t", rmse(G, E, exposureDurationVec, dataVec, w * h)[0]);
+
+            if (debug)
+            {
+                char buf[1000];
+                snprintf(buf, 1000, "photoCalibResult/E-%02d", it + 1);
+                plotE(E, w, h, buf);
+            }
+        }
+
+        // rescale such that maximum response is 255 (fairly arbitrary choice).
+        double rescaleFactor = 255.0 / G[255];
+        for (int i = 0; i < w * h; i++)
+        {
+            E[i] *= rescaleFactor;
+            if (i < 256) G[i] *= rescaleFactor;
+        }
+
+        Vec2d err = rmse(G, E, exposureDurationVec, dataVec, w * h);
+        printf("Rescaled RMSE = %f!  \trescale with %f!\n\n", err[0], rescaleFactor);
+
+        logFile << it << " " << n << " " << err[1] << " " << err[0] << "\n";
+
+        cv::waitKey(100);
+    }
+
+    logFile.flush();
+    logFile.close();
+
+    std::ofstream lg;
+    lg.open("photoCalibResult/pcalib.yaml", std::ios::trunc | std::ios::out);
+    lg << "%YAML:1.0\ngamma: [";
+    lg.precision(15);
+    for (int i = 0; i < 255; i++)
+    {
+        lg << G[i] << ", ";
+    }
+    lg << G[255] << ']';
+    lg << "\n";
+
+    lg.flush();
+    lg.close();
+
+    std::cout << "pcalib file has been saved to: photoCalibResult/pcalib.yaml" << std::endl;
+
+    delete[] E;
+    delete[] En;
+    delete[] G;
+    for (size_t i = 0; i < n; i++)
+    {
+        delete[] dataVec[i];
+    }
+
+    std::cout << "Camera response function calibration finished!" << std::endl;
+}
+
+} // namespace photometric_calib
+} // namespace cv

modules/photometric_calib/src/VignetteRemover.cpp

@@ -0,0 +1,105 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "precomp.hpp"
+#include "opencv2/photometric_calib/VignetteRemover.hpp"
+#include "opencv2/photometric_calib/GammaRemover.hpp"
+
+namespace cv {
+namespace photometric_calib {
+
+VignetteRemover::VignetteRemover(const std::string &vignettePath, const std::string &pcalibPath, int w_, int h_)
+{
+    CV_Assert(vignettePath != "");
+    CV_Assert(pcalibPath != "");
+
+    validVignette = false;
+    vignetteMap = 0;
+    vignetteMapInv = 0;
+    w = w_;
+    h = h_;
+
+
+    Mat vignetteMat = imread(vignettePath, IMREAD_UNCHANGED);
+    vignetteMap = new float[w * h];
+    vignetteMapInv = new float[w * h];
+
+    CV_Assert(vignetteMat.rows == h && vignetteMat.cols == w);
+
+    CV_Assert(vignetteMat.type() == CV_8U || vignetteMat.type() == CV_16U);
+    if (vignetteMat.type() == CV_8U)
+    {
+        float maxV = 0;
+        for (int i = 0; i < w * h; i++)
+        {
+            if (vignetteMat.at<unsigned char>(i) > maxV) maxV = vignetteMat.at<unsigned char>(i);
+        }
+
+        for (int i = 0; i < w * h; i++)
+        {
+            vignetteMap[i] = vignetteMat.at<unsigned char>(i) / maxV;
+        }
+    }
+
+    else
+    {
+        float maxV = 0;
+        for (int i = 0; i < w * h; i++)
+        {
+            if (vignetteMat.at<ushort>(i) > maxV) maxV = vignetteMat.at<ushort>(i);
+        }
+
+        for (int i = 0; i < w * h; i++)
+        {
+            vignetteMap[i] = vignetteMat.at<ushort>(i) / maxV;
+        }
+    }
+
+    for (int i = 0; i < w * h; i++)
+    {
+        vignetteMapInv[i] = 1.0f / vignetteMap[i];
+    }
+
+    _pcalibPath = pcalibPath;
+    validVignette = true;
+
+}
+
+Mat VignetteRemover::getUnVignetteImageMat(Mat oriImMat)
+{
+    std::vector<float> _outImVec(w * h);
+    getUnVignetteImageVec(oriImMat, _outImVec);
+
+    Mat _outIm(h, w, CV_32F, &_outImVec[0]);
+    Mat outIm = _outIm * (1 / 255.0f);
+    return outIm;
+}
+
+void VignetteRemover::getUnVignetteImageVec(Mat oriImMat, std::vector<float> &outImVec)
+{
+    CV_Assert(validVignette);
+    CV_Assert(outImVec.size() == (unsigned long) w * h);
+    photometric_calib::GammaRemover gammaRemover(_pcalibPath, w, h);
+    std::vector<float> unGammaImVec(w * h);
+    gammaRemover.getUnGammaImageVec(oriImMat, unGammaImVec);
+    for (int i = 0; i < w * h; ++i)
+    {
+        outImVec[i] = unGammaImVec[i] * vignetteMapInv[i];
+    }
+}
+
+VignetteRemover::~VignetteRemover()
+{
+    if (vignetteMap != 0)
+    {
+        delete[] vignetteMap;
+    }
+    if (vignetteMapInv != 0)
+    {
+        delete[] vignetteMapInv;
+    }
+}
+
+} // namespace photometric_calib
+} // namespace cv

modules/photometric_calib/src/photometric_calib.cpp

@@ -0,0 +1,10 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "precomp.hpp"
+
+namespace cv {
+namespace photometric_calib {
+}
+} // namespace photometric_calib, cv

modules/photometric_calib/src/precomp.hpp

@@ -0,0 +1,15 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef __OPENCV_PRECOMP_H__
+#define __OPENCV_PRECOMP_H__
+
+#include "opencv2/core.hpp"
+#include "opencv2/highgui.hpp"
+#include "opencv2/imgproc.hpp"
+#include "opencv2/aruco.hpp"
+#include "opencv2/calib3d.hpp"
+#include <vector>
+
+#endif

modules/photometric_calib/test/test_main.cpp

@@ -0,0 +1,3 @@
+#include "test_precomp.hpp"
+
+CV_TEST_MAIN("")

modules/photometric_calib/test/test_precomp.hpp

@@ -0,0 +1,18 @@
+#ifdef __GNUC__
+#  pragma GCC diagnostic ignored "-Wmissing-declarations"
+#  if defined __clang__ || defined __APPLE__
+#    pragma GCC diagnostic ignored "-Wmissing-prototypes"
+#    pragma GCC diagnostic ignored "-Wextra"
+#  endif
+#endif
+
+#ifndef __OPENCV_TEST_PRECOMP_HPP__
+#define __OPENCV_TEST_PRECOMP_HPP__
+
+#include <iostream>
+#include "opencv2/ts.hpp"
+#include "opencv2/imgproc.hpp"
+#include "opencv2/photometric_calib.hpp"
+#include "opencv2/highgui.hpp"
+
+#endif