diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 79667dbd26..497bb7d1f5 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -4,4 +4,4 @@ link_libraries(${OPENCV_LINKER_LIBS}) add_subdirectory(traincascade) add_subdirectory(createsamples) add_subdirectory(annotation) -add_subdirectory(interactive-calibration) +add_subdirectory(visualisation) diff --git a/apps/visualisation/CMakeLists.txt b/apps/visualisation/CMakeLists.txt new file mode 100644 index 0000000000..7c4d6e0a57 --- /dev/null +++ b/apps/visualisation/CMakeLists.txt @@ -0,0 +1,37 @@ +SET(OPENCV_VISUALISATION_DEPS opencv_core opencv_highgui opencv_imgproc opencv_videoio opencv_imgcodecs) +ocv_check_dependencies(${OPENCV_VISUALISATION_DEPS}) + +if(NOT OCV_DEPENDENCIES_FOUND) + return() +endif() + +project(visualisation) +set(the_target opencv_visualisation) + +ocv_target_include_directories(${the_target} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}" "${OpenCV_SOURCE_DIR}/include/opencv") +ocv_target_include_modules_recurse(${the_target} ${OPENCV_VISUALISATION_DEPS}) + +file(GLOB SRCS *.cpp) + +set(visualisation_files ${SRCS}) +ocv_add_executable(${the_target} ${visualisation_files}) +ocv_target_link_libraries(${the_target} ${OPENCV_VISUALISATION_DEPS}) + +set_target_properties(${the_target} PROPERTIES + DEBUG_POSTFIX "${OPENCV_DEBUG_POSTFIX}" + ARCHIVE_OUTPUT_DIRECTORY ${LIBRARY_OUTPUT_PATH} + RUNTIME_OUTPUT_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} + INSTALL_NAME_DIR lib + OUTPUT_NAME "opencv_visualisation") + +if(ENABLE_SOLUTION_FOLDERS) + set_target_properties(${the_target} PROPERTIES FOLDER "applications") +endif() + +if(INSTALL_CREATE_DISTRIB) + if(BUILD_SHARED_LIBS) + install(TARGETS ${the_target} RUNTIME DESTINATION ${OPENCV_BIN_INSTALL_PATH} CONFIGURATIONS Release COMPONENT dev) + endif() +else() + install(TARGETS ${the_target} RUNTIME DESTINATION ${OPENCV_BIN_INSTALL_PATH} COMPONENT dev) +endif() diff --git a/apps/visualisation/opencv_visualisation.cpp b/apps/visualisation/opencv_visualisation.cpp new file mode 100644 index 0000000000..2c685f521a --- /dev/null +++ b/apps/visualisation/opencv_visualisation.cpp @@ -0,0 +1,352 @@ +//////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009, Willow Garage Inc., all rights reserved. +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//////////////////////////////////////////////////////////////////////////////////////// + +/***************************************************************************************************** + +Software for visualising cascade classifier models trained by OpenCV and to get a better +understanding of the used features. + +USAGE: +./visualise_models -model -image -data + +LIMITS +- Use an absolute path for the output folder to ensure the tool works +- Only handles cascade classifier models +- Handles stumps only for the moment +- Needs a valid training/test sample window with the original model dimensions, passed as `ref.png` +- Can handle HAAR and LBP features + +Created by: Puttemans Steven - April 2016 +*****************************************************************************************************/ + +#include +#include +#include +#include +#include + +#include +#include + +using namespace std; +using namespace cv; + +struct rect_data{ + int x; + int y; + int w; + int h; + float weight; +}; + +int main( int argc, const char** argv ) +{ + // Read in the input arguments + string model = ""; + string output_folder = ""; + string image_ref = ""; + for(int i = 1; i < argc; ++i ) + { + if( !strcmp( argv[i], "-model" ) ) + { + model = argv[++i]; + }else if( !strcmp( argv[i], "-image" ) ){ + image_ref = argv[++i]; + }else if( !strcmp( argv[i], "-data" ) ){ + output_folder = argv[++i]; + } + } + + // Value for timing + // You can increase this to have a better visualisation during the generation + int timing = 1; + + // Value for cols of storing elements + int cols_prefered = 5; + + // Open the XML model + FileStorage fs; + fs.open(model, FileStorage::READ); + + // Get a the required information + // First decide which feature type we are using + FileNode cascade = fs["cascade"]; + string feature_type = cascade["featureType"]; + bool haar = false, lbp = false; + if (feature_type.compare("HAAR") == 0){ + haar = true; + } + if (feature_type.compare("LBP") == 0){ + lbp = true; + } + if ( feature_type.compare("HAAR") != 0 && feature_type.compare("LBP")){ + cerr << "The model is not an HAAR or LBP feature based model!" << endl; + cerr << "Please select a model that can be visualized by the software." << endl; + return -1; + } + + // We make a visualisation mask - which increases the window to make it at least a bit more visible + int resize_factor = 10; + int resize_storage_factor = 10; + Mat reference_image = imread(image_ref, IMREAD_GRAYSCALE ); + Mat visualization; + resize(reference_image, visualization, Size(reference_image.cols * resize_factor, reference_image.rows * resize_factor)); + + // First recover for each stage the number of weak features and their index + // Important since it is NOT sequential when using LBP features + vector< vector > stage_features; + FileNode stages = cascade["stages"]; + FileNodeIterator it_stages = stages.begin(), it_stages_end = stages.end(); + int idx = 0; + for( ; it_stages != it_stages_end; it_stages++, idx++ ){ + vector current_feature_indexes; + FileNode weak_classifiers = (*it_stages)["weakClassifiers"]; + FileNodeIterator it_weak = weak_classifiers.begin(), it_weak_end = weak_classifiers.end(); + vector values; + for(int idy = 0; it_weak != it_weak_end; it_weak++, idy++ ){ + (*it_weak)["internalNodes"] >> values; + current_feature_indexes.push_back( (int)values[2] ); + } + stage_features.push_back(current_feature_indexes); + } + + // If the output option has been chosen than we will store a combined image plane for + // each stage, containing all weak classifiers for that stage. + bool draw_planes = false; + stringstream output_video; + output_video << output_folder << "model_visualization.avi"; + VideoWriter result_video; + if( output_folder.compare("") != 0 ){ + draw_planes = true; + result_video.open(output_video.str(), VideoWriter::fourcc('X','V','I','D'), 15, Size(reference_image.cols * resize_factor, reference_image.rows * resize_factor), false); + } + + if(haar){ + // Grab the corresponding features dimensions and weights + FileNode features = cascade["features"]; + vector< vector< rect_data > > feature_data; + FileNodeIterator it_features = features.begin(), it_features_end = features.end(); + for(int idf = 0; it_features != it_features_end; it_features++, idf++ ){ + vector< rect_data > current_feature_rectangles; + FileNode rectangles = (*it_features)["rects"]; + int nrects = (int)rectangles.size(); + for(int k = 0; k < nrects; k++){ + rect_data current_data; + FileNode single_rect = rectangles[k]; + current_data.x = (int)single_rect[0]; + current_data.y = (int)single_rect[1]; + current_data.w = (int)single_rect[2]; + current_data.h = (int)single_rect[3]; + current_data.weight = (float)single_rect[4]; + current_feature_rectangles.push_back(current_data); + } + feature_data.push_back(current_feature_rectangles); + } + + // Loop over each possible feature on its index, visualise on the mask and wait a bit, + // then continue to the next feature. + // If visualisations should be stored then do the in between calculations + Mat image_plane; + Mat metadata = Mat::zeros(150, 1000, CV_8UC1); + vector< rect_data > current_rects; + for(int sid = 0; sid < (int)stage_features.size(); sid ++){ + if(draw_planes){ + int features_nmbr = (int)stage_features[sid].size(); + int cols = cols_prefered; + int rows = features_nmbr / cols; + if( (features_nmbr % cols) > 0){ + rows++; + } + image_plane = Mat::zeros(reference_image.rows * resize_storage_factor * rows, reference_image.cols * resize_storage_factor * cols, CV_8UC1); + } + for(int fid = 0; fid < (int)stage_features[sid].size(); fid++){ + stringstream meta1, meta2; + meta1 << "Stage " << sid << " / Feature " << fid; + meta2 << "Rectangles: "; + Mat temp_window = visualization.clone(); + Mat temp_metadata = metadata.clone(); + int current_feature_index = stage_features[sid][fid]; + current_rects = feature_data[current_feature_index]; + Mat single_feature = reference_image.clone(); + resize(single_feature, single_feature, Size(), resize_storage_factor, resize_storage_factor); + for(int i = 0; i < (int)current_rects.size(); i++){ + rect_data local = current_rects[i]; + if(draw_planes){ + if(local.weight >= 0){ + rectangle(single_feature, Rect(local.x * resize_storage_factor, local.y * resize_storage_factor, local.w * resize_storage_factor, local.h * resize_storage_factor), Scalar(0), FILLED); + }else{ + rectangle(single_feature, Rect(local.x * resize_storage_factor, local.y * resize_storage_factor, local.w * resize_storage_factor, local.h * resize_storage_factor), Scalar(255), FILLED); + } + } + Rect part(local.x * resize_factor, local.y * resize_factor, local.w * resize_factor, local.h * resize_factor); + meta2 << part << " (w " << local.weight << ") "; + if(local.weight >= 0){ + rectangle(temp_window, part, Scalar(0), FILLED); + }else{ + rectangle(temp_window, part, Scalar(255), FILLED); + } + } + imshow("features", temp_window); + putText(temp_window, meta1.str(), Point(15,15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(255)); + result_video.write(temp_window); + // Copy the feature image if needed + if(draw_planes){ + single_feature.copyTo(image_plane(Rect(0 + (fid%cols_prefered)*single_feature.cols, 0 + (fid/cols_prefered) * single_feature.rows, single_feature.cols, single_feature.rows))); + } + putText(temp_metadata, meta1.str(), Point(15,15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(255)); + putText(temp_metadata, meta2.str(), Point(15,40), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(255)); + imshow("metadata", temp_metadata); + waitKey(timing); + } + //Store the stage image if needed + if(draw_planes){ + stringstream save_location; + save_location << output_folder << "stage_" << sid << ".png"; + imwrite(save_location.str(), image_plane); + } + } + } + + if(lbp){ + // Grab the corresponding features dimensions and weights + FileNode features = cascade["features"]; + vector feature_data; + FileNodeIterator it_features = features.begin(), it_features_end = features.end(); + for(int idf = 0; it_features != it_features_end; it_features++, idf++ ){ + FileNode rectangle = (*it_features)["rect"]; + Rect current_feature ((int)rectangle[0], (int)rectangle[1], (int)rectangle[2], (int)rectangle[3]); + feature_data.push_back(current_feature); + } + + // Loop over each possible feature on its index, visualise on the mask and wait a bit, + // then continue to the next feature. + Mat image_plane; + Mat metadata = Mat::zeros(150, 1000, CV_8UC1); + for(int sid = 0; sid < (int)stage_features.size(); sid ++){ + if(draw_planes){ + int features_nmbr = (int)stage_features[sid].size(); + int cols = cols_prefered; + int rows = features_nmbr / cols; + if( (features_nmbr % cols) > 0){ + rows++; + } + image_plane = Mat::zeros(reference_image.rows * resize_storage_factor * rows, reference_image.cols * resize_storage_factor * cols, CV_8UC1); + } + for(int fid = 0; fid < (int)stage_features[sid].size(); fid++){ + stringstream meta1, meta2; + meta1 << "Stage " << sid << " / Feature " << fid; + meta2 << "Rectangle: "; + Mat temp_window = visualization.clone(); + Mat temp_metadata = metadata.clone(); + int current_feature_index = stage_features[sid][fid]; + Rect current_rect = feature_data[current_feature_index]; + Mat single_feature = reference_image.clone(); + resize(single_feature, single_feature, Size(), resize_storage_factor, resize_storage_factor); + + // VISUALISATION + // The rectangle is the top left one of a 3x3 block LBP constructor + Rect resized(current_rect.x * resize_factor, current_rect.y * resize_factor, current_rect.width * resize_factor, current_rect.height * resize_factor); + meta2 << resized; + // Top left + rectangle(temp_window, resized, Scalar(255), 1); + // Top middle + rectangle(temp_window, Rect(resized.x + resized.width, resized.y, resized.width, resized.height), Scalar(255), 1); + // Top right + rectangle(temp_window, Rect(resized.x + 2*resized.width, resized.y, resized.width, resized.height), Scalar(255), 1); + // Middle left + rectangle(temp_window, Rect(resized.x, resized.y + resized.height, resized.width, resized.height), Scalar(255), 1); + // Middle middle + rectangle(temp_window, Rect(resized.x + resized.width, resized.y + resized.height, resized.width, resized.height), Scalar(255), FILLED); + // Middle right + rectangle(temp_window, Rect(resized.x + 2*resized.width, resized.y + resized.height, resized.width, resized.height), Scalar(255), 1); + // Bottom left + rectangle(temp_window, Rect(resized.x, resized.y + 2*resized.height, resized.width, resized.height), Scalar(255), 1); + // Bottom middle + rectangle(temp_window, Rect(resized.x + resized.width, resized.y + 2*resized.height, resized.width, resized.height), Scalar(255), 1); + // Bottom right + rectangle(temp_window, Rect(resized.x + 2*resized.width, resized.y + 2*resized.height, resized.width, resized.height), Scalar(255), 1); + + if(draw_planes){ + Rect resized_inner(current_rect.x * resize_storage_factor, current_rect.y * resize_storage_factor, current_rect.width * resize_storage_factor, current_rect.height * resize_storage_factor); + // Top left + rectangle(single_feature, resized_inner, Scalar(255), 1); + // Top middle + rectangle(single_feature, Rect(resized_inner.x + resized_inner.width, resized_inner.y, resized_inner.width, resized_inner.height), Scalar(255), 1); + // Top right + rectangle(single_feature, Rect(resized_inner.x + 2*resized_inner.width, resized_inner.y, resized_inner.width, resized_inner.height), Scalar(255), 1); + // Middle left + rectangle(single_feature, Rect(resized_inner.x, resized_inner.y + resized_inner.height, resized_inner.width, resized_inner.height), Scalar(255), 1); + // Middle middle + rectangle(single_feature, Rect(resized_inner.x + resized_inner.width, resized_inner.y + resized_inner.height, resized_inner.width, resized_inner.height), Scalar(255), FILLED); + // Middle right + rectangle(single_feature, Rect(resized_inner.x + 2*resized_inner.width, resized_inner.y + resized_inner.height, resized_inner.width, resized_inner.height), Scalar(255), 1); + // Bottom left + rectangle(single_feature, Rect(resized_inner.x, resized_inner.y + 2*resized_inner.height, resized_inner.width, resized_inner.height), Scalar(255), 1); + // Bottom middle + rectangle(single_feature, Rect(resized_inner.x + resized_inner.width, resized_inner.y + 2*resized_inner.height, resized_inner.width, resized_inner.height), Scalar(255), 1); + // Bottom right + rectangle(single_feature, Rect(resized_inner.x + 2*resized_inner.width, resized_inner.y + 2*resized_inner.height, resized_inner.width, resized_inner.height), Scalar(255), 1); + + single_feature.copyTo(image_plane(Rect(0 + (fid%cols_prefered)*single_feature.cols, 0 + (fid/cols_prefered) * single_feature.rows, single_feature.cols, single_feature.rows))); + } + + putText(temp_metadata, meta1.str(), Point(15,15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(255)); + putText(temp_metadata, meta2.str(), Point(15,40), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(255)); + imshow("metadata", temp_metadata); + imshow("features", temp_window); + putText(temp_window, meta1.str(), Point(15,15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(255)); + result_video.write(temp_window); + + waitKey(timing); + } + + //Store the stage image if needed + if(draw_planes){ + stringstream save_location; + save_location << output_folder << "stage_" << sid << ".png"; + imwrite(save_location.str(), image_plane); + } + } + } + return 0; +}