From 6f38d8916039033d5062c36cef30ee305642900c Mon Sep 17 00:00:00 2001 From: Wangyida Date: Sun, 26 Jul 2015 23:42:45 +0800 Subject: [PATCH] add classifier Class and a demo on classification on 3D data --- modules/cnn_3dobj/README.md | 21 +- .../cnn_3dobj/include/opencv2/cnn_3dobj.hpp | 40 +++- modules/cnn_3dobj/samples/CMakeLists.txt | 18 +- modules/cnn_3dobj/samples/classify_demo.cpp | 112 ++++++++++ ...rototxt => 3d_triplet_galleryIMG.prototxt} | 63 +----- .../samples/data/3d_triplet_testIMG.prototxt | 86 ++++++++ .../data/images_mean/triplet_mean.binaryproto | Bin 0 -> 16396 bytes modules/cnn_3dobj/samples/data/label_ant.txt | 0 modules/cnn_3dobj/samples/data/label_ape.txt | 0 modules/cnn_3dobj/samples/data/label_cow.txt | 0 .../cnn_3dobj/samples/data/label_plane.txt | 0 ...{images2db_demo.cpp => datatrans_demo.cpp} | 1 + .../samples/feature_extract_demo.cpp | 18 +- .../samples/sphereview_3dobj_demo.cpp | 21 +- modules/cnn_3dobj/src/cnn_classification.cpp | 197 ++++++++++++++++++ .../{cnn_image2db.cpp => cnn_datatrans.cpp} | 6 +- modules/cnn_3dobj/src/precomp.hpp | 32 --- 17 files changed, 481 insertions(+), 134 deletions(-) create mode 100644 modules/cnn_3dobj/samples/classify_demo.cpp rename modules/cnn_3dobj/samples/data/{3d_triplet_train_test.prototxt => 3d_triplet_galleryIMG.prototxt} (60%) create mode 100644 modules/cnn_3dobj/samples/data/3d_triplet_testIMG.prototxt create mode 100644 modules/cnn_3dobj/samples/data/images_mean/triplet_mean.binaryproto delete mode 100644 modules/cnn_3dobj/samples/data/label_ant.txt delete mode 100644 modules/cnn_3dobj/samples/data/label_ape.txt delete mode 100644 modules/cnn_3dobj/samples/data/label_cow.txt delete mode 100644 modules/cnn_3dobj/samples/data/label_plane.txt rename modules/cnn_3dobj/samples/{images2db_demo.cpp => datatrans_demo.cpp} (90%) create mode 100644 modules/cnn_3dobj/src/cnn_classification.cpp rename modules/cnn_3dobj/src/{cnn_image2db.cpp => cnn_datatrans.cpp} (98%) diff --git a/modules/cnn_3dobj/README.md b/modules/cnn_3dobj/README.md index 4e578d954..7a505a94c 100644 --- a/modules/cnn_3dobj/README.md +++ b/modules/cnn_3dobj/README.md @@ -37,21 +37,21 @@ $ make #Demo1: ###Imagas generation from different pose, 4 models are used, there will be 276 images in all which each class contains 69 iamges ``` -$ ./sphereview_test -ite_depth=2 -plymodel=../3Dmodel/ape.ply -imagedir=../data/images_ape/ -labeldir=../data/label_ape.txt -num_class=4 -label_class=0 +$ ./sphereview_test -ite_depth=2 -plymodel=../3Dmodel/ape.ply -imagedir=../data/images_all/ -labeldir=../data/label_all.txt -num_class=4 -label_class=0 ``` ###press q to start ``` -$ ./sphereview_test -ite_depth=2 -plymodel=../3Dmodel/ant.ply -imagedir=../data/images_ant/ -labeldir=../data/label_ant.txt -num_class=4 -label_class=1 +$ ./sphereview_test -ite_depth=2 -plymodel=../3Dmodel/ant.ply -imagedir=../data/images_all/ -labeldir=../data/label_all.txt -num_class=4 -label_class=1 ``` ###press q to start ``` -$ ./sphereview_test -ite_depth=2 -plymodel=../3Dmodel/cow.ply -imagedir=../data/images_cow/ -labeldir=../data/label_cow.txt -num_class=4 -label_class=2 +$ ./sphereview_test -ite_depth=2 -plymodel=../3Dmodel/cow.ply -imagedir=../data/images_all/ -labeldir=../data/label_all.txt -num_class=4 -label_class=2 ``` ###press q to start ``` -$ ./sphereview_test -ite_depth=2 -plymodel=../3Dmodel/plane.ply -imagedir=../data/images_plane/ -labeldir=../data/label_plane.txt -num_class=4 -label_class=3 +$ ./sphereview_test -ite_depth=2 -plymodel=../3Dmodel/plane.ply -imagedir=../data/images_all/ -labeldir=../data/label_all.txt -num_class=4 -label_class=3 ``` -###press q to start, when all images are created in each class folder, you should copy all images from ../data/images_ape, ../data/images_ant, ../data/images_cow and ../data/images_plane into ../data/images_all folder as a collection of images for network tranining and feature extraction, when all images are copyed correctlly, proceed on. +###press q to start, when all images are created in images_all folder as a collection of images for network tranining and feature extraction, then proceed on. ###After this demo, the binary files of images and labels will be stored as 'binary_image' and 'binary_label' in current path, you should copy them into the leveldb folder in Caffe triplet training, for example: copy these 2 files in /data/linemod and rename them as 'binary_image_train', 'binary_image_test' and 'binary_label_train', 'binary_label_train'. ###We could start triplet tranining using Caffe ``` @@ -70,19 +70,22 @@ $ cd /modules/cnn_3dobj/samples/build #Demo2: ###Convert data into leveldb format from folder ../data/images_all for feature extraction afterwards. The leveldb files including all data will be stored in ../data/dbfile. If you will use the OpenCV defined feature extraction process, you could also skip Demo2 for data converting, just run Demo3 after Demo1 for feature extraction because Demo3 also includes the db file converting process before feature extraction. ``` -$ ./images2db_test -images2db_demo=../data/images_all -src_dst=../data/dbfile -attach_dir=../data/dbfile -channel=1 -width=64 -height=64 +$ ./images2db_test ``` ============== #Demo3: -###feature extraction, this demo will convert a set of images in a particular path into leveldb database for feature extraction using Caffe. +###feature extraction, this demo will convert a set of images in a particular path into leveldb database for feature extraction using Caffe and outputting a binary file including all extracted feature. ``` $ ./feature_extract_test ``` -###This will extract feature from a set of images in a folder as vector for further classification and a binary file with containing all feature vectors of each sample. Pay attention: if it's warning you that 'Check failed: leveldb::DB::Open(options, outputdb, &db).ok()', the reason is that there is alreay leveldb files in ../data/dbfile as previous running of Demo2 or Demo3, just delete all files in ../data/dbfile and run Demo3 again. +###This will extract feature from a set of images in a folder as vector for further classification and a binary file with containing all feature vectors of each sample. ###After running this, you will get a binary file storing features in ../data/feature folder, I can provide a Matlab script reading this file if someone need it. If you don't need the binary file, the feature could also be stored in vector for directly classification using the softmax layer as shown in Demo4. ============== #Demo4: -###Classifier +###Classifier, this will extracting the feature of a single image and compare it with features of gallery samples for prediction. Just run: +``` +$ ./classify_test +``` ============================================== diff --git a/modules/cnn_3dobj/include/opencv2/cnn_3dobj.hpp b/modules/cnn_3dobj/include/opencv2/cnn_3dobj.hpp index 4e9c1e649..1ca78b846 100644 --- a/modules/cnn_3dobj/include/opencv2/cnn_3dobj.hpp +++ b/modules/cnn_3dobj/include/opencv2/cnn_3dobj.hpp @@ -60,10 +60,6 @@ the use of this software, even if advised of the possibility of such damage. #include #include #include -//#include -#include -#include -#include #define CPU_ONLY #include #include @@ -71,6 +67,10 @@ the use of this software, even if advised of the possibility of such damage. #include #include #include +#include "opencv2/viz/vizcore.hpp" +#include "opencv2/highgui.hpp" +#include "opencv2/highgui/highgui_c.h" +#include "opencv2/imgproc.hpp" using std::string; using caffe::Blob; using caffe::Caffe; @@ -161,6 +161,38 @@ class CV_EXPORTS_W DataTrans /** @brief Extract feature into a binary file and vector for classification, the model proto and network proto are needed, All images in the file root will be used for feature extraction. */ }; + +class CV_EXPORTS_W Classification +{ + private: + caffe::shared_ptr > net_; + cv::Size input_geometry_; + int num_channels_; + cv::Mat mean_; + std::vector labels_; + void SetMean(const string& mean_file); + /** @brief Load the mean file in binaryproto format. + */ + void WrapInputLayer(std::vector* input_channels); + /** @brief Wrap the input layer of the network in separate cv::Mat objects(one per channel). This way we save one memcpy operation and we don't need to rely on cudaMemcpy2D. The last preprocessing operation will write the separate channels directly to the input layer. + */ + void Preprocess(const cv::Mat& img, std::vector* input_channels, bool mean_subtract); + /** @brief Convert the input image to the input image format of the network. + */ + public: + Classification(const string& model_file, const string& trained_file, const string& mean_file, const string& label_file); + /** @brief Initiate a classification structure. + */ + std::vector > Classify(const std::vector& reference, const cv::Mat& img, int N = 4, bool mean_substract = false); + /** @brief Make a classification. + */ + cv::Mat feature_extract(const cv::Mat& img, bool mean_subtract); + /** @brief Extract a single featrue of one image. + */ + std::vector Argmax(const std::vector& v, int N); + /** @brief Find the N largest number. + */ +}; //! @} }} diff --git a/modules/cnn_3dobj/samples/CMakeLists.txt b/modules/cnn_3dobj/samples/CMakeLists.txt index 50a4a71ca..66460f876 100644 --- a/modules/cnn_3dobj/samples/CMakeLists.txt +++ b/modules/cnn_3dobj/samples/CMakeLists.txt @@ -3,15 +3,19 @@ SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -Wall -g -ggdb ") SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall") project(sphereview_test) find_package(OpenCV REQUIRED) -set(SOURCES sphereview_3dobj_demo.cpp) +set(SOURCES_1 sphereview_3dobj_demo.cpp) include_directories(${OpenCV_INCLUDE_DIRS}) -add_executable(sphereview_test ${SOURCES}) +add_executable(sphereview_test ${SOURCES_1}) target_link_libraries(sphereview_test ${OpenCV_LIBS}) -set(SOURCES2 images2db_demo.cpp) -add_executable(images2db_test ${SOURCES2}) -target_link_libraries(images2db_test ${OpenCV_LIBS}) +set(SOURCES_2 datatrans_demo.cpp) +add_executable(datatrans_test ${SOURCES_2}) +target_link_libraries(datatrans_test ${OpenCV_LIBS}) -set(SOURCES3 feature_extract_demo.cpp) -add_executable(feature_extract_test ${SOURCES3}) +set(SOURCES_3 feature_extract_demo.cpp) +add_executable(feature_extract_test ${SOURCES_3}) target_link_libraries(feature_extract_test ${OpenCV_LIBS}) + +set(SOURCES_4 classify_demo.cpp) +add_executable(classify_test ${SOURCES_4}) +target_link_libraries(classify_test ${OpenCV_LIBS}) diff --git a/modules/cnn_3dobj/samples/classify_demo.cpp b/modules/cnn_3dobj/samples/classify_demo.cpp new file mode 100644 index 000000000..f13a36085 --- /dev/null +++ b/modules/cnn_3dobj/samples/classify_demo.cpp @@ -0,0 +1,112 @@ +/* + * Software License Agreement (BSD License) + * + * Copyright (c) 2009, Willow Garage, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of Willow Garage, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + */ +#include +#include +using namespace cv; +using namespace std; +using namespace cv::cnn_3dobj; +int main(int argc, char** argv) +{ + const String keys = "{help | | this demo will convert a set of images in a particular path into leveldb database for feature extraction using Caffe.}" + "{src_dir | ../data/images_all/ | Source direction of the images ready for being converted to leveldb dataset.}" + "{src_dst | ../data/dbfile | Aim direction of the converted to leveldb dataset. }" + "{attach_dir | ../data/dbfile | Path for saving additional files which describe the transmission results. }" + "{channel | 1 | Channel of the images. }" + "{width | 64 | Width of images}" + "{height | 64 | Height of images}" + "{caffemodel | ../data/3d_triplet_iter_10000.caffemodel | caffe model for feature exrtaction.}" + "{network_forDB | ../data/3d_triplet_galleryIMG.prototxt | Network definition file used for extracting feature from levelDB data, causion: the path of levelDB training samples must be wrotten in in .prototxt files in Phase TEST}" + "{save_feature_dataset_names | ../data/feature/feature_iter_10000.bin | Output of the extracted feature in form of binary files together with the vector features as the feature.}" + "{extract_feature_blob_names | feat | Layer used for feature extraction in CNN.}" + "{num_mini_batches | 4 | Batches suit for the batches defined in the .proto for the aim of extracting feature from all images.}" + "{device | CPU | Device: CPU or GPU.}" + "{dev_id | 0 | ID of GPU.}" + "{network_forIMG | ../data/3d_triplet_testIMG.prototxt | Network definition file used for extracting feature from a single image and making a classification}" + "{mean_file | ../data/images_mean/triplet_mean.binaryproto | The mean file generated by Caffe from all gallery images, this could be used for mean value substraction from all images.}" + "{label_file | ../data/dbfileimage_filename | A namelist including all gallery images.}" + "{target_img | ../data/images_all/2_13.png | Path of image waiting to be classified.}" + "{num_candidate | 6 | Number of candidates in gallery as the prediction result.}"; + cv::CommandLineParser parser(argc, argv, keys); + parser.about("Demo for Sphere View data generation"); + if (parser.has("help")) + { + parser.printMessage(); + return 0; + } + string src_dir = parser.get("src_dir"); + string src_dst = parser.get("src_dst"); + string attach_dir = parser.get("attach_dir"); + int channel = parser.get("channel"); + int width = parser.get("width"); + int height = parser.get("height"); + string caffemodel = parser.get("caffemodel"); + string network_forDB = parser.get("network_forDB"); + string save_feature_dataset_names = parser.get("save_feature_dataset_names"); + string extract_feature_blob_names = parser.get("extract_feature_blob_names"); + int num_mini_batches = parser.get("num_mini_batches"); + string device = parser.get("device"); + int dev_id = parser.get("dev_id"); + string network_forIMG = parser.get("network_forIMG"); + string mean_file = parser.get("mean_file"); + string label_file = parser.get("label_file"); + string target_img = parser.get("target_img"); + int num_candidate = parser.get("num_candidate"); + cv::cnn_3dobj::DataTrans transTemp; + transTemp.convert(src_dir,src_dst,attach_dir,channel,width,height); + std::vector feature_reference = transTemp.feature_extraction_pipeline(caffemodel, network_forDB, save_feature_dataset_names, extract_feature_blob_names, num_mini_batches, device, dev_id); +////start another demo + cv::cnn_3dobj::Classification classifier(network_forIMG, caffemodel, mean_file, label_file); + + std::cout << std::endl << "---------- Prediction for " + << target_img << " ----------" << std::endl; + + cv::Mat img = cv::imread(target_img, -1); + // CHECK(!img.empty()) << "Unable to decode image " << target_img; + std::cout << std::endl << "---------- Featrue of gallery images ----------" << std::endl; + std::vector > prediction; + for (unsigned int i = 0; i < feature_reference.size(); i++) + std::cout << feature_reference[i] << endl; + cv::Mat feature_test = classifier.feature_extract(img, false); + std::cout << std::endl << "---------- Featrue of target image: " << target_img << "----------" << endl << feature_test.t() << std::endl; + prediction = classifier.Classify(feature_reference, img, num_candidate, false); + // Print the top N prediction. + std::cout << std::endl << "---------- Prediction result(distance - file name in gallery) ----------" << std::endl; + for (size_t i = 0; i < prediction.size(); ++i) { + std::pair p = prediction[i]; + std::cout << std::fixed << std::setprecision(2) << p.second << " - \"" + << p.first << "\"" << std::endl; + } + return 0; +} diff --git a/modules/cnn_3dobj/samples/data/3d_triplet_train_test.prototxt b/modules/cnn_3dobj/samples/data/3d_triplet_galleryIMG.prototxt similarity index 60% rename from modules/cnn_3dobj/samples/data/3d_triplet_train_test.prototxt rename to modules/cnn_3dobj/samples/data/3d_triplet_galleryIMG.prototxt index 30ff45816..a5f7ce15a 100644 --- a/modules/cnn_3dobj/samples/data/3d_triplet_train_test.prototxt +++ b/modules/cnn_3dobj/samples/data/3d_triplet_galleryIMG.prototxt @@ -1,4 +1,4 @@ -name: "3d_test" +name: "3d_triplet" layer { name: "data" type: "Data" @@ -7,12 +7,9 @@ layer { include { phase: TEST } - transform_param { - scale: 0.00390625 - } data_param { source: "/home/wangyida/Desktop/opencv_contrib/modules/nouse_test/samples/data/dbfile" - batch_size: 46 + batch_size: 69 } } layer { @@ -20,24 +17,10 @@ layer { type: "Convolution" bottom: "data" top: "conv1" - param { - name: "conv1_w" - lr_mult: 1 - } - param { - name: "conv1_b" - lr_mult: 2 - } convolution_param { num_output: 16 kernel_size: 8 stride: 1 - weight_filler { - type: "xavier" - } - bias_filler { - type: "constant" - } } } layer { @@ -62,24 +45,10 @@ layer { type: "Convolution" bottom: "pool1" top: "conv2" - param { - name: "conv2_w" - lr_mult: 1 - } - param { - name: "conv2_b" - lr_mult: 2 - } convolution_param { num_output: 7 kernel_size: 5 stride: 1 - weight_filler { - type: "xavier" - } - bias_filler { - type: "constant" - } } } layer { @@ -104,22 +73,8 @@ layer { type: "InnerProduct" bottom: "pool2" top: "ip1" - param { - name: "ip1_w" - lr_mult: 1 - } - param { - name: "ip1_b" - lr_mult: 2 - } inner_product_param { num_output: 256 - weight_filler { - type: "xavier" - } - bias_filler { - type: "constant" - } } } layer { @@ -133,21 +88,7 @@ layer { type: "InnerProduct" bottom: "ip1" top: "feat" - param { - name: "feat_w" - lr_mult: 1 - } - param { - name: "feat_b" - lr_mult: 2 - } inner_product_param { num_output: 4 - weight_filler { - type: "xavier" - } - bias_filler { - type: "constant" - } } } diff --git a/modules/cnn_3dobj/samples/data/3d_triplet_testIMG.prototxt b/modules/cnn_3dobj/samples/data/3d_triplet_testIMG.prototxt new file mode 100644 index 000000000..5a4479a74 --- /dev/null +++ b/modules/cnn_3dobj/samples/data/3d_triplet_testIMG.prototxt @@ -0,0 +1,86 @@ +name: "3d_triplet" +input: "data" +input_dim: 1 +input_dim: 1 +input_dim: 64 +input_dim: 64 +layer { + name: "conv1" + type: "Convolution" + bottom: "data" + top: "conv1" + convolution_param { + num_output: 16 + kernel_size: 8 + stride: 1 + } +} +layer { + name: "pool1" + type: "Pooling" + bottom: "conv1" + top: "pool1" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "relu1" + type: "ReLU" + bottom: "pool1" + top: "pool1" +} +layer { + name: "conv2" + type: "Convolution" + bottom: "pool1" + top: "conv2" + convolution_param { + num_output: 7 + kernel_size: 5 + stride: 1 + } +} +layer { + name: "pool2" + type: "Pooling" + bottom: "conv2" + top: "pool2" + pooling_param { + pool: MAX + kernel_size: 2 + stride: 2 + } +} +layer { + name: "relu2" + type: "ReLU" + bottom: "pool2" + top: "pool2" +} +layer { + name: "ip1" + type: "InnerProduct" + bottom: "pool2" + top: "ip1" + inner_product_param { + num_output: 256 + } +} +layer { + name: "relu3" + type: "ReLU" + bottom: "ip1" + top: "ip1" +} +layer { + name: "feat" + type: "InnerProduct" + bottom: "ip1" + top: "feat" + inner_product_param { + num_output: 4 + } +} diff --git a/modules/cnn_3dobj/samples/data/images_mean/triplet_mean.binaryproto b/modules/cnn_3dobj/samples/data/images_mean/triplet_mean.binaryproto new file mode 100644 index 0000000000000000000000000000000000000000..c088557349b6a2b2963e46ee790d6fd6754d8f40 GIT binary patch literal 16396 zcmbuGeSFpRdH2s_0tSs0mkLhWtc?>UmMXUNrZe{YqnXW=S{$=cvCO7OE0#K}y3A2Z zCoTn?xYVK2LYpa89Jh^=t|(S|LrYzp*wj)V?t*zZC+Fmxlk=Q&l9S|qf4*O-^L6i5 z;r=79Kpwuo>vuhTuFv(kzM0}b-GA1lAG~zVx^;e^&*z=UN6m~A`}ub|d;as2W`6Ue znf+0pw}(<&xPGESbzjW7w*%o&1^gHIpWkE1BiH3#R$bB=+UZ zf_-JP`hWO5eVpjodyA9i;ENO3R^r^8scA`K?~s|>pEL8W9yKjfPW;T{Hgw!ku`xRQ zXx1zqPMNJk1@oG}Vh+}x+?h-Nz~{}{Dt{J@Z(W-A9wKK`X4jm8*?QB2*?h*yo!fik z82&7p@&g%D^Gb@i$M`&K8as<8ZL9p&8!egM8w=!W(i}c-$Siy!W0u@7VotWEW}W5p z<{eRO7R-V_&6>Ik6K3;$L-;mhb3m~??)&%p`N(NLuj}42^;s|t6T|p5YYv&1={%4y z`>)HH_U$Krrk>rfcGA?>l+3ETbJnkIJ%hY|(nPMw8}I&!6Sh-ls-9+NO6K>fz*U z-8E;N+7yNio4KudvvA&!+4j4*S=*Gx{;?CXwfkb1*NZ>m^H$wmG|iQ?>AtvZLThs7 z(9fb~)>#RBkvpL?d;i$y&A!Fw-Tpy7XGYA-hco8jXOpJ;siKL$5a<6<{E@(RXKV63 zVpilDXX~oF$4uAh*m~!XY5Y*hEM1y1^Pi8Jy~Tvt^Q{w$Rc4LPQ}5`ycfu4Np(bBU zaYh+@DiE(IXUFkrnwY)!Ty3rQdDFk)^Ir4&yp?B;nAL3=>M2Fum(7vWhKW;z&vDxe zZ9Y$clkWkyj^66te{Soj+5dVTf5uGY)sorq64=rbAy4Dv>xso`>rFmy`B#13?(bGi z&qD>?d&sn0Mjad*F>8Mw;k^g(amwuYoO^%8_TB1VITgwg|K;?JwxX$CkuqzNNpNL? z^V0mg-z1+KG!1JDHm~IWwyWOx+%o*8^Y2|BUjLI4@1HYuU4y3e`^tp^I5hyjh?&Z! z%xtD*ecpn7W7en5cgBc+3Cu6^S@j<^+pZrn^&4HSF1g{|&(yiI zKJWM1{>kT6uPBf|30v>+7bY#9-oCugwDm+x&)IovYxh$QE8fko?MHoHrQ7FiIDdrt z7nTkZuW{m51P=$WF=}I_bN0n1dFFewwSBwaYnYH9v()AwHD5AY?izyw60>~~`c<4* zjhnSq`0IP`ex~w1xlVTKXAAI>A^KFx^t|Tt)K7YE?4@51P+K{3@E?8N{%!Amrt_)K zTk=PK&#Nnuo5QB5J<8_^YG8yO6Xo0pHYc1<$-y=M`tIk>KgH+O|GD2gnjfVn3;Uzk zHUdsfz%7&1Y9IcHk-Ouj^{;&1%!l9kOy%7CW}l~+Za!m@x}hG|#f(>%GQB&=-xugv zGooNYKX{Wh+h+N^#y)qZ{Ps>cAdKh?`N>s>{jzJs}i5SJqs_FrA_|b$>oiseF z_xgXIr}tiSv-*?E)xEcs;5!);zhaPj&B6KnU;_Q6zt_}uN6C$G1v?0+-hZw>wl6YFDqbm8M_larw@Cv^&CGu!SAUL{ns<)+Xni=aUZYjP(AVP-%sZPrwte#&0*ib*60G`X{dEH(cWP_TDD! z^Lve*WAL(J>L+4$e>cUol-h3ZD`bs~& z%I|HxDMv0Pc-I)Ul{XvCuh3u9Q`FYk&JptPGsNVZ?&H7OT6dxF#P5aH z1iZQb;=AH(fS!Gn8H@h)5HWb(=XET=XJ2$PvAC;Z zCJr(Glz_LgBjD}%R=~^T{Pf@=xirY{_QCa{#2w6{=S5x}1hbEkw}ZrreEt*i@IR@+ zFZsRdN8a-5zS;EAOL^z^fS2?Iyrci-Cuha0hE4KP;pYH-Ibmn34d>_K*uCIkFL+Sq zUHsmLRpkD!eBS!MdF#wspLb_&xeGhJfH&*60WZGU?`h_ZzYu{x^jhEU`*M<8&6BqS zeC;SaBZbBi@S1)p{0n&PA5iT$->QcR|8~XI-~3+9j1WE#kq6-P4XUSz_3QlSnSEN* z@RAsqJwW{qf&=O)#Nd0JcbYy_56)h3)2%o8y&W$FY)!4X8NWVRALfAOY}!u` z>@$UPHUFlV0mLbKZSFSx((lbW%kOpEA#8WC5WcAAul+f8cY%Q)4ZwY|ll~Y!EbVjzOuH74{SRWaPO}^T8OCMQlGbD>ytsR zwSI#BHb^gtTh8m%iJRq_1+(N<6hFK{Pe_u(;skJ|`MmqL#Kqj12S4lTw)zopBov^Z z1)2Tihd41A#5Tn!3ce=sZILtM%)!djKJv6kAMjheP@MG6;z7b_adX+KoT`olus7%} z=`Dbx5oWy@b8iHDl~Xy>awx+y2Efxdz-ej${8{x2KXs$JA~)waTa~AWA0=jGKlTTx z4`JX0`dozHi`d!JVQbdx{Bhd${ z^jDqv+NZ#QRbXg+kU9;5gJJj&eDVIlH#r*lpGopTu1}0up1S$I2)aVl)`a-l(fI** zeZXs*?(Ec@b$AdT&Is5(b$DgaYyScmm>(wZ<6z1Vei(#b4pX;fv-|>ZaIH9XKQ)KW z_VlEwofhz#E3P(Wr~AO4^pPbu1ikLBhwyF4(=5HVNiiBiM-bmj!-1u*WwAK|hB2>Q z#cb7?H|xvHE3ebv_Xy|QnVPls{~QH@%kEqr1S9EJa!lO!iAW_sfY zz8ykiQ9eb<+abO`Y36=72u{I6{!lvHB>kNk;$rH4D}MP+(A#n_iqV%f>v5fP*QFJHq6+dTEuR&+4cw}acVi@x3vLR|Lz^tN~JW6g3 z+T5CVBx5;c=E;oJ7F+98Z?cv7fLdIBK@hvtw~qT1d|d>_{45COR(Tzz0GM60c- zz^f>;bq39^0^XP40>k8Moc~L+KpVJu%=BCvM1Kl+4By*g*B^igr{*_n`6 zJFNVrv;x>l=gSG}Vk3=4Hw$A-9 zHE$gmdo`1X}DAc9x-Y63evxpUoZfs>U~Dcz8?nQR6$z<$F&gg ztizA0ew9yEp0KC+&N%*y)AyBIY0gsaR`6lj;#13I(r<>SDR@@L1b(6qkyl;!()VZj zJk7nNp(Q;gr2jXCmZgn4CxxiSD(KwD}WwtC&tmsP8yn4-9zAcZa|}@u0G;?WU=2kF@6I5sSy-ovMRfkD)XB zH5aGo4Mp;R-B>PPw{IT?Q&Vs9Y|Ss@Mmf8 zhflGySNVY=v&%5pIZV&WGrLZh^?w6zy4-JPK+VCLTl9>*A5*`Pb|$S`Itu%PS99cL z#q8e}_L`rWh7HwTZFd=*O47>`Y}uT0Y%1{W+%k~c2@Yx!FBib?9D$lR~q z9wF{&8;h+sX}4BIElvX`g&SGEAE(zyV=C}&=xyZX$}|1M+Vy{(TOASS3V;J+Xr}bK zd!$dK$%7F%?*zFx6`!B(-M%A)-}C4KDL6-xoJPxQsF=-50^Y)T!Z>VZX4PJz>#+O) zhxzgtwUaaZV*S{H##xi)+!DMe$b2)ExpRuw)-#6vsW*24X?nm2`8sZFzyH3|Lo{Pb z>uvfBGvz-ByNZ^JG)xT8Ptss1bJRY5=kcldwZ_im!dUew<=)}*q=_k(>Z{VD9H-WN zQQJ)((8r`ncP<(PQ`C!7?Ecsv5&ylnRgtHsGS7#>Ue&I6MuMEpSY1qWuxd(Lf@bN4 zOL+HxqX*_V^ANi^c!TECl^uhYn}>2^mg9AlPVtufpc)MZzWtM|^?TR7!@P%h(YTQg zDK4cQk1$2Jy7uRqjnU(#Y0uM-ZQ>_6xLVQX#NkJ$z&)md=c&qoK>Ss6>gF?e*VEBr&h5AFziUp0{+TD& zICpBuTkxlFIr@ljv0*}U7Ikn792PsJ$LVf>uu591IBdrqU_3rQ_~H<}J^~JiPbcln zr=Hioy+l03$wuiT(t^;hqx4+%);{oF{a>}YY$5qcJa>NFZ)cdc>1YaPWtqdJ8x50R z3GhEd4ry*dmwX{?XYp;;wCm|Q3{Eej|0RUA z@Q`l!`B1`Yd`E9(H{VMRO$(y4PO-C{_JJE7kC6{U@D%<0xX~HX`4l7ZQq4ar&kXQh zaKJ_4xI@Hy(E3bwDYRRW_TO_YK0OeiMsjf01bu;i-kIY4N70BB7xp{gp7e>Mw`zYP zPAz^j2JWY*nJDqq?l5X+zWR+tw0>~47y|#*59rlBaq<-YdqIi!XU1jz5m(RT=vDtg zzn%eaoe6*A{2ec`pS(`m)&N*N2%e_lPs8-05I7Vh_oW-e@BFD)YvUQKX!0sgZC zKf=$YFDa*L&=U(Qg4Bd|vYKnq(WdHm;^1ozT;?5TRNHwrl+#@ha^&1N@pS!RO%e|L zDYUT1RnO8v{Fa~3z9m5Jj@y~0=j;LEG(ZhVQ)LIl9;9|tmE~NTW0x(IwvG;5BuD1J z)ultB??_v@{TR5v!D-*KZ}Bs`1gM#^&C}_P!qqq*dGLyv5?y5P$6SB!DCNojG7E9An}4sex^vgPC2Au@ zU7+K^yTlo`9HbtZ)#jhVnFWU_GY8N`2hjX{et1(Fe;%U;N}p7Iv9COqCU-)-Ux0b5 zY%xJNb@;p_HC!PMZYGz$Hv6OEb#C7%UM2mjqf{cV`Z!~Nch3+L`t|8K{vY&e9uUW` zSgzfCXWE`w^Gcb1L2TBOJD+pff;fQojlOkG|J?ujB)vUG?j2(makI+SoATst$sAgb z?)qzH#~B5DLl3(zN}Wn))IBpj&()Q3R62?z{LabYv#rWr1>V8%ZKq@7R=6> zhD(IGyvrRqw4W?}KMgj>w-flCp7blHFDkdB`%43An9%GZ4WNQX)sMYV_$@O-Czx~> z`f^OUq24c_7O5%jSygmzXN=l&`o3zlvdQg(gqPZhsK@E9jOJ6Huh-5Lx|^gtZT|qX z#F_weiZpll*=6*SaEe?>lY^R7@aY!kQ|W63>=ggheowt2`JCo*b|z+s-y4Pxi&vNI z{y_d0chSCG_kwo*xWLa%GPee7uUEg9Po>+gNxGfaf_-4aZDZu&fYp-JtLOI1uX+54 zPH>=tZ8M|4j?QULN6$Qd-&msvgI&d9roWN9_w^P&63g?HYCu-KP^s8K7cn} zr=xqHgROTaI8$?l@@f>UV+MJuV12oIbc}aFW4gKmPY;53%xYorXSci4OskrbKQ^4t zT=A&58uwgYj_|%|>N}0iSz?S<_0t@f#9e@}cojPZ_8I$MN2j>3q4$gXh+|0) zm(CBPIqEdubM@f5AW3cF-NZ(B|j_ll`=ewdFp$r-75GPc(Xt;C;ne` zwWWQVe4+a8c_>MKOQ)4~mZZ05sH+nBG=@LQ)KQt9I!c`|v%ao*u?nskv~kibDW0VL zmG;<&&%^%N0r-YuAzm4`d{tV{{O7^W9`b#4j`Q*3H51m>SzAlY{#D=y8Yi`{b85RM zjdVb1c01mu{^z(*{YIzpX~#4FdENCAw&XZh8jUpIaqJ((?vj}~5ccXfR>8-ER^to7 zsnn~5R~`TEVpg8POxj6)vNO(H6hreG;=MD}t}wqqo^oGzMH%iRjhQ=#d1-1@a56Z@ zRybfMXTg6pr_B3j_QO4{x9VQkmT;Whgj-Or3+HLiC7nY%_AFR1imek?qp9BrPLEZQ z|6$AXgeBV7$+z-DQ@i>AevQFjxrbKSqNRCKt9Dc?JGcwhHG&T+oJ&5`RLIvV zuXiWfz%$e<{7-#ZJ4>A>Jd`G=eP(%vW+3(GA-nI>69Oiu72>`#*4y6=~^dzzgeFIYX` z$Z2EnnUZPxWr(@Fnz{+ueU0ves6VU!X+{;chzn}Iu4&O-H}=!;%gmGLad3jpMT-3} z@6KH_{8`sku(?0;OI7rmDm#;jt7?x}i*3@sgoU~vA)Q*9!1TsEy)H>E#)(&)f1~%E zlLb%2{nQWP24AeA&sBrNVdj*O#d_^|_50Gm7Jkk3EZM&Me=BFxc23bf4}I(jYA;V- zotXzub+@2M{iEH^s6zLwf@_4CbEI{^Rk64EPN&uDeo#k=XW-vecQbpSZFJlre#LI$ zDe1r&%c15s6ZbA=_4_K+hxE&8dUM#m|BjcmTXg#+-Q$ujrv0#ZoZ>2-Ks$=L{ZV>- z%=%M&V^5IXBHT|}(XywfkmJ?pg4!2{@S|{re*Y1-Q&g;^SEy&#Wo7#)xGGH@e_kQK zYBrH(oddg>2hU=cQx!xHs%CzxVm=SsekWU#&*{U>tkSGI4>%qqeW<3TM2}+!`hfJP zq~%fKX7wAh*s9%n$ZL3Z3b+UUt*558k+;-?>PUCOq*be4GZ$(8dvnJ4k+`0EW&$5d zlgiOobjK`D?uwg**>j=^^@n-BFhx6F&1(2{1Gx&e=x;FeZ#_rcL-%Ebqw0g22Nyn} z9d(xa%-XZ}UpHpwfzA+qO@zRwkmahvrMfO=OK?#8^&|J-hrgw-G9MRKprOE{w%nB= z&fI?urog)#b)|dy@E6?;_PoBdAnC#7LH&Mm!CJ6q+db|8wsu%>enUvGfV za5qtRmC6rjE@wx@-vHEJp2r8wt#u*luo}O^Cq9LL4+SlE5(X|$F{3`nvyFDl?6Ik{ zLPS^-u>S2i8!W!3NT26^5%CS@Ur8^snm2BPP>D~f+m`V1+&thkU_vgJk?!celX0~UZ-o4D- zRnbf>ZTVoW-jjZMugg_ol;SAfDea>B>#85_6MrOV=NoZI`Lsa2*ZsBj&&)PkEIyzen|ro?U;M4^!Vugk$a@4W*VXsM`E^G@_$HlB^O|x~ zx~l!VqrHw-NC#D2>0`qx<#Ui7tKt(xqX_b@LF><*Bl3}S3p5&HDc&Tmt2=g@0d%KA z96R^3h9zTxgz93FJqjm~U+gSlvBh&Z{5J%@>-x^=xUm!e;5ao!pNSCh4w}_5~|{ zjV}EK^`6?bL?Owm5dy&##nx9c^xLwZj3ml#cI}~&MosjTI`)<9fW;ET46$f5Q z@01^OzeLYe9*ei@uB2inJlJ+Uex-j%f8BYzxUTBY`BZm3?Oo^&w^PvlZRM!$87XeE zZ*ITS#KZ}-E7so)tok$OXKB2$TijE3`gNyRK2#4@ti}8H-iBTHrt<)^QQ*4J0)FL;=^Swuk8E(y?Qmij5z;FYm!E; z`)Jx52}4zv%4NNW`jBQPeU`SWJ%zZGYF_tj-QUfkvGH~3Bl5YhL0XiaryYrSnCen- z5WNp-KgznTGc4_3TQ+(Gwa^(^gibQjIX&x!*`ClqE1&s5`57Yo*@f{v*&#c{iKGkYe7%=;Tl!lIohct^p03gP@F!_h(tDMYvQ?V0_TTD*np@;Q z)rk6x#fWWS!Kd-#bJ+4IU;CZIOlcdM>y$sz(2wV={?1GFZfog}+}=t(MY*S0KzJ(+ zS9S{Y+hyQt>nSMAgEH&%+x!g2>oVKa4w)^5s z^zQ}x+U@siyPs-gr(nO0-%802`Azkpmx)-E*!X_{%UN!9 literal 0 HcmV?d00001 diff --git a/modules/cnn_3dobj/samples/data/label_ant.txt b/modules/cnn_3dobj/samples/data/label_ant.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/modules/cnn_3dobj/samples/data/label_ape.txt b/modules/cnn_3dobj/samples/data/label_ape.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/modules/cnn_3dobj/samples/data/label_cow.txt b/modules/cnn_3dobj/samples/data/label_cow.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/modules/cnn_3dobj/samples/data/label_plane.txt b/modules/cnn_3dobj/samples/data/label_plane.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/modules/cnn_3dobj/samples/images2db_demo.cpp b/modules/cnn_3dobj/samples/datatrans_demo.cpp similarity index 90% rename from modules/cnn_3dobj/samples/images2db_demo.cpp rename to modules/cnn_3dobj/samples/datatrans_demo.cpp index b4290147d..8df5675f7 100644 --- a/modules/cnn_3dobj/samples/images2db_demo.cpp +++ b/modules/cnn_3dobj/samples/datatrans_demo.cpp @@ -60,4 +60,5 @@ int main(int argc, char* argv[]) int height = parser.get("height"); cv::cnn_3dobj::DataTrans Trans; Trans.convert(src_dir,src_dst,attach_dir,channel,width,height); + std::cout << std::endl << "All images in: " << std::endl << src_dir << std::endl << "have been converted to levelDB data in: " << std::endl << src_dst << std::endl << "for extracting feature of gallery images in classification step efficiently, this convertion is not needed in feature extraction of test image" << std::endl; } diff --git a/modules/cnn_3dobj/samples/feature_extract_demo.cpp b/modules/cnn_3dobj/samples/feature_extract_demo.cpp index 26e9c0fa0..dc54ffe06 100644 --- a/modules/cnn_3dobj/samples/feature_extract_demo.cpp +++ b/modules/cnn_3dobj/samples/feature_extract_demo.cpp @@ -66,11 +66,11 @@ int main(int argc, char* argv[]) "{channel | 1 | Channel of the images. }" "{width | 64 | Width of images}" "{height | 64 | Height of images}" - "{pretrained_binary_proto | ../data/3d_triplet_iter_10000.caffemodel | caffe model for feature exrtaction.}" - "{feature_extraction_proto | ../data/3d_triplet_train_test.prototxt | network definition in .prototxt the path of the training samples must be wrotten in in .prototxt files in Phase TEST}" - "{save_feature_dataset_names | ../data/feature/feature_iter_10000.bin | the output of the extracted feature in form of binary files together with the vector features as the feature.}" + "{caffemodel | ../data/3d_triplet_iter_10000.caffemodel | caffe model for feature exrtaction.}" + "{network_forDB | ../data/3d_triplet_galleryIMG.prototxt | network definition in .prototxt the path of the training samples must be wrotten in in .prototxt files in Phase TEST}" + "{featurename_bin | ../data/feature/feature_iter_10000.bin | the output of the extracted feature in form of binary files together with the vector features as the feature.}" "{extract_feature_blob_names | feat | the layer used for feature extraction in CNN.}" - "{num_mini_batches | 6 | batches suit for the batches defined in the .proto for the aim of extracting feature from all images.}" + "{num_mini_batches | 4 | batches suit for the batches defined in the .proto for the aim of extracting feature from all images.}" "{device | CPU | device}" "{dev_id | 0 | dev_id}"; cv::CommandLineParser parser(argc, argv, keys); @@ -86,14 +86,16 @@ int main(int argc, char* argv[]) int channel = parser.get("channel"); int width = parser.get("width"); int height = parser.get("height"); - string pretrained_binary_proto = parser.get("pretrained_binary_proto"); - string feature_extraction_proto = parser.get("feature_extraction_proto"); - string save_feature_dataset_names = parser.get("save_feature_dataset_names"); + string caffemodel = parser.get("caffemodel"); + string network_forDB = parser.get("network_forDB"); + string featurename_bin = parser.get("featurename_bin"); string extract_feature_blob_names = parser.get("extract_feature_blob_names"); int num_mini_batches = parser.get("num_mini_batches"); string device = parser.get("device"); int dev_id = parser.get("dev_id"); cv::cnn_3dobj::DataTrans transTemp; transTemp.convert(src_dir,src_dst,attach_dir,channel,width,height); - std::vector extractedFeature = transTemp.feature_extraction_pipeline(pretrained_binary_proto, feature_extraction_proto, save_feature_dataset_names, extract_feature_blob_names, num_mini_batches, device, dev_id); + std::cout << std::endl << "All images in: " << std::endl << src_dir << std::endl << "have been converted to levelDB data in: " << std::endl << src_dst << std::endl << "for extracting feature of gallery images in classification step efficiently, this convertion is not needed in feature extraction of test image" << std::endl; + std::vector extractedFeature = transTemp.feature_extraction_pipeline(caffemodel, network_forDB, featurename_bin, extract_feature_blob_names, num_mini_batches, device, dev_id); + std::cout << std::endl << "All featrues of images in: " << std::endl << src_dir << std::endl << "have been extracted as binary file(using levelDB data) in:" << std::endl << featurename_bin << std::endl << "for analysis in Matlab and other software, this function also outputting a vector format gallery feature used for classificatioin."; } diff --git a/modules/cnn_3dobj/samples/sphereview_3dobj_demo.cpp b/modules/cnn_3dobj/samples/sphereview_3dobj_demo.cpp index e72b591cb..941113f0d 100644 --- a/modules/cnn_3dobj/samples/sphereview_3dobj_demo.cpp +++ b/modules/cnn_3dobj/samples/sphereview_3dobj_demo.cpp @@ -64,7 +64,7 @@ int main(int argc, char *argv[]){ std::vector campos = ViewSphere.CameraPos; std::fstream imglabel; char* p=(char*)labeldir.data(); - imglabel.open(p); + imglabel.open(p, fstream::app|fstream::out); bool camera_pov = (true); /// Create a window viz::Viz3d myWindow("Coordinate Frame"); @@ -84,7 +84,15 @@ int main(int argc, char *argv[]){ const char* binaryPath = "./binary_"; ViewSphere.createHeader((int)campos.size(), 64, 64, headerPath); for(int pose = 0; pose < (int)campos.size(); pose++){ - imglabel << campos.at(pose).x << ' ' << campos.at(pose).y << ' ' << campos.at(pose).z << endl; + char* temp = new char; + sprintf (temp,"%d",label_class); + string filename = temp; + filename += "_"; + sprintf (temp,"%d",pose); + filename += temp; + filename += ".png"; + imglabel << filename << ' ' << (int)(campos.at(pose).x*100) << ' ' << (int)(campos.at(pose).y*100) << ' ' << (int)(campos.at(pose).z*100) << endl; + filename = imagedir + filename; /// We can get the pose of the cam using makeCameraPoses Affine3f cam_pose = viz::makeCameraPose(campos.at(pose)*radius+cam_focal_point, cam_focal_point, cam_y_dir*radius+cam_focal_point); /// We can get the transformation matrix from camera coordinate system to global using @@ -111,16 +119,9 @@ int main(int argc, char *argv[]){ /// Set the viewer pose to that of camera if (camera_pov) myWindow.setViewerPose(cam_pose); - char* temp = new char; - sprintf (temp,"%d",label_class); - string filename = temp; - filename += "_"; - filename = imagedir + filename; - sprintf (temp,"%d",pose); - filename += temp; - filename += ".png"; myWindow.saveScreenshot(filename); ViewSphere.writeBinaryfile(filename, binaryPath, headerPath,(int)campos.size()*num_class, label_class); } + imglabel.close(); return 1; }; diff --git a/modules/cnn_3dobj/src/cnn_classification.cpp b/modules/cnn_3dobj/src/cnn_classification.cpp new file mode 100644 index 000000000..b81bce0bf --- /dev/null +++ b/modules/cnn_3dobj/src/cnn_classification.cpp @@ -0,0 +1,197 @@ +#include "precomp.hpp" +using namespace caffe; +using std::string; + +namespace cv +{ +namespace cnn_3dobj +{ + Classification::Classification(const string& model_file, const string& trained_file, const string& mean_file, const string& label_file) { + #ifdef CPU_ONLY + caffe::Caffe::set_mode(caffe::Caffe::CPU); + #else + caffe::Caffe::set_mode(caffe::Caffe::GPU); + #endif + + /* Load the network. */ + net_.reset(new Net(model_file, TEST)); + net_->CopyTrainedLayersFrom(trained_file); + + CHECK_EQ(net_->num_inputs(), 1) << "Network should have exactly one input."; + CHECK_EQ(net_->num_outputs(), 1) << "Network should have exactly one output."; + + Blob* input_layer = net_->input_blobs()[0]; + num_channels_ = input_layer->channels(); + CHECK(num_channels_ == 3 || num_channels_ == 1) + << "Input layer should have 1 or 3 channels."; + input_geometry_ = cv::Size(input_layer->width(), input_layer->height()); + + /* Load the binaryproto mean file. */ + SetMean(mean_file); + + /* Load labels. */ + std::ifstream labels(label_file.c_str()); + CHECK(labels) << "Unable to open labels file " << label_file; + string line; + while (std::getline(labels, line)) + labels_.push_back(string(line)); + + /* Blob* output_layer = net_->output_blobs()[0]; + CHECK_EQ(labels_.size(), output_layer->channels()) + << "Number of labels is different from the output layer dimension.";*/ + } + + /*bool Classifier::PairCompare(const std::pair& lhs, + const std::pair& rhs) { + return lhs.first > rhs.first; + }*/ + + /* Return the indices of the top N values of vector v. */ + std::vector Classification::Argmax(const std::vector& v, int N) { + std::vector > pairs; + for (size_t i = 0; i < v.size(); ++i) + pairs.push_back(std::make_pair(v[i], i)); + std::partial_sort(pairs.begin(), pairs.begin() + N, pairs.end()); + + std::vector result; + for (int i = 0; i < N; ++i) + result.push_back(pairs[i].second); + return result; + } + + //Return the top N predictions. + std::vector > Classification::Classify(const std::vector& reference, const cv::Mat& img, int N, bool mean_substract) { + cv::Mat feature = feature_extract(img, mean_substract); + std::vector output; + for (unsigned int i = 0; i < reference.size(); i++) { + cv::Mat f1 = reference.at(i); + cv::Mat f2 = feature; + cv::Mat output_temp = f1.t()-f2; + output.push_back(cv::norm(output_temp)); + } + std::vector maxN = Argmax(output, N); + std::vector > predictions; + for (int i = 0; i < N; ++i) { + int idx = maxN[i]; + predictions.push_back(std::make_pair(labels_[idx], output[idx])); + } + + return predictions; + } + + /* Load the mean file in binaryproto format. */ + void Classification::SetMean(const string& mean_file) { + BlobProto blob_proto; + ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto); + + /* Convert from BlobProto to Blob */ + Blob mean_blob; + mean_blob.FromProto(blob_proto); + CHECK_EQ(mean_blob.channels(), num_channels_) + << "Number of channels of mean file doesn't match input layer."; + + /* The format of the mean file is planar 32-bit float BGR or grayscale. */ + std::vector channels; + float* data = mean_blob.mutable_cpu_data(); + for (int i = 0; i < num_channels_; ++i) { + /* Extract an individual channel. */ + cv::Mat channel(mean_blob.height(), mean_blob.width(), CV_32FC1, data); + channels.push_back(channel); + data += mean_blob.height() * mean_blob.width(); + } + + /* Merge the separate channels into a single image. */ + cv::Mat mean; + cv::merge(channels, mean); + + /* Compute the global mean pixel value and create a mean image + * filled with this value. */ + cv::Scalar channel_mean = cv::mean(mean); + mean_ = cv::Mat(input_geometry_, mean.type(), channel_mean); + } + + cv::Mat Classification::feature_extract(const cv::Mat& img, bool mean_subtract) { + Blob* input_layer = net_->input_blobs()[0]; + input_layer->Reshape(1, num_channels_, + input_geometry_.height, input_geometry_.width); + /* Forward dimension change to all layers. */ + net_->Reshape(); + + std::vector input_channels; + WrapInputLayer(&input_channels); + + Preprocess(img, &input_channels, mean_subtract); + + net_->ForwardPrefilled(); + + /* Copy the output layer to a std::vector */ + Blob* output_layer = net_->output_blobs()[0]; + const float* begin = output_layer->cpu_data(); + const float* end = begin + output_layer->channels(); + //return std::vector(begin, end); + std::vector featureVec = std::vector(begin, end); + cv::Mat feature = cv::Mat(featureVec, true); + return feature; + } + + /* Wrap the input layer of the network in separate cv::Mat objects + * (one per channel). This way we save one memcpy operation and we + * don't need to rely on cudaMemcpy2D. The last preprocessing + * operation will write the separate channels directly to the input + * layer. */ + void Classification::WrapInputLayer(std::vector* input_channels) { + Blob* input_layer = net_->input_blobs()[0]; + + int width = input_layer->width(); + int height = input_layer->height(); + float* input_data = input_layer->mutable_cpu_data(); + for (int i = 0; i < input_layer->channels(); ++i) { + cv::Mat channel(height, width, CV_32FC1, input_data); + input_channels->push_back(channel); + input_data += width * height; + } + } + + void Classification::Preprocess(const cv::Mat& img, + std::vector* input_channels, bool mean_subtract) { + /* Convert the input image to the input image format of the network. */ + cv::Mat sample; + if (img.channels() == 3 && num_channels_ == 1) + cv::cvtColor(img, sample, CV_BGR2GRAY); + else if (img.channels() == 4 && num_channels_ == 1) + cv::cvtColor(img, sample, CV_BGRA2GRAY); + else if (img.channels() == 4 && num_channels_ == 3) + cv::cvtColor(img, sample, CV_BGRA2BGR); + else if (img.channels() == 1 && num_channels_ == 3) + cv::cvtColor(img, sample, CV_GRAY2BGR); + else + sample = img; + + cv::Mat sample_resized; + if (sample.size() != input_geometry_) + cv::resize(sample, sample_resized, input_geometry_); + else + sample_resized = sample; + + cv::Mat sample_float; + if (num_channels_ == 3) + sample_resized.convertTo(sample_float, CV_32FC3); + else + sample_resized.convertTo(sample_float, CV_32FC1); + + cv::Mat sample_normalized; + if (mean_subtract) + cv::subtract(sample_float, mean_, sample_normalized); + else + sample_normalized = sample_float; + + /* This operation will write the separate BGR planes directly to the + * input layer of the network because it is wrapped by the cv::Mat + * objects in input_channels. */ + cv::split(sample_normalized, *input_channels); + + CHECK(reinterpret_cast(input_channels->at(0).data) + == net_->input_blobs()[0]->cpu_data()) + << "Input channels are not wrapping the input layer of the network."; + } +}} diff --git a/modules/cnn_3dobj/src/cnn_image2db.cpp b/modules/cnn_3dobj/src/cnn_datatrans.cpp similarity index 98% rename from modules/cnn_3dobj/src/cnn_image2db.cpp rename to modules/cnn_3dobj/src/cnn_datatrans.cpp index 8020ca2b2..4cf9fb448 100644 --- a/modules/cnn_3dobj/src/cnn_image2db.cpp +++ b/modules/cnn_3dobj/src/cnn_datatrans.cpp @@ -96,7 +96,7 @@ namespace cnn_3dobj leveldb::DB* db; leveldb::Options options; options.create_if_missing = true; - options.error_if_exists = true; + // options.error_if_exists = true; caffe::Datum datum; datum.set_channels(channel); datum.set_height(height); @@ -213,11 +213,11 @@ namespace cnn_3dobj feature_blob_data = feature_blob->cpu_data() + feature_blob->offset(n); fwrite(feature_blob_data, sizeof(float), dim_features, files[i]); + cv::Mat tempfeat = cv::Mat(1, dim_features, CV_32FC1); for (int dim = 0; dim < dim_features; dim++) { - cv::Mat tempfeat = cv::Mat(1, dim_features, CV_32FC1); tempfeat.at(0,dim) = *(feature_blob_data++); - featureVec.push_back(tempfeat); } + featureVec.push_back(tempfeat); ++image_indices[i]; if (image_indices[i] % 1000 == 0) { LOG(ERROR)<< "Extracted features of " << image_indices[i] << diff --git a/modules/cnn_3dobj/src/precomp.hpp b/modules/cnn_3dobj/src/precomp.hpp index 1f6a05fc0..cdd5e11bf 100644 --- a/modules/cnn_3dobj/src/precomp.hpp +++ b/modules/cnn_3dobj/src/precomp.hpp @@ -43,37 +43,5 @@ the use of this software, even if advised of the possibility of such damage. #define __OPENCV_CNN_3DOBJ_PRECOMP_HPP__ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -//#include -//#include -//#include -#include -#include -#include -#define CPU_ONLY -#include -#include -#include -#include -#include -#include -using std::string; -using caffe::Blob; -using caffe::Caffe; -using caffe::Datum; -using caffe::Net; #endif