Wangyida
9 years ago
parent
d6cb8889b4
commit
6f38d89160
17 changed files with 481 additions and 134 deletions
@ -0,0 +1,112 @@ |
|||||||
|
/*
|
||||||
|
* Software License Agreement (BSD License) |
||||||
|
* |
||||||
|
* Copyright (c) 2009, Willow Garage, Inc. |
||||||
|
* All rights reserved. |
||||||
|
* |
||||||
|
* Redistribution and use in source and binary forms, with or without |
||||||
|
* modification, are permitted provided that the following conditions |
||||||
|
* are met: |
||||||
|
* |
||||||
|
* * Redistributions of source code must retain the above copyright |
||||||
|
* notice, this list of conditions and the following disclaimer. |
||||||
|
* * Redistributions in binary form must reproduce the above |
||||||
|
* copyright notice, this list of conditions and the following |
||||||
|
* disclaimer in the documentation and/or other materials provided |
||||||
|
* with the distribution. |
||||||
|
* * Neither the name of Willow Garage, Inc. nor the names of its |
||||||
|
* contributors may be used to endorse or promote products derived |
||||||
|
* from this software without specific prior written permission. |
||||||
|
* |
||||||
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||||
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||||
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS |
||||||
|
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE |
||||||
|
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
||||||
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, |
||||||
|
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||||
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||||
|
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
||||||
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
||||||
|
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
||||||
|
* POSSIBILITY OF SUCH DAMAGE. |
||||||
|
* |
||||||
|
*/ |
||||||
|
#include <opencv2/cnn_3dobj.hpp> |
||||||
|
#include <iomanip> |
||||||
|
using namespace cv; |
||||||
|
using namespace std; |
||||||
|
using namespace cv::cnn_3dobj; |
||||||
|
int main(int argc, char** argv) |
||||||
|
{ |
||||||
|
const String keys = "{help | | this demo will convert a set of images in a particular path into leveldb database for feature extraction using Caffe.}" |
||||||
|
"{src_dir | ../data/images_all/ | Source direction of the images ready for being converted to leveldb dataset.}" |
||||||
|
"{src_dst | ../data/dbfile | Aim direction of the converted to leveldb dataset. }" |
||||||
|
"{attach_dir | ../data/dbfile | Path for saving additional files which describe the transmission results. }" |
||||||
|
"{channel | 1 | Channel of the images. }" |
||||||
|
"{width | 64 | Width of images}" |
||||||
|
"{height | 64 | Height of images}" |
||||||
|
"{caffemodel | ../data/3d_triplet_iter_10000.caffemodel | caffe model for feature exrtaction.}" |
||||||
|
"{network_forDB | ../data/3d_triplet_galleryIMG.prototxt | Network definition file used for extracting feature from levelDB data, causion: the path of levelDB training samples must be wrotten in in .prototxt files in Phase TEST}" |
||||||
|
"{save_feature_dataset_names | ../data/feature/feature_iter_10000.bin | Output of the extracted feature in form of binary files together with the vector<cv::Mat> features as the feature.}" |
||||||
|
"{extract_feature_blob_names | feat | Layer used for feature extraction in CNN.}" |
||||||
|
"{num_mini_batches | 4 | Batches suit for the batches defined in the .proto for the aim of extracting feature from all images.}" |
||||||
|
"{device | CPU | Device: CPU or GPU.}" |
||||||
|
"{dev_id | 0 | ID of GPU.}" |
||||||
|
"{network_forIMG | ../data/3d_triplet_testIMG.prototxt | Network definition file used for extracting feature from a single image and making a classification}" |
||||||
|
"{mean_file | ../data/images_mean/triplet_mean.binaryproto | The mean file generated by Caffe from all gallery images, this could be used for mean value substraction from all images.}" |
||||||
|
"{label_file | ../data/dbfileimage_filename | A namelist including all gallery images.}" |
||||||
|
"{target_img | ../data/images_all/2_13.png | Path of image waiting to be classified.}" |
||||||
|
"{num_candidate | 6 | Number of candidates in gallery as the prediction result.}"; |
||||||
|
cv::CommandLineParser parser(argc, argv, keys); |
||||||
|
parser.about("Demo for Sphere View data generation"); |
||||||
|
if (parser.has("help")) |
||||||
|
{ |
||||||
|
parser.printMessage(); |
||||||
|
return 0; |
||||||
|
} |
||||||
|
string src_dir = parser.get<string>("src_dir"); |
||||||
|
string src_dst = parser.get<string>("src_dst"); |
||||||
|
string attach_dir = parser.get<string>("attach_dir"); |
||||||
|
int channel = parser.get<int>("channel"); |
||||||
|
int width = parser.get<int>("width"); |
||||||
|
int height = parser.get<int>("height"); |
||||||
|
string caffemodel = parser.get<string>("caffemodel"); |
||||||
|
string network_forDB = parser.get<string>("network_forDB"); |
||||||
|
string save_feature_dataset_names = parser.get<string>("save_feature_dataset_names"); |
||||||
|
string extract_feature_blob_names = parser.get<string>("extract_feature_blob_names"); |
||||||
|
int num_mini_batches = parser.get<int>("num_mini_batches"); |
||||||
|
string device = parser.get<string>("device"); |
||||||
|
int dev_id = parser.get<int>("dev_id"); |
||||||
|
string network_forIMG = parser.get<string>("network_forIMG"); |
||||||
|
string mean_file = parser.get<string>("mean_file"); |
||||||
|
string label_file = parser.get<string>("label_file"); |
||||||
|
string target_img = parser.get<string>("target_img"); |
||||||
|
int num_candidate = parser.get<int>("num_candidate"); |
||||||
|
cv::cnn_3dobj::DataTrans transTemp; |
||||||
|
transTemp.convert(src_dir,src_dst,attach_dir,channel,width,height); |
||||||
|
std::vector<cv::Mat> feature_reference = transTemp.feature_extraction_pipeline(caffemodel, network_forDB, save_feature_dataset_names, extract_feature_blob_names, num_mini_batches, device, dev_id); |
||||||
|
////start another demo
|
||||||
|
cv::cnn_3dobj::Classification classifier(network_forIMG, caffemodel, mean_file, label_file); |
||||||
|
|
||||||
|
std::cout << std::endl << "---------- Prediction for " |
||||||
|
<< target_img << " ----------" << std::endl; |
||||||
|
|
||||||
|
cv::Mat img = cv::imread(target_img, -1); |
||||||
|
// CHECK(!img.empty()) << "Unable to decode image " << target_img;
|
||||||
|
std::cout << std::endl << "---------- Featrue of gallery images ----------" << std::endl; |
||||||
|
std::vector<std::pair<string, float> > prediction; |
||||||
|
for (unsigned int i = 0; i < feature_reference.size(); i++) |
||||||
|
std::cout << feature_reference[i] << endl; |
||||||
|
cv::Mat feature_test = classifier.feature_extract(img, false); |
||||||
|
std::cout << std::endl << "---------- Featrue of target image: " << target_img << "----------" << endl << feature_test.t() << std::endl; |
||||||
|
prediction = classifier.Classify(feature_reference, img, num_candidate, false); |
||||||
|
// Print the top N prediction.
|
||||||
|
std::cout << std::endl << "---------- Prediction result(distance - file name in gallery) ----------" << std::endl; |
||||||
|
for (size_t i = 0; i < prediction.size(); ++i) { |
||||||
|
std::pair<string, float> p = prediction[i]; |
||||||
|
std::cout << std::fixed << std::setprecision(2) << p.second << " - \"" |
||||||
|
<< p.first << "\"" << std::endl; |
||||||
|
} |
||||||
|
return 0; |
||||||
|
} |
||||||
@ -0,0 +1,86 @@ |
|||||||
|
name: "3d_triplet" |
||||||
|
input: "data" |
||||||
|
input_dim: 1 |
||||||
|
input_dim: 1 |
||||||
|
input_dim: 64 |
||||||
|
input_dim: 64 |
||||||
|
layer { |
||||||
|
name: "conv1" |
||||||
|
type: "Convolution" |
||||||
|
bottom: "data" |
||||||
|
top: "conv1" |
||||||
|
convolution_param { |
||||||
|
num_output: 16 |
||||||
|
kernel_size: 8 |
||||||
|
stride: 1 |
||||||
|
} |
||||||
|
} |
||||||
|
layer { |
||||||
|
name: "pool1" |
||||||
|
type: "Pooling" |
||||||
|
bottom: "conv1" |
||||||
|
top: "pool1" |
||||||
|
pooling_param { |
||||||
|
pool: MAX |
||||||
|
kernel_size: 2 |
||||||
|
stride: 2 |
||||||
|
} |
||||||
|
} |
||||||
|
layer { |
||||||
|
name: "relu1" |
||||||
|
type: "ReLU" |
||||||
|
bottom: "pool1" |
||||||
|
top: "pool1" |
||||||
|
} |
||||||
|
layer { |
||||||
|
name: "conv2" |
||||||
|
type: "Convolution" |
||||||
|
bottom: "pool1" |
||||||
|
top: "conv2" |
||||||
|
convolution_param { |
||||||
|
num_output: 7 |
||||||
|
kernel_size: 5 |
||||||
|
stride: 1 |
||||||
|
} |
||||||
|
} |
||||||
|
layer { |
||||||
|
name: "pool2" |
||||||
|
type: "Pooling" |
||||||
|
bottom: "conv2" |
||||||
|
top: "pool2" |
||||||
|
pooling_param { |
||||||
|
pool: MAX |
||||||
|
kernel_size: 2 |
||||||
|
stride: 2 |
||||||
|
} |
||||||
|
} |
||||||
|
layer { |
||||||
|
name: "relu2" |
||||||
|
type: "ReLU" |
||||||
|
bottom: "pool2" |
||||||
|
top: "pool2" |
||||||
|
} |
||||||
|
layer { |
||||||
|
name: "ip1" |
||||||
|
type: "InnerProduct" |
||||||
|
bottom: "pool2" |
||||||
|
top: "ip1" |
||||||
|
inner_product_param { |
||||||
|
num_output: 256 |
||||||
|
} |
||||||
|
} |
||||||
|
layer { |
||||||
|
name: "relu3" |
||||||
|
type: "ReLU" |
||||||
|
bottom: "ip1" |
||||||
|
top: "ip1" |
||||||
|
} |
||||||
|
layer { |
||||||
|
name: "feat" |
||||||
|
type: "InnerProduct" |
||||||
|
bottom: "ip1" |
||||||
|
top: "feat" |
||||||
|
inner_product_param { |
||||||
|
num_output: 4 |
||||||
|
} |
||||||
|
} |
||||||
Binary file not shown.
@ -0,0 +1,197 @@ |
|||||||
|
#include "precomp.hpp" |
||||||
|
using namespace caffe; |
||||||
|
using std::string; |
||||||
|
|
||||||
|
namespace cv |
||||||
|
{ |
||||||
|
namespace cnn_3dobj |
||||||
|
{ |
||||||
|
Classification::Classification(const string& model_file, const string& trained_file, const string& mean_file, const string& label_file) { |
||||||
|
#ifdef CPU_ONLY |
||||||
|
caffe::Caffe::set_mode(caffe::Caffe::CPU); |
||||||
|
#else |
||||||
|
caffe::Caffe::set_mode(caffe::Caffe::GPU); |
||||||
|
#endif |
||||||
|
|
||||||
|
/* Load the network. */ |
||||||
|
net_.reset(new Net<float>(model_file, TEST)); |
||||||
|
net_->CopyTrainedLayersFrom(trained_file); |
||||||
|
|
||||||
|
CHECK_EQ(net_->num_inputs(), 1) << "Network should have exactly one input."; |
||||||
|
CHECK_EQ(net_->num_outputs(), 1) << "Network should have exactly one output."; |
||||||
|
|
||||||
|
Blob<float>* input_layer = net_->input_blobs()[0]; |
||||||
|
num_channels_ = input_layer->channels(); |
||||||
|
CHECK(num_channels_ == 3 || num_channels_ == 1) |
||||||
|
<< "Input layer should have 1 or 3 channels."; |
||||||
|
input_geometry_ = cv::Size(input_layer->width(), input_layer->height()); |
||||||
|
|
||||||
|
/* Load the binaryproto mean file. */ |
||||||
|
SetMean(mean_file); |
||||||
|
|
||||||
|
/* Load labels. */ |
||||||
|
std::ifstream labels(label_file.c_str()); |
||||||
|
CHECK(labels) << "Unable to open labels file " << label_file; |
||||||
|
string line; |
||||||
|
while (std::getline(labels, line)) |
||||||
|
labels_.push_back(string(line)); |
||||||
|
|
||||||
|
/* Blob<float>* output_layer = net_->output_blobs()[0];
|
||||||
|
CHECK_EQ(labels_.size(), output_layer->channels()) |
||||||
|
<< "Number of labels is different from the output layer dimension.";*/ |
||||||
|
} |
||||||
|
|
||||||
|
/*bool Classifier::PairCompare(const std::pair<float, int>& lhs,
|
||||||
|
const std::pair<float, int>& rhs) { |
||||||
|
return lhs.first > rhs.first; |
||||||
|
}*/ |
||||||
|
|
||||||
|
/* Return the indices of the top N values of vector v. */ |
||||||
|
std::vector<int> Classification::Argmax(const std::vector<float>& v, int N) { |
||||||
|
std::vector<std::pair<float, int> > pairs; |
||||||
|
for (size_t i = 0; i < v.size(); ++i) |
||||||
|
pairs.push_back(std::make_pair(v[i], i)); |
||||||
|
std::partial_sort(pairs.begin(), pairs.begin() + N, pairs.end()); |
||||||
|
|
||||||
|
std::vector<int> result; |
||||||
|
for (int i = 0; i < N; ++i) |
||||||
|
result.push_back(pairs[i].second); |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
//Return the top N predictions.
|
||||||
|
std::vector<std::pair<string, float> > Classification::Classify(const std::vector<cv::Mat>& reference, const cv::Mat& img, int N, bool mean_substract) { |
||||||
|
cv::Mat feature = feature_extract(img, mean_substract); |
||||||
|
std::vector<float> output; |
||||||
|
for (unsigned int i = 0; i < reference.size(); i++) { |
||||||
|
cv::Mat f1 = reference.at(i); |
||||||
|
cv::Mat f2 = feature; |
||||||
|
cv::Mat output_temp = f1.t()-f2; |
||||||
|
output.push_back(cv::norm(output_temp)); |
||||||
|
} |
||||||
|
std::vector<int> maxN = Argmax(output, N); |
||||||
|
std::vector<std::pair<string, float> > predictions; |
||||||
|
for (int i = 0; i < N; ++i) { |
||||||
|
int idx = maxN[i]; |
||||||
|
predictions.push_back(std::make_pair(labels_[idx], output[idx])); |
||||||
|
} |
||||||
|
|
||||||
|
return predictions; |
||||||
|
} |
||||||
|
|
||||||
|
/* Load the mean file in binaryproto format. */ |
||||||
|
void Classification::SetMean(const string& mean_file) { |
||||||
|
BlobProto blob_proto; |
||||||
|
ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto); |
||||||
|
|
||||||
|
/* Convert from BlobProto to Blob<float> */ |
||||||
|
Blob<float> mean_blob; |
||||||
|
mean_blob.FromProto(blob_proto); |
||||||
|
CHECK_EQ(mean_blob.channels(), num_channels_) |
||||||
|
<< "Number of channels of mean file doesn't match input layer."; |
||||||
|
|
||||||
|
/* The format of the mean file is planar 32-bit float BGR or grayscale. */ |
||||||
|
std::vector<cv::Mat> channels; |
||||||
|
float* data = mean_blob.mutable_cpu_data(); |
||||||
|
for (int i = 0; i < num_channels_; ++i) { |
||||||
|
/* Extract an individual channel. */ |
||||||
|
cv::Mat channel(mean_blob.height(), mean_blob.width(), CV_32FC1, data); |
||||||
|
channels.push_back(channel); |
||||||
|
data += mean_blob.height() * mean_blob.width(); |
||||||
|
} |
||||||
|
|
||||||
|
/* Merge the separate channels into a single image. */ |
||||||
|
cv::Mat mean; |
||||||
|
cv::merge(channels, mean); |
||||||
|
|
||||||
|
/* Compute the global mean pixel value and create a mean image
|
||||||
|
* filled with this value. */ |
||||||
|
cv::Scalar channel_mean = cv::mean(mean); |
||||||
|
mean_ = cv::Mat(input_geometry_, mean.type(), channel_mean); |
||||||
|
} |
||||||
|
|
||||||
|
cv::Mat Classification::feature_extract(const cv::Mat& img, bool mean_subtract) { |
||||||
|
Blob<float>* input_layer = net_->input_blobs()[0]; |
||||||
|
input_layer->Reshape(1, num_channels_, |
||||||
|
input_geometry_.height, input_geometry_.width); |
||||||
|
/* Forward dimension change to all layers. */ |
||||||
|
net_->Reshape(); |
||||||
|
|
||||||
|
std::vector<cv::Mat> input_channels; |
||||||
|
WrapInputLayer(&input_channels); |
||||||
|
|
||||||
|
Preprocess(img, &input_channels, mean_subtract); |
||||||
|
|
||||||
|
net_->ForwardPrefilled(); |
||||||
|
|
||||||
|
/* Copy the output layer to a std::vector */ |
||||||
|
Blob<float>* output_layer = net_->output_blobs()[0]; |
||||||
|
const float* begin = output_layer->cpu_data(); |
||||||
|
const float* end = begin + output_layer->channels(); |
||||||
|
//return std::vector<float>(begin, end);
|
||||||
|
std::vector<float> featureVec = std::vector<float>(begin, end); |
||||||
|
cv::Mat feature = cv::Mat(featureVec, true); |
||||||
|
return feature; |
||||||
|
} |
||||||
|
|
||||||
|
/* Wrap the input layer of the network in separate cv::Mat objects
|
||||||
|
* (one per channel). This way we save one memcpy operation and we |
||||||
|
* don't need to rely on cudaMemcpy2D. The last preprocessing |
||||||
|
* operation will write the separate channels directly to the input |
||||||
|
* layer. */ |
||||||
|
void Classification::WrapInputLayer(std::vector<cv::Mat>* input_channels) { |
||||||
|
Blob<float>* input_layer = net_->input_blobs()[0]; |
||||||
|
|
||||||
|
int width = input_layer->width(); |
||||||
|
int height = input_layer->height(); |
||||||
|
float* input_data = input_layer->mutable_cpu_data(); |
||||||
|
for (int i = 0; i < input_layer->channels(); ++i) { |
||||||
|
cv::Mat channel(height, width, CV_32FC1, input_data); |
||||||
|
input_channels->push_back(channel); |
||||||
|
input_data += width * height; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void Classification::Preprocess(const cv::Mat& img, |
||||||
|
std::vector<cv::Mat>* input_channels, bool mean_subtract) { |
||||||
|
/* Convert the input image to the input image format of the network. */ |
||||||
|
cv::Mat sample; |
||||||
|
if (img.channels() == 3 && num_channels_ == 1) |
||||||
|
cv::cvtColor(img, sample, CV_BGR2GRAY); |
||||||
|
else if (img.channels() == 4 && num_channels_ == 1) |
||||||
|
cv::cvtColor(img, sample, CV_BGRA2GRAY); |
||||||
|
else if (img.channels() == 4 && num_channels_ == 3) |
||||||
|
cv::cvtColor(img, sample, CV_BGRA2BGR); |
||||||
|
else if (img.channels() == 1 && num_channels_ == 3) |
||||||
|
cv::cvtColor(img, sample, CV_GRAY2BGR); |
||||||
|
else |
||||||
|
sample = img; |
||||||
|
|
||||||
|
cv::Mat sample_resized; |
||||||
|
if (sample.size() != input_geometry_) |
||||||
|
cv::resize(sample, sample_resized, input_geometry_); |
||||||
|
else |
||||||
|
sample_resized = sample; |
||||||
|
|
||||||
|
cv::Mat sample_float; |
||||||
|
if (num_channels_ == 3) |
||||||
|
sample_resized.convertTo(sample_float, CV_32FC3); |
||||||
|
else |
||||||
|
sample_resized.convertTo(sample_float, CV_32FC1); |
||||||
|
|
||||||
|
cv::Mat sample_normalized; |
||||||
|
if (mean_subtract) |
||||||
|
cv::subtract(sample_float, mean_, sample_normalized); |
||||||
|
else |
||||||
|
sample_normalized = sample_float; |
||||||
|
|
||||||
|
/* This operation will write the separate BGR planes directly to the
|
||||||
|
* input layer of the network because it is wrapped by the cv::Mat |
||||||
|
* objects in input_channels. */ |
||||||
|
cv::split(sample_normalized, *input_channels); |
||||||
|
|
||||||
|
CHECK(reinterpret_cast<float*>(input_channels->at(0).data) |
||||||
|
== net_->input_blobs()[0]->cpu_data()) |
||||||
|
<< "Input channels are not wrapping the input layer of the network."; |
||||||
|
} |
||||||
|
}} |
||||||
Loading…
Reference in new issue