Chiebot-Mirror
/
opencv
mirror of https://github.com/opencv/opencv.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Layers for OpenFace face recognition network

Browse Source
pull/9524/head
Dmitry Kurtaev 8 years ago
parent 8a094a5f54
commit 7dc6b1d7d4
10 changed files with 328 additions and 34 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 16
      modules/dnn/include/opencv2/dnn/all_layers.hpp
  2. 2
      modules/dnn/src/dnn.cpp
  3. 1
      modules/dnn/src/init.cpp
  4. 39
      modules/dnn/src/layers/concat_layer.cpp
  5. 2
      modules/dnn/src/layers/convolution_layer.cpp
  6. 78
      modules/dnn/src/layers/lp_normalize_layer.cpp
  7. 11
      modules/dnn/src/layers/pooling_layer.cpp
  8. 17
      modules/dnn/src/layers/reshape_layer.cpp
  9. 149
      modules/dnn/src/torch/torch_importer.cpp
  10. 47
      modules/dnn/test/test_torch_importer.cpp

16
modules/dnn/include/opencv2/dnn/all_layers.hpp
Unescape View File

@ -245,6 +245,7 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
bool globalPooling;
bool computeMaxIdx;
String padMode;
bool ceilMode;
static Ptr<PoolingLayer> create(const LayerParams& params);
};
@ -257,6 +258,14 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
static Ptr<SoftmaxLayer> create(const LayerParams& params);
};
class CV_EXPORTS LPNormalizeLayer : public Layer
{
public:
float pnorm, epsilon;
static Ptr<LPNormalizeLayer> create(const LayerParams& params);
};
class CV_EXPORTS InnerProductLayer : public Layer
{
public:
@ -294,6 +303,13 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
{
public:
int axis;
/**
* @brief Add zero padding in case of concatenation of blobs with different
* spatial sizes.
*
* Details: https://github.com/torch/nn/blob/master/doc/containers.md#depthconcat
*/
bool padding;
static Ptr<ConcatLayer> create(const LayerParams &params);
};

2
modules/dnn/src/dnn.cpp
Unescape View File

@ -1177,7 +1177,7 @@ struct Net::Impl
// (and so we eliminate the concatenation layer, because the channels
// are concatenated implicitly).
Ptr<ConcatLayer> concatLayer = ld.layerInstance.dynamicCast<ConcatLayer>();
if( !concatLayer.empty() && concatLayer->axis == 1 &&
if( !concatLayer.empty() && concatLayer->axis == 1 && !concatLayer->padding &&
ld.outputBlobs.size() == 1 )
{
Mat& output = ld.outputBlobs[0];

1
modules/dnn/src/init.cpp
Unescape View File

@ -91,6 +91,7 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(InnerProduct, InnerProductLayer);
CV_DNN_REGISTER_LAYER_CLASS(Softmax, SoftmaxLayer);
CV_DNN_REGISTER_LAYER_CLASS(MVN, MVNLayer);
CV_DNN_REGISTER_LAYER_CLASS(LPNormalize, LPNormalizeLayer);
CV_DNN_REGISTER_LAYER_CLASS(ReLU, ReLULayer);
CV_DNN_REGISTER_LAYER_CLASS(ChannelsPReLU, ChannelsPReLULayer);

39
modules/dnn/src/layers/concat_layer.cpp
Unescape View File

@ -56,6 +56,7 @@ public:
{
setParamsFrom(params);
axis = params.get<int>("axis", 1);
padding = params.get<bool>("padding", false);
}
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
@ -64,8 +65,7 @@ public:
std::vector<MatShape> &internals) const
{
CV_Assert(inputs.size() > 0);
outputs.clear();
outputs.push_back(inputs[0]);
outputs.resize(1, inputs[0]);
int cAxis = clamp(axis, inputs[0]);
int axisSum = 0;
@ -73,25 +73,33 @@ public:
{
MatShape curShape = inputs[i];
CV_Assert(curShape.size() == outputs.back().size());
for (int curAxis = 0; curAxis < outputs.back().size(); curAxis++)
if (padding)
{
if (curAxis != cAxis && outputs.back()[curAxis] != curShape[curAxis])
CV_Error(Error::StsBadSize, "Inconsitent shape for ConcatLayer");
for (int curAxis = 0; curAxis < outputs[0].size(); curAxis++)
{
outputs[0][curAxis] = std::max(outputs[0][curAxis], curShape[curAxis]);
}
}
else
{
CV_Assert(curShape.size() == outputs[0].size());
for (int curAxis = 0; curAxis < outputs[0].size(); curAxis++)
{
if (curAxis != cAxis && outputs[0][curAxis] != curShape[curAxis])
CV_Error(Error::StsBadSize, "Inconsitent shape for ConcatLayer");
}
}
axisSum += curShape[cAxis];
}
outputs.back()[cAxis] = axisSum;
outputs[0][cAxis] = axisSum;
return false;
}
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT ||
backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1; // By channels
backendId == DNN_BACKEND_HALIDE && haveHalide() && axis == 1 && !padding; // By channels
}
class ChannelConcatInvoker : public ParallelLoopBody
@ -174,7 +182,10 @@ public:
int cAxis = clamp(axis, inputs[0]->dims);
Mat& outMat = outputs[0];
if( cAxis == 1 && outMat.dims == 4 )
if (padding)
outMat.setTo(0);
if( cAxis == 1 && outMat.dims == 4 && !padding)
{
int nstripes = getNumThreads();
ChannelConcatInvoker::run(inputs, outMat, nstripes);
@ -187,6 +198,12 @@ public:
for (size_t i = 0; i < inputs.size(); i++)
{
ranges[cAxis].end = ranges[cAxis].start + inputs[i]->size[cAxis];
for (int j = 0; j < outMat.dims; ++j)
{
if (j == cAxis) continue;
ranges[j].start = (outMat.size[j] - inputs[i]->size[j]) / 2;
ranges[j].end = ranges[j].start + inputs[i]->size[j];
}
inputs[i]->copyTo(outMat(&ranges[0]));
ranges[cAxis].start = ranges[cAxis].end;
}

2
modules/dnn/src/layers/convolution_layer.cpp
Unescape View File

@ -187,7 +187,7 @@ public:
}
int ngroups = inpCn / blobs[0].size[1];
CV_Assert(inpCn % ngroups == 0 && outCn % ngroups == 0);
CV_Assert(ngroups > 0 && inpCn % ngroups == 0 && outCn % ngroups == 0);
int dims[] = {inputs[0][0], outCn, out.height, out.width};
outputs.resize(inputs.size(), shape(dims));

78
modules/dnn/src/layers/lp_normalize_layer.cpp
Unescape View File

@ -0,0 +1,78 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2017, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "layers_common.hpp"
#include <iostream>
namespace cv { namespace dnn {
class LPNormalizeLayerImpl : public LPNormalizeLayer
{
public:
LPNormalizeLayerImpl(const LayerParams& params)
{
setParamsFrom(params);
pnorm = params.get<float>("p", 2);
epsilon = params.get<float>("eps", 1e-10f);
CV_Assert(pnorm > 0);
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const
{
Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals);
if (pnorm != 1 && pnorm != 2)
{
internals.resize(1, inputs[0]);
}
return true;
}
virtual bool supportBackend(int backendId)
{
return backendId == DNN_BACKEND_DEFAULT;
}
void forward(std::vector<Mat*> &inputs, std::vector<Mat> &outputs, std::vector<Mat> &internals)
{
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
CV_Assert(inputs[0]->total() == outputs[0].total());
float norm;
if (pnorm == 1)
norm = cv::norm(*inputs[0], NORM_L1);
else if (pnorm == 2)
norm = cv::norm(*inputs[0], NORM_L2);
else
{
pow(abs(*inputs[0]), pnorm, internals[0]);
norm = pow(sum(internals[0])[0], 1.0f / pnorm);
}
multiply(*inputs[0], 1.0f / (norm + epsilon), outputs[0]);
}
int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &) const
{
int64 flops = 0;
for (int i = 0; i < inputs.size(); i++)
flops += 3 * total(inputs[i]);
return flops;
}
};
Ptr<LPNormalizeLayer> LPNormalizeLayer::create(const LayerParams& params)
{
return Ptr<LPNormalizeLayer>(new LPNormalizeLayerImpl(params));
}
} // namespace dnn
} // namespace cv

11
modules/dnn/src/layers/pooling_layer.cpp
Unescape View File

@ -54,7 +54,6 @@ namespace cv
namespace dnn
{
//TODO: add ceil_mode param
class PoolingLayerImpl : public PoolingLayer
{
public:
@ -79,6 +78,7 @@ public:
getPoolingKernelParams(params, kernel.height, kernel.width, globalPooling,
pad.height, pad.width, stride.height, stride.width, padMode);
setParamsFrom(params);
ceilMode = params.get<bool>("ceil_mode", true);
}
void finalize(const std::vector<Mat*> &inputs, std::vector<Mat> &outputs)
@ -572,11 +572,10 @@ public:
}
else if (padMode.empty())
{
//Yeah, something strange Caffe scheme-)
out.height = static_cast<int>(ceil(static_cast<float>(in.height + 2 * pad.height -
kernel.height) / stride.height)) + 1;
out.width = static_cast<int>(ceil(static_cast<float>(in.width + 2 * pad.width -
kernel.width) / stride.width)) + 1;
float height = (float)(in.height + 2 * pad.height - kernel.height) / stride.height;
float width = (float)(in.width + 2 * pad.width - kernel.width) / stride.width;
out.height = 1 + (ceilMode ? ceil(height) : floor(height));
out.width = 1 + (ceilMode ? ceil(width) : floor(width));
if (pad.height || pad.width)
{

17
modules/dnn/src/layers/reshape_layer.cpp
Unescape View File

@ -75,12 +75,21 @@ static void computeShapeByReshapeMask(const MatShape &srcShape,
if (explicitMask)
{
int maskTotal = total(maskShape);
for (int i = srcRange.start + 1; i < srcRange.end; ++i)
// Go from the end of mask until we collect required total.
bool matched = false;
for (int i = srcRange.end - 1; i >= srcRange.start; --i)
{
if (total(srcShape, i, srcRange.end) != maskTotal)
if (matched)
{
srcRange.start = i - 1;
break;
if (i == 0 || total(srcShape, i, srcRange.end) != maskTotal)
{
srcRange.start = i + 1;
break;
}
}
else
{
matched = total(srcShape, i, srcRange.end) == maskTotal;
}
}
CV_Assert(total(srcShape, srcRange.start, srcRange.end) == maskTotal);

149
modules/dnn/src/torch/torch_importer.cpp
Unescape View File

@ -309,6 +309,7 @@ struct TorchImporter : public ::cv::dnn::Importer
if (vtype == TYPE_TORCH)
{
int index = readInt();
int numModules = curModule->modules.size();
readTorchObject(index);
if (tensors.count(index)) //tensor was readed
@ -324,6 +325,14 @@ struct TorchImporter : public ::cv::dnn::Importer
DictValue scalar = DictValue::arrayReal(matCasted.ptr<double>(), matCasted.total());
scalarParams.set(key, scalar);
}
else
{
// Only tensors and scalars are supported for table fields.
// i.e. nn.Inception has field `transfer` which is an
// activation layer. So we remove added modules as readTorchObject(index).
while (curModule->modules.size() > numModules)
curModule->modules.pop_back();
}
}
else if (vtype == TYPE_NUMBER)
{
@ -469,7 +478,8 @@ struct TorchImporter : public ::cv::dnn::Importer
layerParams.set("torch_index", index);
if (nnName == "Sequential" || nnName == "Parallel" ||
nnName == "Concat" || nnName == "ConcatTable" || nnName == "JoinTable")
nnName == "Concat" || nnName == "ConcatTable" || nnName == "JoinTable" ||
nnName == "DepthConcat" || nnName == "Inception")
{
Module *parentModule = curModule;
curModule->modules.push_back(newModule);
@ -490,8 +500,12 @@ struct TorchImporter : public ::cv::dnn::Importer
{
layerParams.set("dimension", scalarParams.get<int>("dimension"));
}
if (nnName == "DepthConcat")
{
layerParams.set("dimension", scalarParams.get<int>("dimension"));
}
}
else if (nnName == "SpatialConvolution")
else if (nnName == "SpatialConvolution" || nnName == "SpatialConvolutionMM")
{
newModule->apiType = "Convolution";
readTorchTable(scalarParams, tensorParams);
@ -507,8 +521,37 @@ struct TorchImporter : public ::cv::dnn::Importer
layerParams.set("num_output", scalarParams.get<int>("nOutputPlane"));
convertTorchKernelsParams(scalarParams, layerParams);
if (nnName == "SpatialConvolutionMM")
{
// Split weights from a [ outCh x inCh*kH*kW ] 2D matrix
// onto a 4D [ outCh x inCh x kH x kW ] blob.
CV_Assert(layerParams.blobs[0].dims == 2);
const int kernel = layerParams.blobs[0].size[1]; // inCh * kH * kW
MatShape kernelShape(4);
kernelShape[0] = layerParams.blobs[0].size[0]; // outCh.
kernelShape[2] = layerParams.get<int>("kernel_h");
kernelShape[3] = layerParams.get<int>("kernel_w");
kernelShape[1] = kernel / (kernelShape[2] * kernelShape[3]); // inCh.
layerParams.blobs[0] = layerParams.blobs[0].reshape(1, kernelShape);
}
curModule->modules.push_back(newModule);
}
else if (nnName == "SpatialLPPooling")
{
// nn.Sequential {
// [input -> (1) -> (2) -> output]
// (1): nn.Sequential {
// [input -> (1) -> (2) -> (3) -> (4) -> output]
// (1): nn.Power
// (2): nn.SpatialAveragePooling(...)
// (3): nn.MulConstant
// (4): nn.Power
// }
// (2): nn.Sigmoid
// }
// nn.SpatialLPPooling is just a table so we skip it.
readTorchTable(scalarParams, tensorParams);
}
else if (nnName == "SpatialMaxPooling" || nnName == "SpatialAveragePooling")
{
newModule->apiType = "Pooling";
@ -522,6 +565,9 @@ struct TorchImporter : public ::cv::dnn::Importer
layerParams.set("pool", "AVE");
convertTorchKernelsParams(scalarParams, layerParams);
CV_Assert(scalarParams.has("ceil_mode"));
layerParams.set("ceil_mode", scalarParams.get<bool>("ceil_mode"));
curModule->modules.push_back(newModule);
}
else if (nnName == "Linear")
@ -541,7 +587,7 @@ struct TorchImporter : public ::cv::dnn::Importer
layerParams.set("num_output", weightBlob.size[0]);
curModule->modules.push_back(newModule);
}
else if (nnName == "Reshape")
else if (nnName == "Reshape" || nnName == "View")
{
newModule->apiType = "Reshape";
@ -576,15 +622,24 @@ struct TorchImporter : public ::cv::dnn::Importer
newModule->apiType = "BatchNorm";
readTorchTable(scalarParams, tensorParams);
CV_Assert(tensorParams.count("running_var") &&
tensorParams.count("running_mean"));
layerParams.blobs.push_back(tensorParams["running_mean"].second);
layerParams.blobs.push_back(tensorParams["running_var"].second);
CV_Assert(scalarParams.has("eps"));
float eps = float(scalarParams.get<double>("eps"));
layerParams.set("eps", eps);
CV_Assert((tensorParams.count("running_var") || tensorParams.count("running_std")) &&
tensorParams.count("running_mean"));
layerParams.blobs.push_back(tensorParams["running_mean"].second);
if (tensorParams.count("running_var"))
{
layerParams.blobs.push_back(tensorParams["running_var"].second);
}
else
{
layerParams.blobs.push_back(tensorParams["running_std"].second);
pow(layerParams.blobs.back(), -2, layerParams.blobs.back());
subtract(layerParams.blobs.back(), eps, layerParams.blobs.back());
}
if (tensorParams.count("weight"))
{
layerParams.set("has_weight", true);
@ -642,6 +697,18 @@ struct TorchImporter : public ::cv::dnn::Importer
newModule->apiType = "Identity";
curModule->modules.push_back(newModule);
}
else if (nnName == "Normalize")
{
readTorchTable(scalarParams, tensorParams);
CV_Assert(scalarParams.has("p"));
layerParams.set("p", scalarParams.get<float>("p"));
if (scalarParams.has("eps"))
layerParams.set("eps", scalarParams.get<float>("eps"));
newModule->apiType = "LPNormalize";
curModule->modules.push_back(newModule);
}
else if (nnName == "Padding")
{
readTorchTable(scalarParams, tensorParams);
@ -760,6 +827,46 @@ struct TorchImporter : public ::cv::dnn::Importer
layerParams.set("log_softmax", true);
curModule->modules.push_back(newModule);
}
else if (nnName == "SpatialCrossMapLRN")
{
newModule->apiType = "LRN";
readTorchTable(scalarParams, tensorParams);
CV_Assert(scalarParams.has("alpha"));
CV_Assert(scalarParams.has("beta"));
CV_Assert(scalarParams.has("k"));
CV_Assert(scalarParams.has("size"));
layerParams.set("norm_region", "ACROSS_CHANNELS");
layerParams.set("alpha", scalarParams.get<float>("alpha"));
layerParams.set("beta", scalarParams.get<float>("beta"));
layerParams.set("bias", scalarParams.get<float>("k"));
layerParams.set("local_size", scalarParams.get<int>("size"));
layerParams.set("norm_by_size", true);
curModule->modules.push_back(newModule);
}
else if (nnName == "Square" || nnName == "Sqrt" || nnName == "Power")
{
readTorchTable(scalarParams, tensorParams);
float power;
if (nnName == "Square") power = 2.0f;
else if (nnName == "Sqrt") power = 0.5f;
else if (nnName == "Power") power = scalarParams.get<float>("pow", 1.0f);
newModule->apiType = "Power";
layerParams.set("power", power);
curModule->modules.push_back(newModule);
}
else if (nnName == "MulConstant")
{
readTorchTable(scalarParams, tensorParams);
CV_Assert(scalarParams.has("constant_scalar"));
newModule->apiType = "Power";
layerParams.set("scale", scalarParams.get<float>("constant_scalar"));
curModule->modules.push_back(newModule);
}
else
{
CV_Error(Error::StsNotImplemented, "Unknown nn class \"" + className + "\"");
@ -816,7 +923,7 @@ struct TorchImporter : public ::cv::dnn::Importer
}
else
{
if (module->thName == "Sequential")
if (module->thName == "Sequential" || module->thName == "Inception")
{
for (size_t i = 0; i < module->modules.size(); i++)
{
@ -851,6 +958,30 @@ struct TorchImporter : public ::cv::dnn::Importer
addedModules.push_back(std::make_pair(mergeId, module));
return mergeId;
}
else if (module->thName == "DepthConcat")
{
int newId, mergeId;
LayerParams mergeParams;
mergeParams.set("axis", module->params.get<int>("dimension") - 1);
mergeParams.set("padding", true);
std::vector<int> branchIds;
for (int i = 0; i < (int)module->modules.size(); i++)
{
newId = fill(module->modules[i], addedModules, prevLayerId, prevOutNum);
branchIds.push_back(newId);
}
mergeId = net.addLayer(generateLayerName("torchMerge"), "Concat", mergeParams);
for (int i = 0; i < branchIds.size(); i++)
{
net.connect(branchIds[i], 0, mergeId, i);
}
addedModules.push_back(std::make_pair(mergeId, module));
return mergeId;
}
else if (module->thName == "Parallel")
{
int newId, splitId, mergeId, reshapeId;

47
modules/dnn/test/test_torch_importer.cpp
Unescape View File

@ -56,11 +56,11 @@ using namespace cv::dnn;
template<typename TStr>
static std::string _tf(TStr filename, bool inTorchDir = true)
{
String path = getOpenCVExtraDir() + "/dnn/";
String path = "dnn/";
if (inTorchDir)
path += "torch/";
path += filename;
return path;
return findDataFile(path, false);
}
TEST(Torch_Importer, simple_read)
@ -123,6 +123,7 @@ TEST(Torch_Importer, run_reshape)
runTorchNet("net_reshape");
runTorchNet("net_reshape_batch");
runTorchNet("net_reshape_single_sample");
runTorchNet("net_reshape_channels", "", false, true);
}
TEST(Torch_Importer, run_linear)
@ -138,6 +139,7 @@ TEST(Torch_Importer, run_paralel)
TEST(Torch_Importer, run_concat)
{
runTorchNet("net_concat", "l5_torchMerge");
runTorchNet("net_depth_concat", "", false, true);
}
TEST(Torch_Importer, run_deconv)
@ -172,6 +174,27 @@ TEST(Torch_Importer, net_logsoftmax)
runTorchNet("net_logsoftmax_spatial");
}
TEST(Torch_Importer, net_lp_pooling)
{
runTorchNet("net_lp_pooling_square", "", false, true);
runTorchNet("net_lp_pooling_power", "", false, true);
}
TEST(Torch_Importer, net_conv_gemm_lrn)
{
runTorchNet("net_conv_gemm_lrn", "", false, true);
}
TEST(Torch_Importer, net_inception_block)
{
runTorchNet("net_inception_block", "", false, true);
}
TEST(Torch_Importer, net_normalize)
{
runTorchNet("net_normalize", "", false, true);
}
TEST(Torch_Importer, ENet_accuracy)
{
Net net;
@ -202,6 +225,26 @@ TEST(Torch_Importer, ENet_accuracy)
}
}
TEST(Torch_Importer, OpenFace_accuracy)
{
const string model = findDataFile("dnn/openface_nn4.small2.v1.t7", false);
Net net = readNetFromTorch(model);
Mat sample = imread(findDataFile("cv/shared/lena.png", false));
Mat sampleF32(sample.size(), CV_32FC3);
sample.convertTo(sampleF32, sampleF32.type());
sampleF32 /= 255;
resize(sampleF32, sampleF32, Size(96, 96), 0, 0, INTER_NEAREST);
Mat inputBlob = blobFromImage(sampleF32);
net.setInput(inputBlob);
Mat out = net.forward();
Mat outRef = readTorchBlob(_tf("net_openface_output.dat"), true);
normAssert(out, outRef);
}
}
#endif

Write Preview
Loading…
Cancel
Save