Merge pull request #23047 from fengyuentau:layer_norm
dnn: add layer normalization for vision transformers * add layer norm onnx parser, impl and tests * add onnx graph simplifier for layer norm expanded * handle the case when constants are of type Initializer * add test case for layer norm expanded with initializers * use CV_Assert & CV_CheckType in place of CV_Assert_N; use forward_fallback for OCL_FP16 * use const ref / ref in parameters of invoker::run; extract inner const if from nested loop; use size_t in place of ull * template hasBias * remove trailing whitespace * use pointer parameter with null check; move normSize division & mean_square division outside of loop; use std::max to ensure positive value before std::sqrt * refactor implementation, optimize parallel_for * disable layer norm expanded * remove the removal of layer norm optional outputspull/23188/head
Yuantao Feng
2 years ago
committed by
GitHub
parent
52855a39ad
commit
4d918ba40b
7 changed files with 636 additions and 0 deletions
@ -0,0 +1,176 @@ |
|||||||
|
// This file is part of OpenCV project.
|
||||||
|
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||||
|
// of this distribution and at http://opencv.org/license.html.
|
||||||
|
|
||||||
|
#include "../precomp.hpp" |
||||||
|
#include "layers_common.hpp" |
||||||
|
|
||||||
|
namespace cv { namespace dnn { |
||||||
|
|
||||||
|
class LayerNormLayerImpl CV_FINAL : public LayerNormLayer |
||||||
|
{ |
||||||
|
public: |
||||||
|
LayerNormLayerImpl(const LayerParams& params) |
||||||
|
{ |
||||||
|
setParamsFrom(params); |
||||||
|
|
||||||
|
// standard attr
|
||||||
|
axis = params.get<int>("axis", 0); |
||||||
|
epsilon = params.get<float>("epsilon", 1e-5); |
||||||
|
|
||||||
|
// opencv attr
|
||||||
|
hasBias = params.get<bool>("hasBias", false); |
||||||
|
} |
||||||
|
|
||||||
|
virtual bool supportBackend(int backendId) CV_OVERRIDE |
||||||
|
{ |
||||||
|
return backendId == DNN_BACKEND_OPENCV; |
||||||
|
} |
||||||
|
|
||||||
|
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs, |
||||||
|
const int requiredOutputs, |
||||||
|
std::vector<MatShape> &outputs, |
||||||
|
std::vector<MatShape> &internals) const CV_OVERRIDE |
||||||
|
{ |
||||||
|
// check shapes of weight and bias if existed
|
||||||
|
// inputs >= 2 (X and Weight are requested, bias is optional)
|
||||||
|
CV_Check(inputs.size(), inputs.size() >= 2 && inputs.size() <= 3, "LayerNorm: require two (x, weight) or three (x, weight, bias) inputs"); |
||||||
|
|
||||||
|
auto x_shape = inputs[0]; |
||||||
|
int x_ndims = static_cast<int>(x_shape.size()); |
||||||
|
|
||||||
|
auto w_shape = inputs[1]; |
||||||
|
// if axis == last_dim, scale and b are both 1d tensor (represented as 2d mat nx1)
|
||||||
|
int w_ndims = static_cast<int>(w_shape.size()); |
||||||
|
w_ndims = (axis == x_ndims - 1 && w_ndims == 2) ? w_ndims - 1 : w_ndims; |
||||||
|
CV_CheckEQ(x_ndims - axis, w_ndims, "LayerNorm: shape of weight does not match with given axis and shape of input"); |
||||||
|
for (int i = 0; i < w_ndims; ++i) |
||||||
|
CV_CheckEQ(x_shape[axis+i], w_shape[i], "LayerNorm: weight dimensions does not match with input dimensions"); |
||||||
|
if (hasBias) |
||||||
|
{ |
||||||
|
CV_CheckEQ(inputs.size(), (size_t)3, ""); |
||||||
|
auto b_shape = inputs[2]; |
||||||
|
CV_CheckEQ(w_shape.size(), b_shape.size(), "LayerNorm: shape of weight does not match with shape of bias"); |
||||||
|
for (size_t i = 0; i < w_shape.size(); ++i) |
||||||
|
CV_CheckEQ(w_shape[i], b_shape[i], "LayerNorm: bias dimensions does not match with weight dimensions"); |
||||||
|
} |
||||||
|
|
||||||
|
// only one output is needed; Mean & InvStdDev are not needed
|
||||||
|
// in inference and should beomitted in onnx importer
|
||||||
|
outputs.assign(1, inputs[0]); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
template<bool hasBias> |
||||||
|
class LayerNormInvoker : public ParallelLoopBody |
||||||
|
{ |
||||||
|
public: |
||||||
|
const Mat& src; |
||||||
|
const float* scaleData; |
||||||
|
const float* biasData; |
||||||
|
Mat& dst; |
||||||
|
|
||||||
|
float epsilon; |
||||||
|
|
||||||
|
int total; |
||||||
|
int normSize; |
||||||
|
float invNormSize; |
||||||
|
|
||||||
|
LayerNormInvoker(const Mat& src_, const Mat& scale, const Mat* b, Mat& dst_, int axis, float epsilon_) |
||||||
|
: src(src_), scaleData(scale.ptr<float>()), biasData(nullptr), dst(dst_), epsilon(epsilon_) |
||||||
|
{ |
||||||
|
if (hasBias) |
||||||
|
{ |
||||||
|
CV_Assert(b != nullptr); |
||||||
|
CV_Assert(b->isContinuous()); |
||||||
|
biasData = (const float*)b->ptr<float>(); |
||||||
|
} |
||||||
|
|
||||||
|
auto dstShape = shape(dst); |
||||||
|
total = std::accumulate(dstShape.begin(), dstShape.begin() + axis, 1, std::multiplies<int>()); |
||||||
|
normSize = std::accumulate(dstShape.begin() + axis, dstShape.end(), 1, std::multiplies<int>()); |
||||||
|
invNormSize = 1.0f / normSize; |
||||||
|
} |
||||||
|
|
||||||
|
static void run(const Mat& src, const Mat& scale, const Mat* b, Mat& dst, int axis, float epsilon) |
||||||
|
{ |
||||||
|
CV_Assert(src.isContinuous()); |
||||||
|
CV_Assert(dst.isContinuous()); |
||||||
|
CV_CheckTypeEQ(src.type(), CV_32F, "DNN/LayerNorm: only support float32"); |
||||||
|
CV_CheckTypeEQ(src.type(), dst.type(), ""); |
||||||
|
CV_Assert(scale.isContinuous()); |
||||||
|
|
||||||
|
CV_CheckGE(epsilon, 0.0f, ""); |
||||||
|
|
||||||
|
LayerNormInvoker p(src, scale, b, dst, axis, epsilon); |
||||||
|
|
||||||
|
double nstripes = ((size_t)p.total * p.normSize) * (1 / 1024.0); |
||||||
|
// double nstripes = ((size_t)p.total) * (1 / 1024.0);
|
||||||
|
parallel_for_(Range(0, p.total), p, nstripes); |
||||||
|
} |
||||||
|
|
||||||
|
void operator()(const Range& r) const CV_OVERRIDE |
||||||
|
{ |
||||||
|
int stripeStart = r.start; |
||||||
|
int stripeEnd = r.end; |
||||||
|
|
||||||
|
const float* srcData = src.ptr<float>(); |
||||||
|
float* dstData = dst.ptr<float>(); |
||||||
|
|
||||||
|
for (int ofs = stripeStart; ofs < stripeEnd; ++ofs) |
||||||
|
{ |
||||||
|
const float* first = srcData + ofs * normSize; |
||||||
|
float* dstFirst = dstData + ofs * normSize; |
||||||
|
|
||||||
|
float mean = 0; |
||||||
|
float meanSquare = 0; |
||||||
|
for (int h = 0; h < normSize; ++h) |
||||||
|
{ |
||||||
|
float v = first[h]; |
||||||
|
mean += v; |
||||||
|
meanSquare += v * v; |
||||||
|
} |
||||||
|
mean *= invNormSize; |
||||||
|
meanSquare = std::sqrt(std::max(0.f, meanSquare * invNormSize - mean * mean) + epsilon); |
||||||
|
float invMeanSquare = 1.0f / meanSquare; |
||||||
|
for (int h = 0; h < normSize; ++h) |
||||||
|
{ |
||||||
|
float v = (first[h] - mean) * invMeanSquare * scaleData[h]; |
||||||
|
if (hasBias) { |
||||||
|
v = v + biasData[h]; |
||||||
|
} |
||||||
|
dstFirst[h] = v; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE |
||||||
|
{ |
||||||
|
CV_TRACE_FUNCTION(); |
||||||
|
CV_TRACE_ARG_VALUE(name, "name", name.c_str()); |
||||||
|
|
||||||
|
if (inputs_arr.depth() == CV_16S) |
||||||
|
{ |
||||||
|
forward_fallback(inputs_arr, outputs_arr, internals_arr); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
std::vector<Mat> inputs, outputs; |
||||||
|
inputs_arr.getMatVector(inputs); |
||||||
|
outputs_arr.getMatVector(outputs); |
||||||
|
|
||||||
|
if (hasBias) { |
||||||
|
LayerNormInvoker<true>::run(inputs[0], inputs[1], &inputs[2], outputs[0], axis, epsilon); |
||||||
|
} else { |
||||||
|
LayerNormInvoker<false>::run(inputs[0], inputs[1], nullptr, outputs[0], axis, epsilon); |
||||||
|
} |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
Ptr<LayerNormLayer> LayerNormLayer::create(const LayerParams& params) |
||||||
|
{ |
||||||
|
return makePtr<LayerNormLayerImpl>(params); |
||||||
|
} |
||||||
|
|
||||||
|
}} // cv::dnn
|
||||||
Loading…
Reference in new issue