From a478757483c644666ad36219db0aa81550abefc8 Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Mon, 20 Nov 2023 02:19:24 -0600 Subject: [PATCH] Merge pull request #24544 from fengyuentau:layernorm_conformance dnn test: move layer norm tests into conformance tests #24544 Merge with https://github.com/opencv/opencv_extra/pull/1122 ## Motivation Some ONNX operators, such as `LayerNormalization`, `BatchNormalization` and so on, produce outputs for training (mean, stdev). So they have reference outputs of conformance tests for those training outputs as well. However, when it comes to inference, we do not need and produce those outputs for training here in dnn. Hence, output size does not match if we use dnn to infer those conformance models. This has become the barrier if we want to test these operators using their conformance tests. **I checked all ONNX operators with optional outputs. Turns out there are only `BatchNormalization`, `Dropout`, `LayerNormalization` and `MaxPool` has optional outputs for training. All except `LayerNormalization` have models set for training mode and eval mode. Blame ONNX for that.** ## Solution In this pull request, we remove graph outputs if the graph looks like the following: ``` [X] [Scale] [Bias] [X] [Scale] [Bias] \ | / this patch \ | / LayerNormalization -----------> LayerNormalization / | \ | [Y] [Mean] [Stdev] [Y] ``` We can update conformance tests and turn on some cases as well if extending to more layers. Notes: 1. This workaround does not solve expanded function operators if they are fused into a single operator, such as `$onnx/onnx/backend/test/data/node/test_layer_normalization_2d_axis1_expanded`, but they can be run without fusion. Note that either dnn or onnxruntime does not fuse those expanded function operators. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/onnx/onnx_importer.cpp | 13 +++- modules/dnn/test/test_graph_simplifier.cpp | 1 + modules/dnn/test/test_onnx_conformance.cpp | 19 +++++ ...conformance_layer_filter__openvino.inl.hpp | 76 +++++++++++++++++++ modules/dnn/test/test_onnx_importer.cpp | 30 -------- 5 files changed, 107 insertions(+), 32 deletions(-) diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 28dd4d9b77..ba4d542731 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -3187,10 +3187,19 @@ void ONNXImporter::parseLayerNorm(LayerParams& layerParams, const opencv_onnx::N // Remove additional outputs (Mean, InvStdDev) if (node_proto.output_size() > 1) { + // remove from graph proto + for (size_t i = 1; i < node_proto.output_size(); i++) { + for (int j = graph_proto.output_size() - 1; j >= 0; j--) { + if (graph_proto.output(j).name() == node_proto.output(i)) { + graph_proto.mutable_output()->DeleteSubrange(j, 1); + break; + } + } + } + // remove from node proto auto outputName = node_proto.output(0); opencv_onnx::NodeProto node_proto_ = node_proto; - node_proto_.clear_output(); - node_proto_.add_output(outputName); + node_proto_.mutable_output()->DeleteSubrange(1, node_proto_.output_size() - 1); addLayer(layerParams, node_proto_); } else diff --git a/modules/dnn/test/test_graph_simplifier.cpp b/modules/dnn/test/test_graph_simplifier.cpp index f6334f929a..58fffd3713 100644 --- a/modules/dnn/test/test_graph_simplifier.cpp +++ b/modules/dnn/test/test_graph_simplifier.cpp @@ -43,6 +43,7 @@ TEST_F(Test_Graph_Simplifier, GeluApproximationSubGraph) { TEST_F(Test_Graph_Simplifier, LayerNormSubGraph) { test("layer_norm_expanded", "LayerNormalization"); + test("layer_norm_expanded_with_initializers", "LayerNormalization"); } TEST_F(Test_Graph_Simplifier, ResizeSubgraph) { diff --git a/modules/dnn/test/test_onnx_conformance.cpp b/modules/dnn/test/test_onnx_conformance.cpp index 13c5c5b232..5b783722c4 100644 --- a/modules/dnn/test/test_onnx_conformance.cpp +++ b/modules/dnn/test/test_onnx_conformance.cpp @@ -339,6 +339,25 @@ static const TestCase testConformanceConfig[] = { {"test_isinf_negative", 1, 1}, {"test_isinf_positive", 1, 1}, {"test_isnan", 1, 1}, + {"test_layer_normalization_2d_axis0", 3, 1}, + {"test_layer_normalization_2d_axis1", 3, 1}, + {"test_layer_normalization_2d_axis_negative_1", 3, 1}, + {"test_layer_normalization_2d_axis_negative_2", 3, 1}, + {"test_layer_normalization_3d_axis0_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis1_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis2_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis_negative_1_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis_negative_2_epsilon", 3, 1}, + {"test_layer_normalization_3d_axis_negative_3_epsilon", 3, 1}, + {"test_layer_normalization_4d_axis0", 3, 1}, + {"test_layer_normalization_4d_axis1", 3, 1}, + {"test_layer_normalization_4d_axis2", 3, 1}, + {"test_layer_normalization_4d_axis3", 3, 1}, + {"test_layer_normalization_4d_axis_negative_1", 3, 1}, + {"test_layer_normalization_4d_axis_negative_2", 3, 1}, + {"test_layer_normalization_4d_axis_negative_3", 3, 1}, + {"test_layer_normalization_4d_axis_negative_4", 3, 1}, + {"test_layer_normalization_default_axis", 3, 1}, {"test_leakyrelu", 1, 1}, {"test_leakyrelu_default", 1, 1}, {"test_leakyrelu_example", 1, 1}, diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 339746f5f2..94687a1e26 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -792,6 +792,82 @@ CASE(test_isinf_positive) // no filter CASE(test_isnan) // no filter +CASE(test_layer_normalization_2d_axis0) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_2d_axis1) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_2d_axis_negative_1) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_2d_axis_negative_2) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis0_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis1_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis2_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis_negative_1_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis_negative_2_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_3d_axis_negative_3_epsilon) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis0) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis1) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis2) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis3) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis_negative_1) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis_negative_2) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis_negative_3) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_4d_axis_negative_4) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif +CASE(test_layer_normalization_default_axis) +#if SKIP_SET_1 + SKIP_NON_CPU; +#endif CASE(test_leakyrelu) // no filter CASE(test_leakyrelu_default) diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 7be3e8a385..aa60cea890 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -2674,36 +2674,6 @@ TEST_P(Test_ONNX_layers, Tile) testONNXModels("tile", pb); } -TEST_P(Test_ONNX_layers, LayerNorm) -{ - testONNXModels("test_layer_normalization_2d_axis0", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_2d_axis1", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_2d_axis_negative_1", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_2d_axis_negative_2", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis0_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis1_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis2_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis_negative_1_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis_negative_2_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_3d_axis_negative_3_epsilon", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis0", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis1", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis2", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis3", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis_negative_1", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis_negative_2", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis_negative_3", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_4d_axis_negative_4", pb, 0, 0, false, true, 3); - testONNXModels("test_layer_normalization_default_axis", pb, 0, 0, false, true, 3); -} - -// for testing graph simplification -TEST_P(Test_ONNX_layers, LayerNormExpanded) -{ - testONNXModels("layer_norm_expanded"); - testONNXModels("layer_norm_expanded_with_initializers"); -} - TEST_P(Test_ONNX_layers, Gelu) { testONNXModels("gelu");