Merge pull request #24445 from Abdurrahheem:ash/dev_einsum_pref

Einsum Layer Performance Test #24445 ## This PR adds performance tests for Einsum Layer. See below results of performance test on different inputs **Notation:** - WX: windows10_x64 - MX: macos_x64 - MA: macos_arm64 - UX: ubuntu_x64 - UA: ubuntu_arm64 All data in ms (milliseconds). Gemm is backend for matrix multiplication --- Benchmarks: | Equation | Inputs Mat Dims | UX (ms) | UA (ms) | MX (ms) | MA (ms) | WX (ms) | |-------------------------|-----------------------------------|----------------|---------|---------|---------|---------| | "ij, jk -> ik" | [2, 3], [3,2] | 0.04 ± 0.00 | - | - | - | - | | "ij, jk -> ik" | [20, 30], [30,20] | 0.08 ± 0.00 | - | - | - | - | | "ij, jk -> ik" | [113, 127], [127,113] | 2.41 ± 0.05 | - | - | - | - | | "imkj, injs -> imnks" | [1, 4, 7, 9], [1, 5, 9, 8] | 0.11 ± 0.00 | - | - | - | - | | "imkj, injs -> imnks" | [1, 4, 70, 90], [1, 5, 90, 80] | 15.49 ± 0.46 | - | - | - | - | | "imkj, injs -> imnks" | [1, 4, 73, 91], [1, 5, 91, 57] | 11.53 ± 0.06 | - | - | - | - | | "ij -> i" | [30, 40] | 0.03 ± 0.00 | - | - | - | - | | "ij -> i" | [113, 374] | 0.13 ± 0.00 | - | - | - | - | | "...ij -> ...i" | [30, 40] | 0.03 ± 0.00 | - | - | - | - | | "...ij -> ...i" | [113, 374] | 0.13 ± 0.00 | - | - | - | - | | "...ij, ...jk -> ...ik" | [40, 50], [50,80] | 0.37 ± 0.01 | - | - | - | - | | "...ij, ...jk -> ...ik" | [47, 51], [51, 83] | 0.43 ± 0.01 | - | - | - | - | ----- ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
1 year ago · 9d0c8a9edb
parent 81907af74c
commit 9d0c8a9edb
1 changed files with 123 additions and 0 deletions
--- a/modules/dnn/perf/perf_einsum.cpp
+++ b/modules/dnn/perf/perf_einsum.cpp
@ -0,0 +1,123 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "perf_precomp.hpp"
+
+namespace opencv_test {
+
+struct EinsumParams {
+    int inputSize;
+    int outputSize;
+    std::string equation;
+    std::vector<MatShape> einsumInpShapes;
+    EinsumParams(std::string equation_, int inputSize_, int outputSize_,  std::vector<MatShape> einsumInpShapes_ = std::vector<MatShape>())
+    {
+        inputSize = inputSize_;
+        outputSize = outputSize_;
+        equation = equation_;
+        einsumInpShapes = einsumInpShapes_;
+    }
+};
+
+static inline void PrintTo(const EinsumParams& params, ::std::ostream* os) {
+     (*os) << "Eqiation=" << params.equation << ", "
+        << "InputSize=" << params.inputSize << ", "
+        << "OutputSize=" << params.outputSize << ", ";
+
+        (*os) << "InputShape={";
+        for(int i = 0; i < params.einsumInpShapes.size(); i++)
+        {
+            (*os) << "{";
+            for(int j = 0; j < params.einsumInpShapes[i].size(); j++)
+            {
+                (*os) << params.einsumInpShapes[i][j] << ((j < params.einsumInpShapes[i].size() - 1) ?  ", " : "");
+            }
+            (*os) << ((i < params.einsumInpShapes.size() - 1) ? "}, " : "}");
+        }
+        (*os) << "}";
+}
+
+// test cases
+static const EinsumParams testEinsumConfigs[] = {
+    // TODO: Add tests with one input after ellips merge
+    {"ij, jk -> ik", 2, 1,  {{2, 3}, {3, 2}}},
+    {"ij, jk -> ik", 2, 1,  {{20, 30}, {30, 20}}},
+    {"ij, jk -> ik", 2, 1,  {{113, 127}, {127, 113}}},
+
+    {"imkj, injs -> imnks", 2, 1,  {{1, 4, 7, 9}, {1, 5, 9, 8}}},
+    {"imkj, injs -> imnks", 2, 1,  {{1, 4, 70, 90}, {1, 5, 90, 80}}},
+    {"imkj, injs -> imnks", 2, 1,  {{1, 4, 73, 91}, {1, 5, 91, 57}}},
+
+    {"ij -> i", 1, 1, {{30, 40}}},
+    {"ij -> i", 1, 1, {{113, 374}}},
+
+    {"...ij -> ...i", 1, 1, {{30, 40}}},
+    {"...ij -> ...i", 1, 1, {{113, 374}}},
+
+    {"...ij, ...jk -> ...ik", 2, 1, {{40, 50}, {50, 80}}},
+    {"...ij, ...jk -> ...ik", 2, 1, {{47, 51}, {51, 83}}},
+};
+
+class Layer_Einsum: public TestBaseWithParam<EinsumParams> {};
+
+PERF_TEST_P_(Layer_Einsum, einsum) {
+    const EinsumParams& params = GetParam();
+    LayerParams lp;
+    lp.type = "Einsum";
+    lp.name = "testEinsum";
+    lp.set("equation", params.equation);
+    lp.set("inputSize", params.inputSize);
+    lp.set("outputSize", params.outputSize);
+
+    CV_CheckFalse(params.einsumInpShapes.empty(), "ERROR no inputs shapes provided");
+
+    for (int i = 0; i < params.einsumInpShapes.size(); i++) {
+        lp.set("inputShapes" + cv::format("%d", i), DictValue::arrayInt(params.einsumInpShapes[i].begin(), params.einsumInpShapes[i].size()));
+    }
+
+    Net net;
+    std::vector<Mat> inputs;
+    std::vector<std::string> input_names;
+    if (params.inputSize == 1){
+
+        // create inputs
+        inputs.emplace_back(Mat(params.einsumInpShapes[0].size(), params.einsumInpShapes[0].data(), CV_32FC1));
+
+        int id = net.addLayerToPrev(lp.name, lp.type, lp);
+        net.connect(0, 0, id, 0);
+
+        input_names.emplace_back("input1");
+
+    } else {
+
+        // create inputs
+        inputs.emplace_back(Mat(params.einsumInpShapes[0].size(), params.einsumInpShapes[0].data(), CV_32FC1));
+        inputs.emplace_back(Mat(params.einsumInpShapes[1].size(), params.einsumInpShapes[1].data(), CV_32FC1));
+
+        int id = net.addLayerToPrev(lp.name, lp.type, lp);
+        net.connect(0, 0, id, 0);
+        net.connect(0, 1, id, 1);
+
+        input_names.emplace_back("input1");
+        input_names.emplace_back("input2");
+    }
+
+    //warm up
+    net.setInputsNames(input_names);
+    for (int i = 0; i < input_names.size(); i++){
+        net.setInput(inputs[i], input_names[i]);
+    }
+    Mat out = net.forward();
+
+    std::vector<Mat> outputs;
+    TEST_CYCLE()
+    {
+        net.forward(outputs, "testEinsum");
+    }
+    SANITY_CHECK_NOTHING();
+}
+
+INSTANTIATE_TEST_CASE_P(/**/, Layer_Einsum, testing::ValuesIn(testEinsumConfigs));
+
+}; //namespace