Merge pull request #12376 from alalek:issue_12337

modules/core/src/lpsolver.cpp

@@ -98,6 +98,10 @@ int solveLP(const Mat& Func, const Mat& Constr, Mat& z){
     CV_Assert(Constr.type()==CV_64FC1 || Constr.type()==CV_32FC1);
     CV_Assert((Func.rows==1 && (Constr.cols-Func.cols==1))||
             (Func.cols==1 && (Constr.cols-Func.rows==1)));
+    if (!z.empty())
+        CV_CheckTypeEQ(z.type(), CV_64FC1, "");
+    else
+        CV_CheckType(z.type(), z.type() == CV_64FC1 || z.type() == CV_8UC1/*empty cv::Mat*/, "");
 
     //copy arguments for we will shall modify them
     Mat_<double> bigC=Mat_<double>(1,(Func.rows==1?Func.cols:Func.rows)+1),

modules/core/test/test_lpsolver.cpp

@@ -141,4 +141,14 @@ TEST(Core_LPSolver, regression_cycling){
 #endif
 }
 
+TEST(Core_LPSolver, issue_12337)
+{
+    Mat A=(cv::Mat_<double>(3,1)<<3,1,2);
+    Mat B=(cv::Mat_<double>(3,4)<<1,1,3,30,2,2,5,24,4,1,2,36);
+    EXPECT_ANY_THROW(Mat1f z_float; cv::solveLP(A, B, z_float));
+    EXPECT_NO_THROW(Mat1d z_double; cv::solveLP(A, B, z_double));
+    EXPECT_ANY_THROW(Mat1i z_int; cv::solveLP(A, B, z_int));
+    //need to update interface: EXPECT_ANY_THROW(Mat1b z_8u; cv::solveLP(A, B, z_8u));
+}
+
 }} // namespace