From 7465125f67fa8c535477193e0730867a7a88136a Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 14 Mar 2020 23:17:25 +0000 Subject: [PATCH] build: add winpack_dldt build scripts --- cmake/OpenCVModule.cmake | 25 +- platforms/winpack_dldt/.gitattributes | 1 + ...20200313-dldt-disable-unused-targets.patch | 158 ++++++ .../20200313-dldt-fix-binaries-location.patch | 13 + ...200313-ngraph-disable-tests-examples.patch | 25 + .../2020.1/20200318-dldt-pdb.patch | 14 + .../20200319-dldt-fix-msvs2019-v16.5.0.patch | 13 + platforms/winpack_dldt/2020.1/patch.config.py | 5 + .../winpack_dldt/2020.1/sysroot.config.py | 51 ++ platforms/winpack_dldt/build_package.py | 503 ++++++++++++++++++ .../cmake-opencv-checks/POST_FINALIZE.cmake | 17 + .../package-tests/test_dnn_backends.py | 13 + 12 files changed, 836 insertions(+), 2 deletions(-) create mode 100644 platforms/winpack_dldt/.gitattributes create mode 100644 platforms/winpack_dldt/2020.1/20200313-dldt-disable-unused-targets.patch create mode 100644 platforms/winpack_dldt/2020.1/20200313-dldt-fix-binaries-location.patch create mode 100644 platforms/winpack_dldt/2020.1/20200313-ngraph-disable-tests-examples.patch create mode 100644 platforms/winpack_dldt/2020.1/20200318-dldt-pdb.patch create mode 100644 platforms/winpack_dldt/2020.1/20200319-dldt-fix-msvs2019-v16.5.0.patch create mode 100644 platforms/winpack_dldt/2020.1/patch.config.py create mode 100644 platforms/winpack_dldt/2020.1/sysroot.config.py create mode 100644 platforms/winpack_dldt/build_package.py create mode 100644 platforms/winpack_dldt/cmake-opencv-checks/POST_FINALIZE.cmake create mode 100644 platforms/winpack_dldt/package-tests/test_dnn_backends.py diff --git a/cmake/OpenCVModule.cmake b/cmake/OpenCVModule.cmake index f014ee7f5b..634b5b264c 100644 --- a/cmake/OpenCVModule.cmake +++ b/cmake/OpenCVModule.cmake @@ -1083,6 +1083,17 @@ macro(ocv_check_dependencies) endforeach() endmacro() +################################################################################ +# OpenCV tests +################################################################################ + +if(DEFINED OPENCV_BUILD_TEST_MODULES_LIST) + string(REPLACE "," ";" OPENCV_BUILD_TEST_MODULES_LIST "${OPENCV_BUILD_TEST_MODULES_LIST}") # support comma-separated list (,) too +endif() +if(DEFINED OPENCV_BUILD_PERF_TEST_MODULES_LIST) + string(REPLACE "," ";" OPENCV_BUILD_PERF_TEST_MODULES_LIST "${OPENCV_BUILD_PERF_TEST_MODULES_LIST}") # support comma-separated list (,) too +endif() + # auxiliary macro to parse arguments of ocv_add_accuracy_tests and ocv_add_perf_tests commands macro(__ocv_parse_test_sources tests_type) set(OPENCV_${tests_type}_${the_module}_SOURCES "") @@ -1123,7 +1134,12 @@ function(ocv_add_perf_tests) endif() set(perf_path "${CMAKE_CURRENT_LIST_DIR}/perf") - if(BUILD_PERF_TESTS AND EXISTS "${perf_path}") + if(BUILD_PERF_TESTS AND EXISTS "${perf_path}" + AND (NOT DEFINED OPENCV_BUILD_PERF_TEST_MODULES_LIST + OR OPENCV_BUILD_PERF_TEST_MODULES_LIST STREQUAL "all" + OR ";${OPENCV_BUILD_PERF_TEST_MODULES_LIST};" MATCHES ";${name};" + ) + ) __ocv_parse_test_sources(PERF ${ARGN}) # opencv_imgcodecs is required for imread/imwrite @@ -1199,7 +1215,12 @@ function(ocv_add_accuracy_tests) ocv_debug_message("ocv_add_accuracy_tests(" ${ARGN} ")") set(test_path "${CMAKE_CURRENT_LIST_DIR}/test") - if(BUILD_TESTS AND EXISTS "${test_path}") + if(BUILD_TESTS AND EXISTS "${test_path}" + AND (NOT DEFINED OPENCV_BUILD_TEST_MODULES_LIST + OR OPENCV_BUILD_TEST_MODULES_LIST STREQUAL "all" + OR ";${OPENCV_BUILD_TEST_MODULES_LIST};" MATCHES ";${name};" + ) + ) __ocv_parse_test_sources(TEST ${ARGN}) # opencv_imgcodecs is required for imread/imwrite diff --git a/platforms/winpack_dldt/.gitattributes b/platforms/winpack_dldt/.gitattributes new file mode 100644 index 0000000000..767b7126cb --- /dev/null +++ b/platforms/winpack_dldt/.gitattributes @@ -0,0 +1 @@ +*.patch text eol=crlf -whitespace diff --git a/platforms/winpack_dldt/2020.1/20200313-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2020.1/20200313-dldt-disable-unused-targets.patch new file mode 100644 index 0000000000..2dcfd4e975 --- /dev/null +++ b/platforms/winpack_dldt/2020.1/20200313-dldt-disable-unused-targets.patch @@ -0,0 +1,158 @@ +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index d5feedb..1b7aa7e 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -59,11 +59,11 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-add_subdirectory(samples) ++#add_subdirectory(samples) + + file(GLOB_RECURSE SAMPLES_SOURCES samples/*.cpp samples/*.hpp samples/*.h) + add_cpplint_target(sample_cpplint +@@ -134,7 +134,7 @@ install(DIRECTORY ${ie_python_api_SOURCE_DIR}/sample/ + add_custom_target(ie_dev_targets ALL DEPENDS inference_engine HeteroPlugin) + + # Developer package +-ie_developer_export_targets(format_reader) ++#ie_developer_export_targets(format_reader) + + if (ENABLE_NGRAPH) + ie_developer_export_targets(${NGRAPH_LIBRARIES}) +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index 54e264c..c0b7495 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -228,7 +228,7 @@ target_include_directories(${TARGET_NAME}_nn_builder PRIVATE "${CMAKE_CURRENT_SO + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + ${NN_BUILDER_LIBRARY_SRC}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index cd727fd..2f09b44 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -184,9 +184,9 @@ endif() + add_library(mkldnn_plugin_layers_no_opt OBJECT ${CROSS_COMPILED_SOURCES}) + set_ie_threading_interface_for(mkldnn_plugin_layers_no_opt) + +-add_library(mkldnn_plugin_layers_no_opt_s OBJECT ${CROSS_COMPILED_SOURCES}) +-set_ie_threading_interface_for(mkldnn_plugin_layers_no_opt_s) +-target_compile_definitions(mkldnn_plugin_layers_no_opt_s PRIVATE USE_STATIC_IE) ++#add_library(mkldnn_plugin_layers_no_opt_s OBJECT ${CROSS_COMPILED_SOURCES}) ++#set_ie_threading_interface_for(mkldnn_plugin_layers_no_opt_s) ++#target_compile_definitions(mkldnn_plugin_layers_no_opt_s PRIVATE USE_STATIC_IE) + + set(object_libraries mkldnn_plugin_layers_no_opt) + set(mkldnn_plugin_object_libraries mkldnn_plugin_layers_no_opt_s) +@@ -220,7 +220,7 @@ if (ENABLE_SSE42) + endfunction() + + mkldnn_create_sse42_layers(mkldnn_plugin_layers_sse42) +- mkldnn_create_sse42_layers(mkldnn_plugin_layers_sse42_s) ++ #mkldnn_create_sse42_layers(mkldnn_plugin_layers_sse42_s) + + list(APPEND object_libraries mkldnn_plugin_layers_sse42) + list(APPEND mkldnn_plugin_object_libraries mkldnn_plugin_layers_sse42_s) +@@ -259,7 +259,7 @@ if (ENABLE_AVX2) + endfunction() + + mkldnn_create_avx2_layers(mkldnn_plugin_layers_avx2) +- mkldnn_create_avx2_layers(mkldnn_plugin_layers_avx2_s) ++ #mkldnn_create_avx2_layers(mkldnn_plugin_layers_avx2_s) + + list(APPEND object_libraries mkldnn_plugin_layers_avx2) + list(APPEND mkldnn_plugin_object_libraries mkldnn_plugin_layers_avx2_s) +@@ -297,7 +297,7 @@ if (ENABLE_AVX512F) + endfunction() + + mkldnn_create_avx512f_layers(mkldnn_plugin_layers_avx512) +- mkldnn_create_avx512f_layers(mkldnn_plugin_layers_avx512_s) ++ #mkldnn_create_avx512f_layers(mkldnn_plugin_layers_avx512_s) + + list(APPEND object_libraries mkldnn_plugin_layers_avx512) + list(APPEND mkldnn_plugin_object_libraries mkldnn_plugin_layers_avx512_s) +@@ -317,7 +317,7 @@ target_link_libraries(${TARGET_NAME} PRIVATE inference_engine ${INTEL_ITT_LIBS} + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $) + +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index 41f14a9..0e1b4f6 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -81,7 +81,7 @@ endif() + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 8995390..8413faf 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -49,7 +49,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index e77296e..333f560 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -60,7 +60,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/CMakeLists.txt b/inference-engine/thirdparty/CMakeLists.txt +index ec22761..8bb3325 100644 +--- a/inference-engine/thirdparty/CMakeLists.txt ++++ b/inference-engine/thirdparty/CMakeLists.txt +@@ -36,7 +36,7 @@ function(build_with_lto) + endif() + + add_subdirectory(pugixml) +- add_subdirectory(stb_lib) ++ #add_subdirectory(stb_lib) + add_subdirectory(ade) + add_subdirectory(fluid/modules/gapi) + +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb280..5a17fa3 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") diff --git a/platforms/winpack_dldt/2020.1/20200313-dldt-fix-binaries-location.patch b/platforms/winpack_dldt/2020.1/20200313-dldt-fix-binaries-location.patch new file mode 100644 index 0000000000..82da0d8cec --- /dev/null +++ b/platforms/winpack_dldt/2020.1/20200313-dldt-fix-binaries-location.patch @@ -0,0 +1,13 @@ +diff --git a/cmake/developer_package.cmake b/cmake/developer_package.cmake +index e59edb2..e42ac19 100644 +--- a/cmake/developer_package.cmake ++++ b/cmake/developer_package.cmake +@@ -99,7 +99,7 @@ if(UNIX) + SET(LIB_DL ${CMAKE_DL_LIBS}) + endif() + +-set(OUTPUT_ROOT ${OpenVINO_MAIN_SOURCE_DIR}) ++set(OUTPUT_ROOT ${CMAKE_BINARY_DIR}) + + # Enable postfixes for Debug/Release builds + set(IE_DEBUG_POSTFIX_WIN "d") diff --git a/platforms/winpack_dldt/2020.1/20200313-ngraph-disable-tests-examples.patch b/platforms/winpack_dldt/2020.1/20200313-ngraph-disable-tests-examples.patch new file mode 100644 index 0000000000..bfdd78ff41 --- /dev/null +++ b/platforms/winpack_dldt/2020.1/20200313-ngraph-disable-tests-examples.patch @@ -0,0 +1,25 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 631465f..723153b 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -567,7 +567,7 @@ if (NGRAPH_ONNX_IMPORT_ENABLE) + endif() + endif() + +-include(cmake/external_gtest.cmake) ++#include(cmake/external_gtest.cmake) + if(NGRAPH_JSON_ENABLE) + include(cmake/external_json.cmake) + endif() +@@ -623,8 +623,8 @@ endif() + + add_subdirectory(src) + +-add_subdirectory(test) +-add_subdirectory(doc/examples) ++#add_subdirectory(test) ++#add_subdirectory(doc/examples) + + if (NGRAPH_DOC_BUILD_ENABLE) + add_subdirectory(doc) + diff --git a/platforms/winpack_dldt/2020.1/20200318-dldt-pdb.patch b/platforms/winpack_dldt/2020.1/20200318-dldt-pdb.patch new file mode 100644 index 0000000000..d94a84abe9 --- /dev/null +++ b/platforms/winpack_dldt/2020.1/20200318-dldt-pdb.patch @@ -0,0 +1,14 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e48cee5..5823e92 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -12,6 +12,9 @@ endif() + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(CMAKE_MODULE_PATH "${OpenVINO_MAIN_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) + diff --git a/platforms/winpack_dldt/2020.1/20200319-dldt-fix-msvs2019-v16.5.0.patch b/platforms/winpack_dldt/2020.1/20200319-dldt-fix-msvs2019-v16.5.0.patch new file mode 100644 index 0000000000..7f864fbd45 --- /dev/null +++ b/platforms/winpack_dldt/2020.1/20200319-dldt-fix-msvs2019-v16.5.0.patch @@ -0,0 +1,13 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 7900b382..b5c53d09 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -24,6 +24,8 @@ include(features) + # include developer package + include(developer_package) + ++disable_deprecated_warnings() ++ + # These options are shared with 3rdparty plugins + # by means of developer package + include(check_features) diff --git a/platforms/winpack_dldt/2020.1/patch.config.py b/platforms/winpack_dldt/2020.1/patch.config.py new file mode 100644 index 0000000000..ded4f3f325 --- /dev/null +++ b/platforms/winpack_dldt/2020.1/patch.config.py @@ -0,0 +1,5 @@ +applyPatch('20200313-ngraph-disable-tests-examples.patch', 'ngraph') +applyPatch('20200313-dldt-disable-unused-targets.patch') +applyPatch('20200313-dldt-fix-binaries-location.patch') +applyPatch('20200318-dldt-pdb.patch') +applyPatch('20200319-dldt-fix-msvs2019-v16.5.0.patch') diff --git a/platforms/winpack_dldt/2020.1/sysroot.config.py b/platforms/winpack_dldt/2020.1/sysroot.config.py new file mode 100644 index 0000000000..f99c3425fe --- /dev/null +++ b/platforms/winpack_dldt/2020.1/sysroot.config.py @@ -0,0 +1,51 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cldnn_global_custom_kernels') +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_nn_builder') +copy_dll('MKLDNNPlugin') +copy_dll('myriadPlugin') +copy_dll('ngraph') +copy_bin('plugins.xml') +copytree(self.build_dir / 'bin' / 'intel64' / 'pcie-ma248x.elf', sysroot_bin_dir / 'pcie-ma248x.elf') +copytree(self.build_dir / 'bin' / 'intel64' / 'usb-ma2x8x.mvcmd', sysroot_bin_dir / 'usb-ma2x8x.mvcmd') +copytree(self.build_dir / 'bin' / 'intel64' / 'usb-ma2450.mvcmd', sysroot_bin_dir / 'usb-ma2450.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(self.build_dir / 'install' / 'lib' / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_nn_builder.lib', sysroot_ie_lib_dir / 'inference_engine_nn_builder.lib') +else: + copytree(self.build_dir / 'install' / 'lib' / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_nn_builderd.lib', sysroot_ie_lib_dir / 'inference_engine_nn_builderd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.srcdir / 'ngraph/LICENSE', sysroot_license_dir / 'ngraph-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py new file mode 100644 index 0000000000..d182f9f3b7 --- /dev/null +++ b/platforms/winpack_dldt/build_package.py @@ -0,0 +1,503 @@ +#!/usr/bin/env python + +import os, sys +import argparse +import glob +import re +import shutil +import subprocess +import time + +import logging as log + +if sys.version_info[0] == 2: + sys.exit("FATAL: Python 2.x is not supported") + +from pathlib import Path + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +class Fail(Exception): + def __init__(self, text=None): + self.t = text + def __str__(self): + return "ERROR" if self.t is None else self.t + +def execute(cmd, cwd=None, shell=False): + try: + log.debug("Executing: %s" % cmd) + log.info('Executing: ' + ' '.join(cmd)) + if cwd: + log.info(" in: %s" % cwd) + retcode = subprocess.call(cmd, shell=shell, cwd=str(cwd) if cwd else None) + if retcode < 0: + raise Fail("Child was terminated by signal: %s" % -retcode) + elif retcode > 0: + raise Fail("Child returned: %s" % retcode) + except OSError as e: + raise Fail("Execution failed: %d / %s" % (e.errno, e.strerror)) + +def check_executable(cmd): + try: + log.debug("Executing: %s" % cmd) + result = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + if not isinstance(result, str): + result = result.decode("utf-8") + log.debug("Result: %s" % (result + '\n').split('\n')[0]) + return True + except OSError as e: + log.debug('Failed: %s' % e) + return False + + +def rm_one(d): + d = str(d) # Python 3.5 may not handle Path + d = os.path.abspath(d) + if os.path.exists(d): + if os.path.isdir(d): + log.info("Removing dir: %s", d) + shutil.rmtree(d) + elif os.path.isfile(d): + log.info("Removing file: %s", d) + os.remove(d) + + +def prepare_dir(d, clean=False): + d = str(d) # Python 3.5 may not handle Path + d = os.path.abspath(d) + log.info("Preparing directory: '%s' (clean: %r)", d, clean) + if os.path.exists(d): + if not os.path.isdir(d): + raise Fail("Not a directory: %s" % d) + if clean: + for item in os.listdir(d): + rm_one(os.path.join(d, item)) + else: + os.makedirs(d) + return Path(d) + + +def check_dir(d): + d = str(d) # Python 3.5 may not handle Path + d = os.path.abspath(d) + log.info("Check directory: '%s'", d) + if os.path.exists(d): + if not os.path.isdir(d): + raise Fail("Not a directory: %s" % d) + else: + raise Fail("The directory is missing: %s" % d) + return Path(d) + + +# shutil.copytree fails if dst exists +def copytree(src, dst, exclude=None): + log.debug('copytree(%s, %s)', src, dst) + src = str(src) # Python 3.5 may not handle Path + dst = str(dst) # Python 3.5 may not handle Path + if os.path.isfile(src): + shutil.copy2(src, dst) + return + def copy_recurse(subdir): + if exclude and subdir in exclude: + log.debug(' skip: %s', subdir) + return + s = os.path.join(src, subdir) + d = os.path.join(dst, subdir) + if os.path.exists(d) or exclude: + if os.path.isfile(s): + shutil.copy2(s, d) + elif os.path.isdir(s): + if not os.path.isdir(d): + os.makedirs(d) + for item in os.listdir(s): + copy_recurse(os.path.join(subdir, item)) + else: + assert False, s + " => " + d + else: + if os.path.isfile(s): + shutil.copy2(s, d) + elif os.path.isdir(s): + shutil.copytree(s, d) + else: + assert False, s + " => " + d + copy_recurse('') + + +def git_checkout(dst, url, branch, revision, clone_extra_args, noFetch=False): + assert isinstance(dst, Path) + log.info("Git checkout: '%s' (%s @ %s)", dst, url, revision) + if noFetch: + pass + elif not os.path.exists(str(dst / '.git')): + execute(cmd=['git', 'clone'] + + (['-b', branch] if branch else []) + + clone_extra_args + [url, '.'], cwd=dst) + else: + execute(cmd=['git', 'fetch', 'origin'] + ([branch] if branch else []), cwd=dst) + execute(cmd=['git', 'reset', '--hard'], cwd=dst) + execute(cmd=['git', 'checkout', '-B', 'winpack_dldt', revision], cwd=dst) + execute(cmd=['git', 'clean', '-f', '-d'], cwd=dst) + execute(cmd=['git', 'submodule', 'init'], cwd=dst) + execute(cmd=['git', 'submodule', 'update', '--force', '--depth=1000'], cwd=dst) + log.info("Git checkout: DONE") + execute(cmd=['git', 'status'], cwd=dst) + execute(cmd=['git', 'log', '--max-count=1', 'HEAD'], cwd=dst) + + +def git_apply_patch(src_dir, patch_file): + src_dir = str(src_dir) # Python 3.5 may not handle Path + patch_file = str(patch_file) # Python 3.5 may not handle Path + assert os.path.exists(patch_file), patch_file + execute(cmd=['git', 'apply', '--3way', '-v', '--ignore-space-change', str(patch_file)], cwd=src_dir) + + +#=================================================================================================== + +class BuilderDLDT: + def __init__(self, config): + self.config = config + + cpath = self.config.dldt_config + log.info('DLDT build configration: %s', cpath) + if not os.path.exists(cpath): + cpath = os.path.join(SCRIPT_DIR, cpath) + if not os.path.exists(cpath): + raise Fail('Config "%s" is missing' % cpath) + self.cpath = Path(cpath) + + clean_src_dir = self.config.clean_dldt + if self.config.dldt_src_dir: + assert os.path.exists(self.config.dldt_src_dir), self.config.dldt_src_dir + dldt_dir_name = 'dldt-custom' + self.srcdir = self.config.dldt_src_dir + clean_src_dir = False + else: + assert not self.config.dldt_src_dir + self.init_patchset() + dldt_dir_name = 'dldt-' + self.config.dldt_src_commit + \ + ('/patch-' + self.patch_hashsum if self.patch_hashsum else '') + if self.config.build_debug: + dldt_dir_name += '-debug' + self.srcdir = None # updated below + log.info('DLDT directory: %s', dldt_dir_name) + self.outdir = prepare_dir(os.path.join(self.config.build_cache_dir, dldt_dir_name)) + if self.srcdir is None: + self.srcdir = prepare_dir(self.outdir / 'sources', clean=clean_src_dir) + self.build_dir = prepare_dir(self.outdir / 'build', clean=self.config.clean_dldt) + self.sysrootdir = prepare_dir(self.outdir / 'sysroot', clean=self.config.clean_dldt) + + def init_patchset(self): + cpath = self.cpath + self.patch_file = str(cpath / 'patch.config.py') # Python 3.5 may not handle Path + with open(self.patch_file, 'r') as f: + self.patch_file_contents = f.read() + + patch_hashsum = None + try: + import hashlib + patch_hashsum = hashlib.md5(self.patch_file_contents.encode('utf-8')).hexdigest() + except: + log.warn("Can't compute hashsum of patches: %s", self.patch_file) + self.patch_hashsum = patch_hashsum + + + def prepare_sources(self): + if self.config.dldt_src_dir: + log.info('Using DLDT custom repository: %s', self.srcdir) + return + + def do_clone(srcdir, noFetch): + git_checkout(srcdir, self.config.dldt_src_url, self.config.dldt_src_branch, self.config.dldt_src_commit, + ['-n', '--depth=100', '--recurse-submodules'] + + (self.config.dldt_src_git_clone_extra or []), + noFetch=noFetch + ) + + if not os.path.exists(str(self.srcdir / '.git')): + log.info('DLDT git checkout through "reference" copy.') + reference_dir = self.config.dldt_reference_dir + if reference_dir is None: + reference_dir = prepare_dir(os.path.join(self.config.build_cache_dir, 'dldt-git-reference-repository')) + do_clone(reference_dir, False) + log.info('DLDT reference git checkout completed. Copying...') + else: + log.info('Using DLDT reference repository. Copying...') + copytree(reference_dir, self.srcdir) + do_clone(self.srcdir, True) + else: + do_clone(self.srcdir, False) + + log.info('DLDT git checkout completed. Patching...') + + def applyPatch(patch_file, subdir = None): + if subdir: + log.info('Patching "%s": %s' % (subdir, patch_file)) + else: + log.info('Patching: %s' % (patch_file)) + git_apply_patch(self.srcdir / subdir if subdir else self.srcdir, self.cpath / patch_file) + + exec(compile(self.patch_file_contents, self.patch_file, 'exec')) + + log.info('DLDT patches applied') + + + def build(self): + self.cmake_path = 'cmake' + build_config = 'Release' if not self.config.build_debug else 'Debug' + + cmd = [self.cmake_path, '-G', 'Visual Studio 16 2019', '-A', 'x64'] + + cmake_vars = dict( + CMAKE_BUILD_TYPE=build_config, + ENABLE_SAMPLES='OFF', + ENABLE_TESTS='OFF', + BUILD_TESTS='OFF', + ENABLE_OPENCV='OFF', + ENABLE_GNA='OFF', + NGRAPH_UNIT_TEST_ENABLE='OFF', + CMAKE_INSTALL_PREFIX=str(self.build_dir / 'install'), + ) + + cmd += [ '-D%s=%s' % (k, v) for (k, v) in cmake_vars.items() if v is not None] + if self.config.cmake_option_dldt: + cmd += self.config.cmake_option_dldt + + cmd.append(str(self.srcdir)) + execute(cmd, cwd=self.build_dir) + + # build + cmd = [self.cmake_path, '--build', '.', '--config', build_config, # '--target', 'install', + '--', '/v:n', '/m:2', '/consoleloggerparameters:NoSummary' + ] + execute(cmd, cwd=self.build_dir) + + # install ngraph only + cmd = [self.cmake_path, '-DBUILD_TYPE=' + build_config, '-P', 'cmake_install.cmake'] + execute(cmd, cwd=self.build_dir / 'ngraph') + + log.info('DLDT build completed') + + + def make_sysroot(self): + cfg_file = str(self.cpath / 'sysroot.config.py') # Python 3.5 may not handle Path + with open(cfg_file, 'r') as f: + cfg = f.read() + exec(compile(cfg, cfg_file, 'exec')) + + log.info('DLDT sysroot preparation completed') + + +#=================================================================================================== + +class Builder: + def __init__(self, config): + self.config = config + build_dir_name = 'opencv_build' if not self.config.build_debug else 'opencv_build_debug' + self.build_dir = prepare_dir(Path(self.config.output_dir) / build_dir_name, clean=self.config.clean_opencv) + self.package_dir = prepare_dir(Path(self.config.output_dir) / 'package/opencv', clean=True) + self.install_dir = prepare_dir(self.package_dir / 'build') + self.src_dir = check_dir(self.config.opencv_dir) + + + def build(self, builderDLDT): + self.cmake_path = 'cmake' + build_config = 'Release' if not self.config.build_debug else 'Debug' + + cmd = [self.cmake_path, '-G', 'Visual Studio 16 2019', '-A', 'x64'] + + cmake_vars = dict( + CMAKE_BUILD_TYPE=build_config, + INSTALL_CREATE_DISTRIB='ON', + BUILD_opencv_world='OFF', + BUILD_TESTS='OFF', + BUILD_PERF_TESTS='OFF', + ENABLE_CXX11='ON', + WITH_INF_ENGINE='ON', + INF_ENGINE_RELEASE=str(self.config.dldt_release), + WITH_TBB='ON', + CPU_BASELINE='AVX2', + CMAKE_INSTALL_PREFIX=str(self.install_dir), + INSTALL_PDB='ON', + INSTALL_PDB_COMPONENT_EXCLUDE_FROM_ALL='OFF', + + OPENCV_SKIP_CMAKE_ROOT_CONFIG='ON', + OPENCV_BIN_INSTALL_PATH='bin', + OPENCV_INCLUDE_INSTALL_PATH='include', + OPENCV_LIB_INSTALL_PATH='lib', + OPENCV_CONFIG_INSTALL_PATH='cmake', + OPENCV_3P_LIB_INSTALL_PATH='3rdparty', + OPENCV_SAMPLES_SRC_INSTALL_PATH='samples', + OPENCV_DOC_INSTALL_PATH='doc', + OPENCV_OTHER_INSTALL_PATH='etc', + OPENCV_LICENSES_INSTALL_PATH='etc/licenses', + + OPENCV_INSTALL_DATA_DIR_RELATIVE='../../src/opencv', + + BUILD_opencv_python2='OFF', + BUILD_opencv_python3='ON', + PYTHON3_LIMITED_API='ON', + OPENCV_PYTHON_INSTALL_PATH='python', + ) + + cmake_vars['INF_ENGINE_LIB_DIRS:PATH'] = str(builderDLDT.sysrootdir / 'deployment_tools/inference_engine/lib/intel64') + cmake_vars['INF_ENGINE_INCLUDE_DIRS:PATH'] = str(builderDLDT.sysrootdir / 'deployment_tools/inference_engine/include') + cmake_vars['ngraph_DIR:PATH'] = str(builderDLDT.sysrootdir / 'ngraph/cmake') + cmake_vars['TBB_DIR:PATH'] = str(builderDLDT.sysrootdir / 'tbb/cmake') + + if self.config.build_debug: + cmake_vars['CMAKE_BUILD_TYPE'] = 'Debug' + cmake_vars['BUILD_opencv_python3'] ='OFF' # python3x_d.lib is missing + cmake_vars['OPENCV_INSTALL_APPS_LIST'] = 'all' + + if self.config.build_tests: + cmake_vars['BUILD_TESTS'] = 'ON' + cmake_vars['BUILD_PERF_TESTS'] = 'ON' + cmake_vars['BUILD_opencv_ts'] = 'ON' + cmake_vars['INSTALL_TESTS']='ON' + + if self.config.build_tests_dnn: + cmake_vars['BUILD_TESTS'] = 'ON' + cmake_vars['BUILD_PERF_TESTS'] = 'ON' + cmake_vars['BUILD_opencv_ts'] = 'ON' + cmake_vars['OPENCV_BUILD_TEST_MODULES_LIST'] = 'dnn' + cmake_vars['OPENCV_BUILD_PERF_TEST_MODULES_LIST'] = 'dnn' + cmake_vars['INSTALL_TESTS']='ON' + + cmd += [ "-D%s=%s" % (k, v) for (k, v) in cmake_vars.items() if v is not None] + if self.config.cmake_option: + cmd += self.config.cmake_option + + cmd.append(str(self.src_dir)) + + log.info('Configuring OpenCV...') + + execute(cmd, cwd=self.build_dir) + + log.info('Building OpenCV...') + + # build + cmd = [self.cmake_path, '--build', '.', '--config', build_config, '--target', 'install', + '--', '/v:n', '/m:2', '/consoleloggerparameters:NoSummary' + ] + execute(cmd, cwd=self.build_dir) + + log.info('OpenCV build/install completed') + + + def copy_sysroot(self, builderDLDT): + log.info('Copy sysroot files') + + copytree(builderDLDT.sysrootdir / 'bin', self.install_dir / 'bin') + copytree(builderDLDT.sysrootdir / 'etc', self.install_dir / 'etc') + + log.info('Copy sysroot files - DONE') + + + def package_sources(self): + package_opencv = prepare_dir(self.package_dir / 'src/opencv', clean=True) + package_opencv = str(package_opencv) # Python 3.5 may not handle Path + execute(cmd=['git', 'clone', '-s', str(self.src_dir), '.'], cwd=str(package_opencv)) + for item in os.listdir(package_opencv): + if str(item).startswith('.git'): + rm_one(os.path.join(package_opencv, item)) + + with open(str(self.package_dir / 'README.md'), 'w') as f: + f.write('See licensing/copying statements in "build/etc/licenses"') + + log.info('Package OpenCV sources - DONE') + + +#=================================================================================================== + +def main(): + + dldt_src_url = 'https://github.com/opencv/dldt.git' + dldt_src_commit = '2020.1' + dldt_release = '2020010000' + + build_cache_dir_default = os.environ.get('BUILD_CACHE_DIR', '.build_cache') + + parser = argparse.ArgumentParser( + description='Build OpenCV Windows package with Inference Engine (DLDT)', + ) + parser.add_argument('output_dir', nargs='?', default='.', help='Output directory') + parser.add_argument('opencv_dir', nargs='?', default=os.path.join(SCRIPT_DIR, '../..'), help='Path to OpenCV source dir') + parser.add_argument('--build_cache_dir', default=build_cache_dir_default, help='Build cache directory (sources and binaries cache of build dependencies, default = "%s")' % build_cache_dir_default) + + parser.add_argument('--cmake_option', action='append', help='Append OpenCV CMake option') + parser.add_argument('--cmake_option_dldt', action='append', help='Append CMake option for DLDT project') + + parser.add_argument('--clean_dldt', action='store_true', help='Clear DLDT build and sysroot directories') + parser.add_argument('--clean_opencv', action='store_true', help='Clear OpenCV build directory') + + parser.add_argument('--build_debug', action='store_true', help='Build debug binaries') + parser.add_argument('--build_tests', action='store_true', help='Build OpenCV tests') + parser.add_argument('--build_tests_dnn', action='store_true', help='Build OpenCV DNN accuracy and performance tests only') + + parser.add_argument('--dldt_src_url', default=dldt_src_url, help='DLDT source URL (tag / commit, default: %s)' % dldt_src_url) + parser.add_argument('--dldt_src_branch', help='DLDT checkout branch') + parser.add_argument('--dldt_src_commit', default=dldt_src_commit, help='DLDT source commit / tag (default: %s)' % dldt_src_commit) + parser.add_argument('--dldt_src_git_clone_extra', action='append', help='DLDT git clone extra args') + parser.add_argument('--dldt_release', default=dldt_release, help='DLDT release code for INF_ENGINE_RELEASE (default: %s)' % dldt_release) + + parser.add_argument('--dldt_reference_dir', help='DLDT reference git repository (optional)') + parser.add_argument('--dldt_src_dir', help='DLDT custom source repository (skip git checkout and patching, use for TESTING only)') + + parser.add_argument('--dldt_config', help='Specify DLDT build configuration (defaults to DLDT commit)') + + args = parser.parse_args() + + log.basicConfig( + format='%(asctime)s %(levelname)-8s %(message)s', + level=os.environ.get('LOGLEVEL', 'INFO'), + datefmt='%Y-%m-%d %H:%M:%S' + ) + log.debug('Args: %s', args) + + if not check_executable(['git', '--version']): + sys.exit("FATAL: 'git' is not available") + if not check_executable(['cmake', '--version']): + sys.exit("FATAL: 'cmake' is not available") + + if os.path.realpath(args.output_dir) == os.path.realpath(SCRIPT_DIR): + raise Fail("Specify output_dir (building from script directory is not supported)") + if os.path.realpath(args.output_dir) == os.path.realpath(args.opencv_dir): + raise Fail("Specify output_dir (building from OpenCV source directory is not supported)") + + # Relative paths become invalid in sub-directories + if args.opencv_dir is not None and not os.path.isabs(args.opencv_dir): + args.opencv_dir = os.path.abspath(args.opencv_dir) + + if not args.dldt_config: + args.dldt_config = args.dldt_src_commit + + _opencv_dir = check_dir(args.opencv_dir) + _outdir = prepare_dir(args.output_dir) + _cachedir = prepare_dir(args.build_cache_dir) + + ocv_hooks_dir = os.environ.get('OPENCV_CMAKE_HOOKS_DIR', None) + hooks_dir = os.path.join(SCRIPT_DIR, 'cmake-opencv-checks') + os.environ['OPENCV_CMAKE_HOOKS_DIR'] = hooks_dir if ocv_hooks_dir is None else (hooks_dir + ';' + ocv_hooks_dir) + + builder_dldt = BuilderDLDT(args) + + builder_dldt.prepare_sources() + builder_dldt.build() + builder_dldt.make_sysroot() + + builder_opencv = Builder(args) + builder_opencv.build(builder_dldt) + builder_opencv.copy_sysroot(builder_dldt) + builder_opencv.package_sources() + + log.info("=====") + log.info("===== Build finished") + log.info("=====") + + +if __name__ == "__main__": + try: + main() + except: + log.info('FATAL: Error occured. To investigate problem try to change logging level using LOGLEVEL=DEBUG environment variable.') + raise diff --git a/platforms/winpack_dldt/cmake-opencv-checks/POST_FINALIZE.cmake b/platforms/winpack_dldt/cmake-opencv-checks/POST_FINALIZE.cmake new file mode 100644 index 0000000000..0952262a84 --- /dev/null +++ b/platforms/winpack_dldt/cmake-opencv-checks/POST_FINALIZE.cmake @@ -0,0 +1,17 @@ +message(STATUS "Winpack-DLDT: Validating OpenCV build configuration...") + +if(NOT INF_ENGINE_TARGET) + message(SEND_ERROR "Inference engine must be detected") + set(HAS_ERROR 1) +endif() +if(NOT HAVE_NGRAPH) + message(SEND_ERROR "Inference engine nGraph must be detected") + set(HAS_ERROR 1) +endif() + +if(HAS_ERROR) + ocv_cmake_dump_vars("^IE_|INF_|INFERENCE|ngraph") + message(FATAL_ERROR "Winpack-DLDT: Validating OpenCV build configuration... FAILED") +endif() + +message(STATUS "Winpack-DLDT: Validating OpenCV build configuration... DONE") diff --git a/platforms/winpack_dldt/package-tests/test_dnn_backends.py b/platforms/winpack_dldt/package-tests/test_dnn_backends.py new file mode 100644 index 0000000000..e3b6e16f3b --- /dev/null +++ b/platforms/winpack_dldt/package-tests/test_dnn_backends.py @@ -0,0 +1,13 @@ +import sys +print(sys.version_info) +try: + import cv2 as cv + print(cv.__version__) + print(cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE)) +except: + print(sys.path) + import os + print(os.environ.get('PATH', '')) + raise + +print('OK')