From f7ac724c5d2b7f7580e840c4c623609098b17bdc Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Fri, 18 Mar 2022 17:50:21 +0800 Subject: [PATCH] Merge pull request #21531 from fengyuentau:add_3rdparty_mirrors Support downloading 3rdparty resources from Gitcode & Gitlab-style mirrors * replace github.com with gitcode.net for ocv_download * replace raw.githubusercontent.com with gitcode.net for ocv_download * rename funtions and remove some comments * add options for custom mirrors, which simply replace domain github.com & githubusercontent.com * run ocv_init_download once; replace DL_URL with mirrored one when calling ocv_download * fix for empty download links when not using mirror * fix bugs: set(.. .. PARENT_SCOPE) for ocv_init_download; correct macro names for replace github archives and raw githubusercontent * adjusted mirror swapping impl: replace with mirrored link before each ocv_download; update md5sum for archives * fix a bug: macro invoked with incorrect arguments by non-set vars * enclose if statement * workable impl * shorten the var names of two key options * scalable implementation of downloading from mirror and using custom mirror * improve ocv_init_download help message * fix the different extracted directory name in case of ADE & TBB which are downloaded from release page * improve help message printing * Download ADE & TBB using commit ids instead of from release pages * support custom mirrors on downloading archives * improve hints * add missing parentheses * reset ocv_download calls * mirror support implementation using ocv_cmake_hook & ocv_cmake_hook_append * move ocv_init_download into cmake/OpenCVDownload.cmake * move ocv_cmake_hook before checking CMake cache * improve hints when not fetching as git repo * add WORKING_DIRECTORY in execute_process in ocv_init_download * use OPENCV_DOWNLOAD_MIRROR_ID * add custom.cmake for custom mirror * detect github origin * fix broken var name * download from github by default if custom tbb is set * add checksum checks for gitcode.cmake before replacing urls and checksums * add checksum checks for custom.cmake before replacing urls and checkusms * use description specify instead of set for messages in custom.cmake; use warning message for warnings * updates and fixes --- cmake/OpenCVDownload.cmake | 53 ++++++++++++++++++++++++- cmake/mirrors/custom.cmake | 77 +++++++++++++++++++++++++++++++++++++ cmake/mirrors/gitcode.cmake | 70 +++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 cmake/mirrors/custom.cmake create mode 100644 cmake/mirrors/gitcode.cmake diff --git a/cmake/OpenCVDownload.cmake b/cmake/OpenCVDownload.cmake index a427a41227..3e46515537 100644 --- a/cmake/OpenCVDownload.cmake +++ b/cmake/OpenCVDownload.cmake @@ -37,6 +37,50 @@ file(WRITE "${OPENCV_DOWNLOAD_LOG}" "#use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n") file(REMOVE "${OPENCV_DOWNLOAD_WITH_CURL}") file(REMOVE "${OPENCV_DOWNLOAD_WITH_WGET}") +ocv_check_environment_variables(OPENCV_DOWNLOAD_MIRROR_ID) + +function(ocv_init_download_mirror) + if(NOT DEFINED OPENCV_DOWNLOAD_MIRROR_ID) + # Run `git remote get-url origin` to get remote source + execute_process( + COMMAND + git remote get-url origin + WORKING_DIRECTORY + ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE + RESULT_STATUS + OUTPUT_VARIABLE + OCV_GIT_ORIGIN_URL_OUTPUT + ERROR_QUIET + ) + # if non-git, OCV_GIT_ORIGIN_URL_OUTPUT is empty + if(NOT OCV_GIT_ORIGIN_URL_OUTPUT) + message(STATUS "ocv_init_download: OpenCV source tree is not fetched as git repository. 3rdparty resources will be downloaded from github.com by default.") + return() + else() + # Check if git origin is github.com + string(FIND "${OCV_GIT_ORIGIN_URL_OUTPUT}" "github.com" _found_github) + if(NOT ${_found_github} EQUAL -1) + set(OPENCV_DOWNLOAD_MIRROR_ID "github" CACHE STRING "") + endif() + # Check if git origin is gitcode.net + string(FIND "${OCV_GIT_ORIGIN_URL_OUTPUT}" "gitcode.net" _found_gitcode) + if(NOT ${_found_gitcode} EQUAL -1) + set(OPENCV_DOWNLOAD_MIRROR_ID "gitcode" CACHE STRING "") + endif() + endif() + endif() + + if(OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "gitcode" OR OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "custom") + message(STATUS "ocv_init_download: Using ${OPENCV_DOWNLOAD_MIRROR_ID}-hosted mirror to download 3rdparty components.") + ocv_cmake_hook_append(OPENCV_DOWNLOAD_PRE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/mirrors/${OPENCV_DOWNLOAD_MIRROR_ID}.cmake") + elseif(OPENCV_DOWNLOAD_MIRROR_ID STREQUAL "github") + return() + else() + message(STATUS "ocv_init_download: Unable to recognize git server of OpenCV source code. Using github.com to download 3rdparty components.") + endif() +endfunction() + function(ocv_download) cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;ID;STATUS" "URL" ${ARGN}) @@ -67,6 +111,8 @@ function(ocv_download) set(${DL_STATUS} TRUE PARENT_SCOPE) endif() + ocv_cmake_hook(OPENCV_DOWNLOAD_PRE) + # Check CMake cache for already processed tasks string(FIND "${DL_DESTINATION_DIR}" "${CMAKE_BINARY_DIR}" DL_BINARY_PATH_POS) if(DL_BINARY_PATH_POS EQUAL 0) @@ -115,7 +161,7 @@ function(ocv_download) if(DL_ID) set(__msg_prefix "${DL_ID}: ") endif() - message(STATUS "${__msg_prefix}Download: ${DL_FILENAME}") + message(STATUS "${__msg_prefix}Downloading ${DL_FILENAME} from ${DL_URL}") # Copy mode: check if copy destination exists and is correct if(NOT DL_UNPACK) @@ -252,3 +298,8 @@ ${OPENCV_DOWNLOAD_LOG} set(${OCV_DOWNLOAD_HASH_NAME} "${DL_HASH}" CACHE INTERNAL "") endif() endfunction() + +# ---------------------------------------------------------------------------- +# Initialize download in case mirror is used +# ---------------------------------------------------------------------------- +ocv_init_download_mirror() diff --git a/cmake/mirrors/custom.cmake b/cmake/mirrors/custom.cmake new file mode 100644 index 0000000000..2be93d8165 --- /dev/null +++ b/cmake/mirrors/custom.cmake @@ -0,0 +1,77 @@ +# Gitlab-style mirror +# CMake scripts look for opencv/opencv_3rdparty, +# OAID/Tengine, 01org/tbb(oneAPI/oneTBB), opencv/ade +# from OPENCV_DOWNLOAD_MIRROR +ocv_update(OPENCV_DOWNLOAD_MIRROR_URL "") +# Tengine +ocv_update(TENGINE_PKG_MD5_CUSTOM "") +ocv_update(TENGINE_PKG_MD5_ORIGINAL 23f61ebb1dd419f1207d8876496289c5) # same as tengine_md5sum for TENGINE commit of e89cf8870de2ff0a80cfe626c0b52b2a16fb302e + +# TBB +ocv_update(TBB_RELEASE_CUSTOM "") +ocv_update(TBB_PKG_NAME_CUSTOM "") +ocv_update(TBB_PKG_MD5_CUSTOM "") +ocv_update(TBB_PKG_MD5_ORIGINAL 5af6f6c2a24c2043e62e47205e273b1f) # same as OPENCV_TBB_RELEASE_MD5 for TBB release of v2020.2 + +# ADE +ocv_update(ADE_RELEASE_CUSTOM "") +ocv_update(ADE_PKG_NAME_CUSTOM "") +ocv_update(ADE_PKG_MD5_CUSTOM "") +ocv_update(ADE_PKG_MD5_ORIGINAL b624b995ec9c439cbc2e9e6ee940d3a2) # same as ade_md5 for ADE release of v0.1.1f + +macro(ocv_download_url_custom_usercontent) + string(REPLACE "/" ";" DL_URL_split ${DL_URL}) + list(GET DL_URL_split 5 __COMMIT_ID) + list(GET DL_URL_split 6 __PKG_NAME) + set(DL_URL "https://${OPENCV_DOWNLOAD_MIRROR_URL}/opencv/opencv_3rdparty/-/raw/${__COMMIT_ID}/${__PKG_NAME}/") +endmacro() +macro(ocv_download_url_custom_archive_commit_id) + if("m${${DL_ID}_PKG_MD5_CUSTOM}" STREQUAL "m") + message(WARNING "ocv_download: specify ${DL_ID}_PKG_MD5_CUSTOM to download ${DL_ID} from custom source.") + elseif(${DL_ID}_PKG_MD5_ORIGINAL STREQUAL "${DL_HASH}") + string(REPLACE "/" ";" DL_URL_split ${DL_URL}) + list(GET DL_URL_split 3 __OWNER) + list(GET DL_URL_split 4 __REPO_NAME) + set(DL_URL "https://${OPENCV_DOWNLOAD_MIRROR_URL}/${__OWNER}/${__REPO_NAME}/-/archive/") + set(DL_HASH "${${DL_ID}_PKG_MD5_CUSTOM}") + else() + message(WARNING "No information about mirrors for downloading ${DL_FILENAME} from URL='${DL_URL}' and MD5=${DL_HASH}.") + endif() +endmacro() +macro(ocv_download_url_custom_archive_release) + if("m${${DL_ID}_RELEASE_CUSTOM}" STREQUAL "m") + message(WARNING "ocv_download: specify ${DL_ID}_RELEASE_CUSTOM to download ${DL_ID} from custom source.") + return() + endif() + if("m${${DL_ID}_PKG_NAME_CUSTOM}" STREQUAL "m") + message(WARNING "ocv_download: specify ${DL_ID}_PKG_NAME_CUSTOM to download ${DL_ID} from custom source.") + return() + endif() + if("m${${DL_ID}_PKG_MD5_CUSTOM}" STREQUAL "m") + message(WARNING "ocv_download: specify ${DL_ID}_PKG_MD5_CUSTOM to download ${DL_ID} from custom source.") + return() + endif() + string(REPLACE "/" ";" DL_URL_split ${DL_URL}) + list(GET DL_URL_split 3 __OWNER) + list(GET DL_URL_split 4 __REPO_NAME) + set(DL_URL "https://${OPENCV_DOWNLOAD_MIRROR_URL}/${__OWNER}/${__REPO_NAME}/-/archive/${${DL_ID}_PKG_RELEASE_CUSTOM}/${__REPO_NAME}-") + set(DL_HASH "${${DL_ID}_PKG_MD5_CUSTOM}") +endmacro() + +if("m${OPENCV_DOWNLOAD_MIRROR_URL}" STREQUAL "m") + message(WARNING "ocv_download: specify OPENCV_DOWNLOAD_MIRROR_URL to use custom mirror.") +else() + if((DL_ID STREQUAL "FFMPEG") OR (DL_ID STREQUAL "IPPICV")) + ocv_download_url_custom_usercontent() + elseif(DL_ID STREQUAL "TENGINE") + ocv_download_url_custom_archive_commit_id() + elseif(DL_ID STREQUAL "TBB") + ocv_download_url_custom_archive_release() + set(OPENCV_TBB_SUBDIR "${TBB_PKG_NAME_CUSTOM}" PARENT_SCOPE) + elseif(DL_ID STREQUAL "ADE") + ocv_download_url_custom_archive_release() + set(ade_subdir "${ADE_PKG_NAME_CUSTOM}" PARENT_SCOPE) + else() + message(STATUS "ocv_download: Unknown download ID ${DL_ID} for using mirror ${OPENCV_DOWNLOAD_MIRROR_URL}. Use original source instead.") + endif() +endif() \ No newline at end of file diff --git a/cmake/mirrors/gitcode.cmake b/cmake/mirrors/gitcode.cmake new file mode 100644 index 0000000000..aafe9635f3 --- /dev/null +++ b/cmake/mirrors/gitcode.cmake @@ -0,0 +1,70 @@ +# Tengine (Download via commit id) +ocv_update(TENGINE_PKG_MD5_GITCODE 1b5908632b557275cd6e85b0c03f9690) +ocv_update(TENGINE_PKG_MD5_ORIGINAL 23f61ebb1dd419f1207d8876496289c5) # same as tengine_md5sum for TENGINE commit of e89cf8870de2ff0a80cfe626c0b52b2a16fb302e + +# TBB (Download from release page) +ocv_update(TBB_RELEASE_GITCODE "v2020.2") +ocv_update(TBB_PKG_NAME_GITCODE "tbb-${TBB_RELEASE_GITCODE}") +ocv_update(TBB_PKG_MD5_GITCODE 4eeafdf16a90cb66e39a31c8d6c6804e) +ocv_update(TBB_PKG_MD5_ORIGINAL 5af6f6c2a24c2043e62e47205e273b1f) # same as OPENCV_TBB_RELEASE_MD5 for TBB release of v2020.2 + +# ADE (Download from release page) +ocv_update(ADE_RELEASE_GITCODE "v0.1.1f") +ocv_update(ADE_PKG_NAME_GITCODE "ade-${ADE_RELEASE_GITCODE}") +ocv_update(ADE_PKG_MD5_GITCODE c12909e0ccfa93138c820ba91ff37b3c) +ocv_update(ADE_PKG_MD5_ORIGINAL b624b995ec9c439cbc2e9e6ee940d3a2) # same as ade_md5 for ADE release of v0.1.1f + +# +# Replace download links for packages in opencv/opencv_3rdparty: +# 1. Extract repo owner and repo name from DL_URL. +# 2. Put repo owner and repo name into the placeholders of new DL_URL. +# +macro(ocv_download_url_gitcode_usercontent) + string(REPLACE "/" ";" DL_URL_split ${DL_URL}) + list(GET DL_URL_split 5 __COMMIT_ID) + list(GET DL_URL_split 6 __PKG_NAME) + set(DL_URL "https://gitcode.net/opencv/opencv_3rdparty/-/raw/${__COMMIT_ID}/${__PKG_NAME}/") +endmacro() +# +# Replace download links and checksums for archives/releases in other repositories: +# 1. Check if versions matched. If not matched, download from github instead. +# 2. Extract repo owner and repo name from DL_URL. +# 3. Put repo owner and repo name into the placeholders of new DL_URL. +# 4. Replace DL_HASH with the one downloaded from gitcode.net. +# +macro(ocv_download_url_gitcode_archive_commit_id) + if(DL_HASH STREQUAL "${${DL_ID}_PKG_MD5_ORIGINAL}") + string(REPLACE "/" ";" DL_URL_split ${DL_URL}) + list(GET DL_URL_split 3 __OWNER) + list(GET DL_URL_split 4 __REPO_NAME) + set(DL_URL "https://gitcode.net/${__OWNER}/${__REPO_NAME}/-/archive/") + set(DL_HASH "${${DL_ID}_PKG_MD5_GITCODE}") + else() + message(WARNING "Package ${DL_ID} from mirror gitcode.net is outdated and will be downloaded from github.com instead.") + endif() +endmacro() +macro(ocv_download_url_gitcode_archive_release) + if(DL_HASH STREQUAL "${${DL_ID}_PKG_MD5_ORIGINAL}") + string(REPLACE "/" ";" DL_URL_split ${DL_URL}) + list(GET DL_URL_split 3 __OWNER) + list(GET DL_URL_split 4 __REPO_NAME) + set(DL_URL "https://gitcode.net/${__OWNER}/${__REPO_NAME}/-/archive/${${DL_ID}_RELEASE_GITCODE}/${__REPO_NAME}-") + set(DL_HASH "${${DL_ID}_PKG_MD5_GITCODE}") + else() + message(WARNING "Package ${DL_ID} from mirror gitcode.net is outdated and will be downloaded from github.com instead.") + endif() +endmacro() + +if((DL_ID STREQUAL "FFMPEG") OR (DL_ID STREQUAL "IPPICV")) + ocv_download_url_gitcode_usercontent() +elseif(DL_ID STREQUAL "TENGINE") + ocv_download_url_gitcode_archive_commit_id() +elseif(DL_ID STREQUAL "TBB") + ocv_download_url_gitcode_archive_release() + set(OPENCV_TBB_SUBDIR "${TBB_PKG_NAME_GITCODE}" PARENT_SCOPE) +elseif(DL_ID STREQUAL "ADE") + ocv_download_url_gitcode_archive_release() + set(ade_subdir "${ADE_PKG_NAME_GITCODE}" PARENT_SCOPE) +else() + message(STATUS "ocv_download: Unknown download ID ${DL_ID} for using mirror gitcode.net. Use original source instead.") +endif()