From a7de4aca6ab096aa99a57b301875bc16760f37d1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 28 Mar 2017 18:21:13 +0300 Subject: [PATCH] cmake: update ocv_download - more aggressive cache for files under CMAKE_BINARY_CACHE * don't re-read files MD5 * don't repack archives - add support for ENV{OPENCV_DOWNLOAD_PATH} - added ID parameter as project/group identifier - non-flat .cache directory (based on ID) - download message prefix based on ID - more detailed logging via ocv_download_log() macro - force .gitignore file for .cache folder (with '*' pattern) --- .gitignore | 1 - 3rdparty/ffmpeg/ffmpeg.cmake | 1 + 3rdparty/ippicv/ippicv.cmake | 1 + 3rdparty/tbb/CMakeLists.txt | 1 + cmake/OpenCVDownload.cmake | 119 +++++++++++++++++++++++++++++------ 5 files changed, 104 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 0fdd3e81ef..adef6e08d1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ *.user *~ .*.swp -.cache .DS_Store .sw[a-z] Thumbs.db diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index c26cc7a3c6..b9f5292352 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -25,6 +25,7 @@ function(download_win_ffmpeg script_var) "${OPENCV_FFMPEG_URL}" "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${FFMPEG_BINARIES_COMMIT}/ffmpeg/" DESTINATION_DIR ${FFMPEG_DOWNLOAD_DIR} + ID FFMPEG RELATIVE_URL STATUS res) if(NOT res) diff --git a/3rdparty/ippicv/ippicv.cmake b/3rdparty/ippicv/ippicv.cmake index ce100a1ffe..a477d60b8a 100644 --- a/3rdparty/ippicv/ippicv.cmake +++ b/3rdparty/ippicv/ippicv.cmake @@ -31,6 +31,7 @@ function(download_ippicv root_var) "$ENV{OPENCV_IPPICV_URL}" "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${IPPICV_COMMIT}/ippicv/" DESTINATION_DIR "${THE_ROOT}" + ID IPPICV STATUS res UNPACK RELATIVE_URL) diff --git a/3rdparty/tbb/CMakeLists.txt b/3rdparty/tbb/CMakeLists.txt index fa37cc26fc..41e2cad1a9 100644 --- a/3rdparty/tbb/CMakeLists.txt +++ b/3rdparty/tbb/CMakeLists.txt @@ -21,6 +21,7 @@ ocv_download(FILENAME ${tbb_filename} "$ENV{OPENCV_TBB_URL}" "https://github.com/01org/tbb/archive/" DESTINATION_DIR ${tbb_src_dir} + ID TBB STATUS res UNPACK RELATIVE_URL) if(NOT res) diff --git a/cmake/OpenCVDownload.cmake b/cmake/OpenCVDownload.cmake index e118e28f87..76151bb8ed 100644 --- a/cmake/OpenCVDownload.cmake +++ b/cmake/OpenCVDownload.cmake @@ -1,11 +1,12 @@ # # Download and optionally unpack a file # -# ocv_download(FILENAME p HASH h URL u1 [u2 ...] DESTINATION_DIR d [STATUS s] [UNPACK] [RELATIVE]) +# ocv_download(FILENAME p HASH h URL u1 [u2 ...] DESTINATION_DIR d [ID id] [STATUS s] [UNPACK] [RELATIVE_URL]) # FILENAME - filename # HASH - MD5 hash # URL - full download url (first nonempty value will be chosen) # DESTINATION_DIR - file will be copied to this directory +# ID - identifier for project/group of downloaded files # STATUS - passed variable will be updated in parent scope, # function will not fail the build in case of download problem if this option is provided, # but will fail in case when other operations (copy, remove, etc.) failed @@ -13,28 +14,76 @@ # RELATIVE_URL - if set, then URL is treated as a base, and FILENAME will be appended to it # Note: uses OPENCV_DOWNLOAD_PATH folder as cache, default is /.cache -set(OPENCV_DOWNLOAD_PATH "${OpenCV_SOURCE_DIR}/.cache" CACHE PATH "Cache directory for downloaded files") +set(HELP_OPENCV_DOWNLOAD_PATH "Cache directory for downloaded files") +if(DEFINED ENV{OPENCV_DOWNLOAD_PATH}) + set(OPENCV_DOWNLOAD_PATH "$ENV{OPENCV_DOWNLOAD_PATH}" CACHE PATH "${HELP_OPENCV_DOWNLOAD_PATH}") +endif() +set(OPENCV_DOWNLOAD_PATH "${OpenCV_SOURCE_DIR}/.cache" CACHE PATH "${HELP_OPENCV_DOWNLOAD_PATH}") set(OPENCV_DOWNLOAD_LOG "${OpenCV_BINARY_DIR}/CMakeDownloadLog.txt") # Init download cache directory and log file if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}") file(MAKE_DIRECTORY ${OPENCV_DOWNLOAD_PATH}) endif() +if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}/.gitignore") + file(WRITE "${OPENCV_DOWNLOAD_PATH}/.gitignore" "*\n") +endif() file(WRITE "${OPENCV_DOWNLOAD_LOG}" "use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n") function(ocv_download) - cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;STATUS" "URL" ${ARGN}) - - ocv_assert(DEFINED DL_FILENAME) - ocv_assert(DEFINED DL_HASH) - ocv_assert(DEFINED DL_URL) - ocv_assert(DEFINED DL_DESTINATION_DIR) + cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;ID;STATUS" "URL" ${ARGN}) + + macro(ocv_download_log) + file(APPEND "${OPENCV_DOWNLOAD_LOG}" "${ARGN}\n") + endmacro() + + ocv_assert(DL_FILENAME) + ocv_assert(DL_HASH) + ocv_assert(DL_URL) + ocv_assert(DL_DESTINATION_DIR) + if((NOT " ${DL_UNPARSED_ARGUMENTS}" STREQUAL " ") + OR DL_FILENAME STREQUAL "" + OR DL_HASH STREQUAL "" + OR DL_URL STREQUAL "" + OR DL_DESTINATION_DIR STREQUAL "" + ) + set(msg_level FATAL_ERROR) + if(DEFINED DL_STATUS) + set(${DL_STATUS} FALSE PARENT_SCOPE) + set(msg_level WARNING) + endif() + message(${msg_level} "ERROR: ocv_download() unsupported arguments: ${ARGV}") + return() + endif() if(DEFINED DL_STATUS) set(${DL_STATUS} TRUE PARENT_SCOPE) endif() + # Check CMake cache for already processed tasks + string(FIND "${DL_DESTINATION_DIR}" "${CMAKE_BINARY_DIR}" DL_BINARY_PATH_POS) + if(DL_BINARY_PATH_POS EQUAL 0) + set(__file_id "${DL_DESTINATION_DIR}/${DL_FILENAME}") + file(RELATIVE_PATH __file_id "${CMAKE_BINARY_DIR}" "${__file_id}") + string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" __file_id "${__file_id}") + if(DL_ID) + string(TOUPPER ${DL_ID} __id) + string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" __id "${__id}") + set(OCV_DOWNLOAD_HASH_NAME "OCV_DOWNLOAD_${__id}_HASH_${__file_id}") + else() + set(OCV_DOWNLOAD_HASH_NAME "OCV_DOWNLOAD_HASH_${__file_id}") + endif() + if(" ${${OCV_DOWNLOAD_HASH_NAME}}" STREQUAL " ${DL_HASH}") + ocv_download_log("#match_hash_in_cmake_cache \"${OCV_DOWNLOAD_HASH_NAME}\"") + return() + endif() + unset("${OCV_DOWNLOAD_HASH_NAME}" CACHE) + else() + set(OCV_DOWNLOAD_HASH_NAME "") + #message(WARNING "Download destination is not in CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}: ${DL_DESTINATION_DIR}") + endif() + # Select first non-empty url foreach(url ${DL_URL}) if(url) @@ -54,58 +103,85 @@ function(ocv_download) endif() # Log all calls to file - file(APPEND "${OPENCV_DOWNLOAD_LOG}" "do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"\n") + ocv_download_log("do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"") # ... and to console - message(STATUS "Download: ${DL_FILENAME}") + set(__msg_prefix "") + if(DL_ID) + set(__msg_prefix "${DL_ID}: ") + endif() + message(STATUS "${__msg_prefix}Download: ${DL_FILENAME}") # Copy mode: check if copy destination exists and is correct if(NOT DL_UNPACK) set(COPY_DESTINATION "${DL_DESTINATION_DIR}/${DL_FILENAME}") if(EXISTS "${COPY_DESTINATION}") + ocv_download_log("#check_md5 \"${COPY_DESTINATION}\"") file(MD5 "${COPY_DESTINATION}" target_md5) if(target_md5 STREQUAL DL_HASH) + ocv_download_log("#match_md5 \"${COPY_DESTINATION}\" \"${target_md5}\"") + if(OCV_DOWNLOAD_HASH_NAME) + set(${OCV_DOWNLOAD_HASH_NAME} "${DL_HASH}" CACHE INTERNAL "") + endif() return() endif() + ocv_download_log("#mismatch_md5 \"${COPY_DESTINATION}\" \"${target_md5}\"") + else() + ocv_download_log("#missing \"${COPY_DESTINATION}\"") endif() endif() # Check cache first - set(CACHE_CANDIDATE "${OPENCV_DOWNLOAD_PATH}/${DL_HASH}-${DL_FILENAME}") + if(DL_ID) + string(TOLOWER "${DL_ID}" __id) + string(REGEX REPLACE "[^a-zA-Z0-9_/ ]" "_" __id "${__id}") + set(CACHE_CANDIDATE "${OPENCV_DOWNLOAD_PATH}/${__id}/${DL_HASH}-${DL_FILENAME}") + else() + set(CACHE_CANDIDATE "${OPENCV_DOWNLOAD_PATH}/${DL_HASH}-${DL_FILENAME}") + endif() if(EXISTS "${CACHE_CANDIDATE}") + ocv_download_log("#check_md5 \"${CACHE_CANDIDATE}\"") file(MD5 "${CACHE_CANDIDATE}" target_md5) if(NOT target_md5 STREQUAL DL_HASH) + ocv_download_log("#mismatch_md5 \"${CACHE_CANDIDATE}\" \"${target_md5}\"") + ocv_download_log("#delete \"${CACHE_CANDIDATE}\"") file(REMOVE ${CACHE_CANDIDATE}) endif() endif() # Download if(NOT EXISTS "${CACHE_CANDIDATE}") + ocv_download_log("#cmake_download \"${CACHE_CANDIDATE}\" \"${DL_URL}\"") file(DOWNLOAD "${DL_URL}" "${CACHE_CANDIDATE}" INACTIVITY_TIMEOUT 60 TIMEOUT 600 STATUS status - LOG log) - string(REPLACE "\n" "\n# " log "# ${log}") - file(APPEND "${OPENCV_DOWNLOAD_LOG}" "${log}\n\n") + LOG __log) + string(LENGTH "${__log}" __log_length) + if(__log_length LESS 65536) + string(REPLACE "\n" "\n# " __log "${__log}") + ocv_download_log("# ${__log}\n") + endif() if(NOT status EQUAL 0) set(msg_level FATAL_ERROR) if(DEFINED DL_STATUS) set(${DL_STATUS} FALSE PARENT_SCOPE) set(msg_level WARNING) endif() - message(${msg_level} "Download failed: ${status}") + message(${msg_level} "${__msg_prefix}Download failed: ${status}") return() endif() # Don't remove this code, because EXPECTED_MD5 parameter doesn't fail "file(DOWNLOAD)" step on wrong hash + ocv_download_log("#check_md5 \"${CACHE_CANDIDATE}\"") file(MD5 "${CACHE_CANDIDATE}" target_md5) if(NOT target_md5 STREQUAL DL_HASH) + ocv_download_log("#mismatch_md5 \"${CACHE_CANDIDATE}\" \"${target_md5}\"") set(msg_level FATAL_ERROR) if(DEFINED DL_STATUS) set(${DL_STATUS} FALSE PARENT_SCOPE) set(msg_level WARNING) endif() - message(${msg_level} "Hash mismatch: ${target_md5}") + message(${msg_level} "${__msg_prefix}Hash mismatch: ${target_md5}") return() endif() endif() @@ -113,21 +189,28 @@ function(ocv_download) # Unpack or copy if(DL_UNPACK) if(EXISTS "${DL_DESTINATION_DIR}") + ocv_download_log("#remove_unpack \"${DL_DESTINATION_DIR}\"") file(REMOVE_RECURSE "${DL_DESTINATION_DIR}") endif() + ocv_download_log("#mkdir \"${DL_DESTINATION_DIR}\"") file(MAKE_DIRECTORY "${DL_DESTINATION_DIR}") + ocv_download_log("#unpack \"${DL_DESTINATION_DIR}\" \"${CACHE_CANDIDATE}\"") execute_process(COMMAND "${CMAKE_COMMAND}" -E tar xz "${CACHE_CANDIDATE}" WORKING_DIRECTORY "${DL_DESTINATION_DIR}" RESULT_VARIABLE res) if(NOT res EQUAL 0) - message(FATAL_ERROR "Unpack failed: ${res}") + message(FATAL_ERROR "${__msg_prefix}Unpack failed: ${res}") endif() else() + ocv_download_log("#copy \"${COPY_DESTINATION}\" \"${CACHE_CANDIDATE}\"") execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CACHE_CANDIDATE}" "${COPY_DESTINATION}" RESULT_VARIABLE res) if(NOT res EQUAL 0) - message(FATAL_ERROR "Copy failed: ${res}") + message(FATAL_ERROR "${__msg_prefix}Copy failed: ${res}") endif() endif() + if(OCV_DOWNLOAD_HASH_NAME) + set(${OCV_DOWNLOAD_HASH_NAME} "${DL_HASH}" CACHE INTERNAL "") + endif() endfunction()