mirror of https://github.com/opencv/opencv.git
Merge pull request #8421 from mshabunin:download-cache
Alexander Alekhin
8 years ago
commit
7cc97fd96c
13 changed files with 243 additions and 319 deletions
@ -1,25 +1,37 @@ |
||||
# Binary branch name: ffmpeg/master_20161202 |
||||
# Binaries were created for OpenCV: 594c136d1fcbb5816c57516e50f9cbeffbd90835 |
||||
set(FFMPEG_BINARIES_COMMIT "2a19d0006415955c79431116e4634f04d5eb5a74") |
||||
set(FFMPEG_FILE_HASH_BIN32 "f081abd9d6ca7e425d340ce586f9c090") |
||||
set(FFMPEG_FILE_HASH_BIN64 "a423363a6eb76d362ca6c406c96c8db6") |
||||
set(FFMPEG_FILE_HASH_CMAKE "5346ae1854fc7aa569a722e85af480ec") |
||||
function(download_win_ffmpeg script_var) |
||||
set(${script_var} "" PARENT_SCOPE) |
||||
|
||||
set(FFMPEG_DOWNLOAD_URL ${OPENCV_FFMPEG_URL};$ENV{OPENCV_FFMPEG_URL};https://raw.githubusercontent.com/opencv/opencv_3rdparty/${FFMPEG_BINARIES_COMMIT}/ffmpeg/) |
||||
set(ids BIN32 BIN64 CMAKE) |
||||
set(name_BIN32 "opencv_ffmpeg.dll") |
||||
set(name_BIN64 "opencv_ffmpeg_64.dll") |
||||
set(name_CMAKE "ffmpeg_version.cmake") |
||||
|
||||
ocv_download(PACKAGE opencv_ffmpeg.dll |
||||
HASH ${FFMPEG_FILE_HASH_BIN32} |
||||
URL ${FFMPEG_DOWNLOAD_URL} |
||||
DESTINATION_DIR ${CMAKE_CURRENT_LIST_DIR}) |
||||
# Binary branch name: ffmpeg/master_20161202 |
||||
# Binaries were created for OpenCV: 594c136d1fcbb5816c57516e50f9cbeffbd90835 |
||||
set(FFMPEG_BINARIES_COMMIT "2a19d0006415955c79431116e4634f04d5eb5a74") |
||||
set(FFMPEG_FILE_HASH_BIN32 "f081abd9d6ca7e425d340ce586f9c090") |
||||
set(FFMPEG_FILE_HASH_BIN64 "a423363a6eb76d362ca6c406c96c8db6") |
||||
set(FFMPEG_FILE_HASH_CMAKE "5346ae1854fc7aa569a722e85af480ec") |
||||
|
||||
ocv_download(PACKAGE opencv_ffmpeg_64.dll |
||||
HASH ${FFMPEG_FILE_HASH_BIN64} |
||||
URL ${FFMPEG_DOWNLOAD_URL} |
||||
DESTINATION_DIR ${CMAKE_CURRENT_LIST_DIR}) |
||||
set(FFMPEG_BINARIES_COMMIT "2a19d0006415955c79431116e4634f04d5eb5a74") |
||||
set(FFMPEG_DOWNLOAD_DIR "${OpenCV_BINARY_DIR}/3rdparty/ffmpeg") |
||||
|
||||
ocv_download(PACKAGE ffmpeg_version.cmake |
||||
HASH ${FFMPEG_FILE_HASH_CMAKE} |
||||
URL ${FFMPEG_DOWNLOAD_URL} |
||||
DESTINATION_DIR ${CMAKE_CURRENT_LIST_DIR}) |
||||
|
||||
include(${CMAKE_CURRENT_LIST_DIR}/ffmpeg_version.cmake) |
||||
set(status TRUE) |
||||
foreach(id ${ids}) |
||||
ocv_download(FILENAME ${name_${id}} |
||||
HASH ${FFMPEG_FILE_HASH_${id}} |
||||
URL |
||||
"$ENV{OPENCV_FFMPEG_URL}" |
||||
"${OPENCV_FFMPEG_URL}" |
||||
"https://raw.githubusercontent.com/opencv/opencv_3rdparty/${FFMPEG_BINARIES_COMMIT}/ffmpeg/" |
||||
DESTINATION_DIR ${FFMPEG_DOWNLOAD_DIR} |
||||
RELATIVE_URL |
||||
STATUS res) |
||||
if(NOT res) |
||||
set(status FALSE) |
||||
endif() |
||||
endforeach() |
||||
if(status) |
||||
set(${script_var} "${FFMPEG_DOWNLOAD_DIR}/ffmpeg_version.cmake" PARENT_SCOPE) |
||||
endif() |
||||
endfunction() |
||||
|
||||
@ -1,2 +0,0 @@ |
||||
downloads/ |
||||
unpack/ |
||||
@ -1,110 +0,0 @@ |
||||
# |
||||
# The script downloads ICV package |
||||
# |
||||
# On return this will define: |
||||
# OPENCV_ICV_PATH - path to unpacked downloaded package |
||||
# |
||||
|
||||
function(_icv_downloader) |
||||
# Commit SHA in the opencv_3rdparty repo |
||||
set(IPPICV_BINARIES_COMMIT "81a676001ca8075ada498583e4166079e5744668") |
||||
# Define actual ICV versions |
||||
if(APPLE) |
||||
set(OPENCV_ICV_PACKAGE_NAME "ippicv_macosx_20151201.tgz") |
||||
set(OPENCV_ICV_PACKAGE_HASH "4ff1fde9a7cfdfe7250bfcd8334e0f2f") |
||||
set(OPENCV_ICV_PLATFORM "macosx") |
||||
set(OPENCV_ICV_PACKAGE_SUBDIR "/ippicv_osx") |
||||
elseif(UNIX) |
||||
if(ANDROID AND NOT (ANDROID_ABI STREQUAL x86 OR ANDROID_ABI STREQUAL x86_64)) |
||||
return() |
||||
endif() |
||||
set(OPENCV_ICV_PACKAGE_NAME "ippicv_linux_20151201.tgz") |
||||
set(OPENCV_ICV_PACKAGE_HASH "808b791a6eac9ed78d32a7666804320e") |
||||
set(OPENCV_ICV_PLATFORM "linux") |
||||
set(OPENCV_ICV_PACKAGE_SUBDIR "/ippicv_lnx") |
||||
elseif(WIN32 AND NOT ARM) |
||||
set(OPENCV_ICV_PACKAGE_NAME "ippicv_windows_20151201.zip") |
||||
set(OPENCV_ICV_PACKAGE_HASH "04e81ce5d0e329c3fbc606ae32cad44d") |
||||
set(OPENCV_ICV_PLATFORM "windows") |
||||
set(OPENCV_ICV_PACKAGE_SUBDIR "/ippicv_win") |
||||
else() |
||||
return() # Not supported |
||||
endif() |
||||
|
||||
set(OPENCV_ICV_UNPACK_PATH "${CMAKE_BINARY_DIR}/3rdparty/ippicv") |
||||
set(OPENCV_ICV_PATH "${OPENCV_ICV_UNPACK_PATH}${OPENCV_ICV_PACKAGE_SUBDIR}") |
||||
|
||||
if(DEFINED OPENCV_ICV_PACKAGE_DOWNLOADED |
||||
AND OPENCV_ICV_PACKAGE_DOWNLOADED STREQUAL OPENCV_ICV_PACKAGE_HASH |
||||
AND EXISTS ${OPENCV_ICV_PATH}) |
||||
# Package has been downloaded and checked by the previous build |
||||
set(OPENCV_ICV_PATH "${OPENCV_ICV_PATH}" PARENT_SCOPE) |
||||
return() |
||||
else() |
||||
if(EXISTS ${OPENCV_ICV_UNPACK_PATH}) |
||||
message(STATUS "ICV: Removing previous unpacked package: ${OPENCV_ICV_UNPACK_PATH}") |
||||
file(REMOVE_RECURSE ${OPENCV_ICV_UNPACK_PATH}) |
||||
endif() |
||||
endif() |
||||
unset(OPENCV_ICV_PACKAGE_DOWNLOADED CACHE) |
||||
|
||||
set(OPENCV_ICV_PACKAGE_ARCHIVE "${CMAKE_CURRENT_LIST_DIR}/downloads/${OPENCV_ICV_PLATFORM}-${OPENCV_ICV_PACKAGE_HASH}/${OPENCV_ICV_PACKAGE_NAME}") |
||||
get_filename_component(OPENCV_ICV_PACKAGE_ARCHIVE_DIR "${OPENCV_ICV_PACKAGE_ARCHIVE}" PATH) |
||||
if(EXISTS "${OPENCV_ICV_PACKAGE_ARCHIVE}") |
||||
file(MD5 "${OPENCV_ICV_PACKAGE_ARCHIVE}" archive_md5) |
||||
if(NOT archive_md5 STREQUAL OPENCV_ICV_PACKAGE_HASH) |
||||
message(WARNING "ICV: Local copy of ICV package has invalid MD5 hash: ${archive_md5} (expected: ${OPENCV_ICV_PACKAGE_HASH})") |
||||
file(REMOVE "${OPENCV_ICV_PACKAGE_ARCHIVE}") |
||||
file(REMOVE_RECURSE "${OPENCV_ICV_PACKAGE_ARCHIVE_DIR}") |
||||
endif() |
||||
endif() |
||||
|
||||
if(NOT EXISTS "${OPENCV_ICV_PACKAGE_ARCHIVE}") |
||||
if(NOT DEFINED OPENCV_ICV_URL) |
||||
if(DEFINED ENV{OPENCV_ICV_URL}) |
||||
set(OPENCV_ICV_URL $ENV{OPENCV_ICV_URL}) |
||||
else() |
||||
set(OPENCV_ICV_URL "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${IPPICV_BINARIES_COMMIT}/ippicv") |
||||
endif() |
||||
endif() |
||||
|
||||
file(MAKE_DIRECTORY ${OPENCV_ICV_PACKAGE_ARCHIVE_DIR}) |
||||
message(STATUS "ICV: Downloading ${OPENCV_ICV_PACKAGE_NAME}...") |
||||
file(DOWNLOAD "${OPENCV_ICV_URL}/${OPENCV_ICV_PACKAGE_NAME}" "${OPENCV_ICV_PACKAGE_ARCHIVE}" |
||||
TIMEOUT 600 STATUS __status |
||||
EXPECTED_MD5 ${OPENCV_ICV_PACKAGE_HASH}) |
||||
if(NOT __status EQUAL 0) |
||||
message(FATAL_ERROR "ICV: Failed to download ICV package: ${OPENCV_ICV_PACKAGE_NAME}. Status=${__status}") |
||||
else() |
||||
# Don't remove this code, because EXPECTED_MD5 parameter doesn't fail "file(DOWNLOAD)" step |
||||
# on wrong hash |
||||
file(MD5 "${OPENCV_ICV_PACKAGE_ARCHIVE}" archive_md5) |
||||
if(NOT archive_md5 STREQUAL OPENCV_ICV_PACKAGE_HASH) |
||||
message(FATAL_ERROR "ICV: Downloaded copy of ICV package has invalid MD5 hash: ${archive_md5} (expected: ${OPENCV_ICV_PACKAGE_HASH})") |
||||
endif() |
||||
endif() |
||||
endif() |
||||
|
||||
ocv_assert(EXISTS "${OPENCV_ICV_PACKAGE_ARCHIVE}") |
||||
ocv_assert(NOT EXISTS "${OPENCV_ICV_UNPACK_PATH}") |
||||
file(MAKE_DIRECTORY ${OPENCV_ICV_UNPACK_PATH}) |
||||
ocv_assert(EXISTS "${OPENCV_ICV_UNPACK_PATH}") |
||||
|
||||
message(STATUS "ICV: Unpacking ${OPENCV_ICV_PACKAGE_NAME} to ${OPENCV_ICV_UNPACK_PATH}...") |
||||
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xz "${OPENCV_ICV_PACKAGE_ARCHIVE}" |
||||
WORKING_DIRECTORY "${OPENCV_ICV_UNPACK_PATH}" |
||||
RESULT_VARIABLE __result) |
||||
|
||||
if(NOT __result EQUAL 0) |
||||
message(FATAL_ERROR "ICV: Failed to unpack ICV package from ${OPENCV_ICV_PACKAGE_ARCHIVE} to ${OPENCV_ICV_UNPACK_PATH} with error ${__result}") |
||||
endif() |
||||
|
||||
ocv_assert(EXISTS "${OPENCV_ICV_PATH}") |
||||
|
||||
set(OPENCV_ICV_PACKAGE_DOWNLOADED "${OPENCV_ICV_PACKAGE_HASH}" CACHE INTERNAL "ICV package hash") |
||||
|
||||
message(STATUS "ICV: Package successfully downloaded") |
||||
set(OPENCV_ICV_PATH "${OPENCV_ICV_PATH}" PARENT_SCOPE) |
||||
endfunction() |
||||
|
||||
_icv_downloader() |
||||
@ -0,0 +1,40 @@ |
||||
function(download_ippicv root_var) |
||||
set(${root_var} "" PARENT_SCOPE) |
||||
|
||||
# Commit SHA in the opencv_3rdparty repo |
||||
set(IPPICV_COMMIT "81a676001ca8075ada498583e4166079e5744668") |
||||
# Define actual ICV versions |
||||
if(APPLE) |
||||
set(OPENCV_ICV_NAME "ippicv_macosx_20151201.tgz") |
||||
set(OPENCV_ICV_HASH "4ff1fde9a7cfdfe7250bfcd8334e0f2f") |
||||
set(OPENCV_ICV_PLATFORM "macosx") |
||||
set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_osx") |
||||
elseif((UNIX AND NOT ANDROID) OR (UNIX AND ANDROID_ABI MATCHES "x86")) |
||||
set(OPENCV_ICV_NAME "ippicv_linux_20151201.tgz") |
||||
set(OPENCV_ICV_HASH "808b791a6eac9ed78d32a7666804320e") |
||||
set(OPENCV_ICV_PLATFORM "linux") |
||||
set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_lnx") |
||||
elseif(WIN32 AND NOT ARM) |
||||
set(OPENCV_ICV_NAME "ippicv_windows_20151201.zip") |
||||
set(OPENCV_ICV_HASH "04e81ce5d0e329c3fbc606ae32cad44d") |
||||
set(OPENCV_ICV_PLATFORM "windows") |
||||
set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_win") |
||||
else() |
||||
return() |
||||
endif() |
||||
|
||||
set(THE_ROOT "${OpenCV_BINARY_DIR}/3rdparty/ippicv") |
||||
ocv_download(FILENAME ${OPENCV_ICV_NAME} |
||||
HASH ${OPENCV_ICV_HASH} |
||||
URL |
||||
"${OPENCV_IPPICV_URL}" |
||||
"$ENV{OPENCV_IPPICV_URL}" |
||||
"https://raw.githubusercontent.com/opencv/opencv_3rdparty/${IPPICV_COMMIT}/ippicv/" |
||||
DESTINATION_DIR "${THE_ROOT}" |
||||
STATUS res |
||||
UNPACK RELATIVE_URL) |
||||
|
||||
if(res) |
||||
set(${root_var} "${THE_ROOT}/${OPENCV_ICV_PACKAGE_SUBDIR}" PARENT_SCOPE) |
||||
endif() |
||||
endfunction() |
||||
@ -0,0 +1,133 @@ |
||||
# |
||||
# Download and optionally unpack a file |
||||
# |
||||
# ocv_download(FILENAME p HASH h URL u1 [u2 ...] DESTINATION_DIR d [STATUS s] [UNPACK] [RELATIVE]) |
||||
# FILENAME - filename |
||||
# HASH - MD5 hash |
||||
# URL - full download url (first nonempty value will be chosen) |
||||
# DESTINATION_DIR - file will be copied to this directory |
||||
# STATUS - passed variable will be updated in parent scope, |
||||
# function will not fail the build in case of download problem if this option is provided, |
||||
# but will fail in case when other operations (copy, remove, etc.) failed |
||||
# UNPACK - downloaded file will be unpacked to DESTINATION_DIR |
||||
# RELATIVE_URL - if set, then URL is treated as a base, and FILENAME will be appended to it |
||||
# Note: uses OPENCV_DOWNLOAD_PATH folder as cache, default is <opencv>/.cache |
||||
|
||||
set(OPENCV_DOWNLOAD_PATH "${OpenCV_SOURCE_DIR}/.cache" CACHE PATH "Cache directory for downloaded files") |
||||
set(OPENCV_DOWNLOAD_LOG "${OpenCV_BINARY_DIR}/CMakeDownloadLog.txt") |
||||
|
||||
# Init download cache directory and log file |
||||
if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}") |
||||
file(MAKE_DIRECTORY ${OPENCV_DOWNLOAD_PATH}) |
||||
endif() |
||||
file(WRITE "${OPENCV_DOWNLOAD_LOG}" "use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n") |
||||
|
||||
|
||||
function(ocv_download) |
||||
cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;STATUS" "URL" ${ARGN}) |
||||
|
||||
ocv_assert(DEFINED DL_FILENAME) |
||||
ocv_assert(DEFINED DL_HASH) |
||||
ocv_assert(DEFINED DL_URL) |
||||
ocv_assert(DEFINED DL_DESTINATION_DIR) |
||||
|
||||
if(DEFINED DL_STATUS) |
||||
set(${DL_STATUS} TRUE PARENT_SCOPE) |
||||
endif() |
||||
|
||||
# Select first non-empty url |
||||
foreach(url ${DL_URL}) |
||||
if(url) |
||||
set(DL_URL "${url}") |
||||
break() |
||||
endif() |
||||
endforeach() |
||||
|
||||
# Append filename to url if needed |
||||
if(DL_RELATIVE_URL) |
||||
set(DL_URL "${DL_URL}${DL_FILENAME}") |
||||
endif() |
||||
|
||||
set(mode "copy") |
||||
if(DL_UNPACK) |
||||
set(mode "unpack") |
||||
endif() |
||||
|
||||
# Log all calls to file |
||||
file(APPEND "${OPENCV_DOWNLOAD_LOG}" "do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"\n") |
||||
# ... and to console |
||||
message(STATUS "Download: ${DL_FILENAME}") |
||||
|
||||
# Copy mode: check if copy destination exists and is correct |
||||
if(NOT DL_UNPACK) |
||||
set(COPY_DESTINATION "${DL_DESTINATION_DIR}/${DL_FILENAME}") |
||||
if(EXISTS "${COPY_DESTINATION}") |
||||
file(MD5 "${COPY_DESTINATION}" target_md5) |
||||
if(target_md5 STREQUAL DL_HASH) |
||||
return() |
||||
endif() |
||||
endif() |
||||
endif() |
||||
|
||||
# Check cache first |
||||
set(CACHE_CANDIDATE "${OPENCV_DOWNLOAD_PATH}/${DL_HASH}-${DL_FILENAME}") |
||||
if(EXISTS "${CACHE_CANDIDATE}") |
||||
file(MD5 "${CACHE_CANDIDATE}" target_md5) |
||||
if(NOT target_md5 STREQUAL DL_HASH) |
||||
file(REMOVE ${CACHE_CANDIDATE}) |
||||
endif() |
||||
endif() |
||||
|
||||
# Download |
||||
if(NOT EXISTS "${CACHE_CANDIDATE}") |
||||
file(DOWNLOAD "${DL_URL}" "${CACHE_CANDIDATE}" |
||||
INACTIVITY_TIMEOUT 60 |
||||
TIMEOUT 600 |
||||
STATUS status |
||||
LOG log) |
||||
string(REPLACE "\n" "\n# " log "# ${log}") |
||||
file(APPEND "${OPENCV_DOWNLOAD_LOG}" "${log}\n\n") |
||||
if(NOT status EQUAL 0) |
||||
set(msg_level FATAL_ERROR) |
||||
if(DEFINED DL_STATUS) |
||||
set(${DL_STATUS} FALSE PARENT_SCOPE) |
||||
set(msg_level WARNING) |
||||
endif() |
||||
message(${msg_level} "Download failed: ${status}") |
||||
return() |
||||
endif() |
||||
|
||||
# Don't remove this code, because EXPECTED_MD5 parameter doesn't fail "file(DOWNLOAD)" step on wrong hash |
||||
file(MD5 "${CACHE_CANDIDATE}" target_md5) |
||||
if(NOT target_md5 STREQUAL DL_HASH) |
||||
set(msg_level FATAL_ERROR) |
||||
if(DEFINED DL_STATUS) |
||||
set(${DL_STATUS} FALSE PARENT_SCOPE) |
||||
set(msg_level WARNING) |
||||
endif() |
||||
message(${msg_level} "Hash mismatch: ${target_md5}") |
||||
return() |
||||
endif() |
||||
endif() |
||||
|
||||
# Unpack or copy |
||||
if(DL_UNPACK) |
||||
if(EXISTS "${DL_DESTINATION_DIR}") |
||||
file(REMOVE_RECURSE "${DL_DESTINATION_DIR}") |
||||
endif() |
||||
file(MAKE_DIRECTORY "${DL_DESTINATION_DIR}") |
||||
execute_process(COMMAND "${CMAKE_COMMAND}" -E tar xz "${CACHE_CANDIDATE}" |
||||
WORKING_DIRECTORY "${DL_DESTINATION_DIR}" |
||||
RESULT_VARIABLE res) |
||||
if(NOT res EQUAL 0) |
||||
message(FATAL_ERROR "Unpack failed: ${res}") |
||||
endif() |
||||
else() |
||||
execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CACHE_CANDIDATE}" "${COPY_DESTINATION}" |
||||
RESULT_VARIABLE res) |
||||
if(NOT res EQUAL 0) |
||||
message(FATAL_ERROR "Copy failed: ${res}") |
||||
endif() |
||||
endif() |
||||
|
||||
endfunction() |
||||
Loading…
Reference in new issue