From 2df0345985bbf45748d7a473f4ae4e7609846812 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 1 Aug 2018 15:57:53 +0300 Subject: [PATCH 1/2] cmake: add download helper scripts --- cmake/OpenCVDownload.cmake | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmake/OpenCVDownload.cmake b/cmake/OpenCVDownload.cmake index 7724147d31..cdc47ad2cb 100644 --- a/cmake/OpenCVDownload.cmake +++ b/cmake/OpenCVDownload.cmake @@ -20,16 +20,19 @@ if(DEFINED ENV{OPENCV_DOWNLOAD_PATH}) endif() set(OPENCV_DOWNLOAD_PATH "${OpenCV_SOURCE_DIR}/.cache" CACHE PATH "${HELP_OPENCV_DOWNLOAD_PATH}") set(OPENCV_DOWNLOAD_LOG "${OpenCV_BINARY_DIR}/CMakeDownloadLog.txt") +set(OPENCV_DOWNLOAD_WITH_CURL "${OpenCV_BINARY_DIR}/download_with_curl.sh") +set(OPENCV_DOWNLOAD_WITH_WGET "${OpenCV_BINARY_DIR}/download_with_wget.sh") -# Init download cache directory and log file +# Init download cache directory and log file and helper scripts if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}") file(MAKE_DIRECTORY ${OPENCV_DOWNLOAD_PATH}) endif() if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}/.gitignore") file(WRITE "${OPENCV_DOWNLOAD_PATH}/.gitignore" "*\n") endif() -file(WRITE "${OPENCV_DOWNLOAD_LOG}" "use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n") - +file(WRITE "${OPENCV_DOWNLOAD_LOG}" "#use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n") +file(REMOVE "${OPENCV_DOWNLOAD_WITH_CURL}") +file(REMOVE "${OPENCV_DOWNLOAD_WITH_WGET}") function(ocv_download) cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;ID;STATUS" "URL" ${ARGN}) @@ -103,7 +106,7 @@ function(ocv_download) endif() # Log all calls to file - ocv_download_log("do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"") + ocv_download_log("#do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"") # ... and to console set(__msg_prefix "") if(DL_ID) @@ -191,6 +194,9 @@ function(ocv_download) For details please refer to the download log file: ${OPENCV_DOWNLOAD_LOG} ") + # write helper scripts for failed downloads + file(APPEND "${OPENCV_DOWNLOAD_WITH_CURL}" "curl --output \"${CACHE_CANDIDATE}\" \"${DL_URL}\"\n") + file(APPEND "${OPENCV_DOWNLOAD_WITH_WGET}" "wget -O \"${CACHE_CANDIDATE}\" \"${DL_URL}\"\n") return() endif() From df9efbbd1f9faa8f16be14e09e52be541b1324f0 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Wed, 1 Aug 2018 15:58:20 +0300 Subject: [PATCH 2/2] Install data for samples to correct directories, do not download face_detector models in cmake --- samples/CMakeLists.txt | 4 +- samples/dnn/CMakeLists.txt | 26 ------- samples/dnn/face_detector/download_weights.py | 74 +++++++++++++++++++ samples/dnn/face_detector/weights.meta4 | 13 ++++ 4 files changed, 88 insertions(+), 29 deletions(-) create mode 100755 samples/dnn/face_detector/download_weights.py create mode 100644 samples/dnn/face_detector/weights.meta4 diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 45dae6a5ad..5daa941df0 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -70,9 +70,7 @@ endif() ocv_install_example_src("." CMakeLists.txt) if(INSTALL_C_EXAMPLES) - install(DIRECTORY data - DESTINATION "${OPENCV_SAMPLES_SRC_INSTALL_PATH}/data" - COMPONENT samples_data) + install(DIRECTORY data DESTINATION "${OPENCV_SAMPLES_SRC_INSTALL_PATH}" COMPONENT samples_data) endif() else() diff --git a/samples/dnn/CMakeLists.txt b/samples/dnn/CMakeLists.txt index 0df76517a5..4af6d40928 100644 --- a/samples/dnn/CMakeLists.txt +++ b/samples/dnn/CMakeLists.txt @@ -13,32 +13,6 @@ if(NOT BUILD_EXAMPLES OR NOT OCV_DEPENDENCIES_FOUND) return() endif() -function(download_net name commit hash) - set(DNN_FACE_DETECTOR_MODEL_DOWNLOAD_DIR "${CMAKE_CURRENT_LIST_DIR}/face_detector") - if(COMMAND ocv_download) - ocv_download(FILENAME ${name} - HASH ${hash} - URL - "$ENV{OPENCV_DNN_MODELS_URL}" - "${OPENCV_DNN_MODELS_URL}" - "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${commit}/" - DESTINATION_DIR ${DNN_FACE_DETECTOR_MODEL_DOWNLOAD_DIR} - ID DNN_FACE_DETECTOR - RELATIVE_URL - STATUS res) - endif() -endfunction() - -# Model branch name: dnn_samples_face_detector_20180205_fp16 -download_net("res10_300x300_ssd_iter_140000_fp16.caffemodel" - "19512576c112aa2c7b6328cb0e8d589a4a90a26d" - "f737f886e33835410c69e3ccfe0720a1") - -# Model branch name: dnn_samples_face_detector_20180220_uint8 -download_net("opencv_face_detector_uint8.pb" - "7b425df276ba2161b8edaab0f0756f4a735d61b9" - "56acf81f55d9b9e96c3347bc65409b9e") - project(dnn_samples) ocv_include_modules_recurse(${OPENCV_DNN_SAMPLES_REQUIRED_DEPS}) file(GLOB_RECURSE dnn_samples RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp) diff --git a/samples/dnn/face_detector/download_weights.py b/samples/dnn/face_detector/download_weights.py new file mode 100755 index 0000000000..f872190d02 --- /dev/null +++ b/samples/dnn/face_detector/download_weights.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +from __future__ import print_function +import hashlib +import time +import sys +import xml.etree.ElementTree as ET +if sys.version_info[0] < 3: + from urllib2 import urlopen +else: + from urllib.request import urlopen + +class HashMismatchException(Exception): + def __init__(self, expected, actual): + Exception.__init__(self) + self.expected = expected + self.actual = actual + def __str__(self): + return 'Hash mismatch: {} vs {}'.format(self.expected, self.actual) + +class MetalinkDownloader(object): + BUFSIZE = 10*1024*1024 + NS = {'ml': 'urn:ietf:params:xml:ns:metalink'} + tick = 0 + + def download(self, metalink_file): + status = True + for file_elem in ET.parse(metalink_file).getroot().findall('ml:file', self.NS): + url = file_elem.find('ml:url', self.NS).text + fname = file_elem.attrib['name'] + hash_sum = file_elem.find('ml:hash', self.NS).text + print('*** {}'.format(fname)) + try: + self.verify(hash_sum, fname) + except Exception as ex: + print(' {}'.format(ex)) + try: + print(' {}'.format(url)) + with open(fname, 'wb') as file_stream: + self.buffered_read(urlopen(url), file_stream.write) + self.verify(hash_sum, fname) + except Exception as ex: + print(' {}'.format(ex)) + print(' FAILURE') + status = False + continue + print(' SUCCESS') + return status + + def print_progress(self, msg, timeout = 0): + if time.time() - self.tick > timeout: + print(msg, end='') + sys.stdout.flush() + self.tick = time.time() + + def buffered_read(self, in_stream, processing): + self.print_progress(' >') + while True: + buf = in_stream.read(self.BUFSIZE) + if not buf: + break + processing(buf) + self.print_progress('>', 5) + print(' done') + + def verify(self, hash_sum, fname): + sha = hashlib.sha1() + with open(fname, 'rb') as file_stream: + self.buffered_read(file_stream, sha.update) + if hash_sum != sha.hexdigest(): + raise HashMismatchException(hash_sum, sha.hexdigest()) + +if __name__ == '__main__': + sys.exit(0 if MetalinkDownloader().download('weights.meta4') else 1) diff --git a/samples/dnn/face_detector/weights.meta4 b/samples/dnn/face_detector/weights.meta4 new file mode 100644 index 0000000000..35d303085b --- /dev/null +++ b/samples/dnn/face_detector/weights.meta4 @@ -0,0 +1,13 @@ + + + + OpenCV face detector FP16 weights + 31fc22bfdd907567a04bb45b7cfad29966caddc1 + https://raw.githubusercontent.com/opencv/opencv_3rdparty/dnn_samples_face_detector_20180205_fp16/res10_300x300_ssd_iter_140000_fp16.caffemodel + + + OpenCV face detector UINT8 weights + 4f2fdf6f231d759d7bbdb94353c5a68690f3d2ae + https://raw.githubusercontent.com/opencv/opencv_3rdparty/dnn_samples_face_detector_20180220_uint8/opencv_face_detector_uint8.pb + +