Merge pull request #12124 from mshabunin:download-improve

cmake/OpenCVDownload.cmake

@@ -20,16 +20,19 @@ if(DEFINED ENV{OPENCV_DOWNLOAD_PATH})
 endif()
 set(OPENCV_DOWNLOAD_PATH "${OpenCV_SOURCE_DIR}/.cache" CACHE PATH "${HELP_OPENCV_DOWNLOAD_PATH}")
 set(OPENCV_DOWNLOAD_LOG "${OpenCV_BINARY_DIR}/CMakeDownloadLog.txt")
+set(OPENCV_DOWNLOAD_WITH_CURL "${OpenCV_BINARY_DIR}/download_with_curl.sh")
+set(OPENCV_DOWNLOAD_WITH_WGET "${OpenCV_BINARY_DIR}/download_with_wget.sh")
 
-# Init download cache directory and log file
+# Init download cache directory and log file and helper scripts
 if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}")
   file(MAKE_DIRECTORY ${OPENCV_DOWNLOAD_PATH})
 endif()
 if(NOT EXISTS "${OPENCV_DOWNLOAD_PATH}/.gitignore")
   file(WRITE "${OPENCV_DOWNLOAD_PATH}/.gitignore" "*\n")
 endif()
-file(WRITE "${OPENCV_DOWNLOAD_LOG}" "use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n")
+file(WRITE "${OPENCV_DOWNLOAD_LOG}" "#use_cache \"${OPENCV_DOWNLOAD_PATH}\"\n")
-
+file(REMOVE "${OPENCV_DOWNLOAD_WITH_CURL}")
+file(REMOVE "${OPENCV_DOWNLOAD_WITH_WGET}")
 
 function(ocv_download)
   cmake_parse_arguments(DL "UNPACK;RELATIVE_URL" "FILENAME;HASH;DESTINATION_DIR;ID;STATUS" "URL" ${ARGN})
@@ -103,7 +106,7 @@ function(ocv_download)
   endif()
 
   # Log all calls to file
-  ocv_download_log("do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"")
+  ocv_download_log("#do_${mode} \"${DL_FILENAME}\" \"${DL_HASH}\" \"${DL_URL}\" \"${DL_DESTINATION_DIR}\"")
   # ... and to console
   set(__msg_prefix "")
   if(DL_ID)
@@ -191,6 +194,9 @@ function(ocv_download)
 For details please refer to the download log file:
 ${OPENCV_DOWNLOAD_LOG}
 ")
+      # write helper scripts for failed downloads
+      file(APPEND "${OPENCV_DOWNLOAD_WITH_CURL}" "curl --output \"${CACHE_CANDIDATE}\" \"${DL_URL}\"\n")
+      file(APPEND "${OPENCV_DOWNLOAD_WITH_WGET}" "wget -O \"${CACHE_CANDIDATE}\" \"${DL_URL}\"\n")
       return()
     endif()
 

samples/CMakeLists.txt

@@ -70,9 +70,7 @@ endif()
 
 ocv_install_example_src("." CMakeLists.txt)
 if(INSTALL_C_EXAMPLES)
-  install(DIRECTORY data
+  install(DIRECTORY data DESTINATION "${OPENCV_SAMPLES_SRC_INSTALL_PATH}" COMPONENT samples_data)
-          DESTINATION "${OPENCV_SAMPLES_SRC_INSTALL_PATH}/data"
-        COMPONENT samples_data)
 endif()
 
 else()

samples/dnn/CMakeLists.txt

@@ -13,32 +13,6 @@ if(NOT BUILD_EXAMPLES OR NOT OCV_DEPENDENCIES_FOUND)
   return()
 endif()
 
-function(download_net name commit hash)
-  set(DNN_FACE_DETECTOR_MODEL_DOWNLOAD_DIR "${CMAKE_CURRENT_LIST_DIR}/face_detector")
-  if(COMMAND ocv_download)
-    ocv_download(FILENAME ${name}
-               HASH ${hash}
-               URL
-                 "$ENV{OPENCV_DNN_MODELS_URL}"
-                 "${OPENCV_DNN_MODELS_URL}"
-                 "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${commit}/"
-               DESTINATION_DIR ${DNN_FACE_DETECTOR_MODEL_DOWNLOAD_DIR}
-               ID DNN_FACE_DETECTOR
-               RELATIVE_URL
-               STATUS res)
-  endif()
-endfunction()
-
-# Model branch name: dnn_samples_face_detector_20180205_fp16
-download_net("res10_300x300_ssd_iter_140000_fp16.caffemodel"
-             "19512576c112aa2c7b6328cb0e8d589a4a90a26d"
-             "f737f886e33835410c69e3ccfe0720a1")
-
-# Model branch name: dnn_samples_face_detector_20180220_uint8
-download_net("opencv_face_detector_uint8.pb"
-             "7b425df276ba2161b8edaab0f0756f4a735d61b9"
-             "56acf81f55d9b9e96c3347bc65409b9e")
-
 project(dnn_samples)
 ocv_include_modules_recurse(${OPENCV_DNN_SAMPLES_REQUIRED_DEPS})
 file(GLOB_RECURSE dnn_samples RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp)

samples/dnn/face_detector/download_weights.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import hashlib
+import time
+import sys
+import xml.etree.ElementTree as ET
+if sys.version_info[0] < 3:
+    from urllib2 import urlopen
+else:
+    from urllib.request import urlopen
+
+class HashMismatchException(Exception):
+    def __init__(self, expected, actual):
+        Exception.__init__(self)
+        self.expected = expected
+        self.actual = actual
+    def __str__(self):
+        return 'Hash mismatch: {} vs {}'.format(self.expected, self.actual)
+
+class MetalinkDownloader(object):
+    BUFSIZE = 10*1024*1024
+    NS = {'ml': 'urn:ietf:params:xml:ns:metalink'}
+    tick = 0
+
+    def download(self, metalink_file):
+        status = True
+        for file_elem in ET.parse(metalink_file).getroot().findall('ml:file', self.NS):
+            url = file_elem.find('ml:url', self.NS).text
+            fname = file_elem.attrib['name']
+            hash_sum = file_elem.find('ml:hash', self.NS).text
+            print('*** {}'.format(fname))
+            try:
+                self.verify(hash_sum, fname)
+            except Exception as ex:
+                print('  {}'.format(ex))
+                try:
+                    print('  {}'.format(url))
+                    with open(fname, 'wb') as file_stream:
+                        self.buffered_read(urlopen(url), file_stream.write)
+                    self.verify(hash_sum, fname)
+                except Exception as ex:
+                    print('  {}'.format(ex))
+                    print('  FAILURE')
+                    status = False
+                    continue
+            print('  SUCCESS')
+        return status
+
+    def print_progress(self, msg, timeout = 0):
+        if time.time() - self.tick > timeout:
+            print(msg, end='')
+            sys.stdout.flush()
+            self.tick = time.time()
+
+    def buffered_read(self, in_stream, processing):
+        self.print_progress('  >')
+        while True:
+            buf = in_stream.read(self.BUFSIZE)
+            if not buf:
+                break
+            processing(buf)
+            self.print_progress('>', 5)
+        print(' done')
+
+    def verify(self, hash_sum, fname):
+        sha = hashlib.sha1()
+        with open(fname, 'rb') as file_stream:
+            self.buffered_read(file_stream, sha.update)
+        if hash_sum != sha.hexdigest():
+            raise HashMismatchException(hash_sum, sha.hexdigest())
+
+if __name__ == '__main__':
+    sys.exit(0 if MetalinkDownloader().download('weights.meta4') else 1)

samples/dnn/face_detector/weights.meta4

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metalink xmlns="urn:ietf:params:xml:ns:metalink">
+    <file name="res10_300x300_ssd_iter_140000_fp16.caffemodel">
+        <identity>OpenCV face detector FP16 weights</identity>
+        <hash type="sha-1">31fc22bfdd907567a04bb45b7cfad29966caddc1</hash>
+        <url>https://raw.githubusercontent.com/opencv/opencv_3rdparty/dnn_samples_face_detector_20180205_fp16/res10_300x300_ssd_iter_140000_fp16.caffemodel</url>
+    </file>
+    <file name="opencv_face_detector_uint8.pb">
+        <identity>OpenCV face detector UINT8 weights</identity>
+        <hash type="sha-1">4f2fdf6f231d759d7bbdb94353c5a68690f3d2ae</hash>
+        <url>https://raw.githubusercontent.com/opencv/opencv_3rdparty/dnn_samples_face_detector_20180220_uint8/opencv_face_detector_uint8.pb</url>
+    </file>
+</metalink>