From f89bfd7e0157fff46d87ed567f7bb0e586300f7d Mon Sep 17 00:00:00 2001
From: Glenn Jocher <glenn.jocher@ultralytics.com>
Date: Wed, 29 Nov 2023 21:17:22 +0100
Subject: [PATCH] Pre-download `onnx2tf` calib file to preempt S3 issues
 (#6669)

Signed-off-by: Glenn Jocher <glenn.jocher@ultralytics.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Abirami Vina <abirami.vina@gmail.com>
---
 .github/workflows/ci.yaml       |  1 -
 docs/en/integrations/clearml.md |  2 +-
 ultralytics/engine/exporter.py  |  5 ++
 ultralytics/utils/downloads.py  | 89 +++++++++++++++++++++++++++------
 4 files changed, 81 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 994522bd..3e8f07ef 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -110,7 +110,6 @@ jobs:
           pip install -e ".[export]" coverage --extra-index-url https://download.pytorch.org/whl/cpu
           # Fix SavedModel issue "partially initialized module 'jax' has no attribute 'version' (most likely due to a circular import)" in https://github.com/google/jax/discussions/14036
           # pip install -U 'jax!=0.4.15' 'jaxlib!=0.4.15'
-          # yolo settings tensorboard=False
           yolo export format=tflite imgsz=32 || true
       - name: Check environment
         run: |
diff --git a/docs/en/integrations/clearml.md b/docs/en/integrations/clearml.md
index 1c65d4ab..43b28a9b 100644
--- a/docs/en/integrations/clearml.md
+++ b/docs/en/integrations/clearml.md
@@ -142,7 +142,7 @@ For a visual walkthrough of what the ClearML Results Page looks like, watch the
 
 <p align="center">
   <br>
-  <iframe width="720" height="405" src="https://youtu.be/iLcC7m3bCes"
+  <iframe width="720" height="405" src="https://www.youtube.com/embed/iLcC7m3bCes?si=oSEAoZbrg8inCg_2"
     title="YouTube video player" frameborder="0"
     allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share"
     allowfullscreen>
diff --git a/ultralytics/engine/exporter.py b/ultralytics/engine/exporter.py
index 0a787f8b..feaaf98c 100644
--- a/ultralytics/engine/exporter.py
+++ b/ultralytics/engine/exporter.py
@@ -663,6 +663,11 @@ class Exporter:
             import shutil
             shutil.rmtree(f)  # delete output folder
 
+        # Pre-download calibration file to fix https://github.com/PINTO0309/onnx2tf/issues/545
+        onnx2tf_file = Path('calibration_image_sample_data_20x128x128x3_float32.npy')
+        if not onnx2tf_file.exists():
+            attempt_download_asset(f'{onnx2tf_file}.zip', unzip=True, delete=True)
+
         # Export to ONNX
         self.args.simplify = True
         f_onnx, _ = self.export_onnx()
diff --git a/ultralytics/utils/downloads.py b/ultralytics/utils/downloads.py
index e46ff1f6..98bf1ee5 100644
--- a/ultralytics/utils/downloads.py
+++ b/ultralytics/utils/downloads.py
@@ -28,7 +28,23 @@ GITHUB_ASSETS_STEMS = [Path(k).stem for k in GITHUB_ASSETS_NAMES]
 
 
 def is_url(url, check=True):
-    """Check if string is URL and check if URL exists."""
+    """
+    Validates if the given string is a URL and optionally checks if the URL exists online.
+
+    Args:
+        url (str): The string to be validated as a URL.
+        check (bool, optional): If True, performs an additional check to see if the URL exists online.
+            Defaults to True.
+
+    Returns:
+        bool: Returns True if the string is a valid URL. If 'check' is True, also returns True if the URL exists online.
+            Returns False otherwise.
+
+    Example:
+        ```python
+        valid = is_url("https://www.example.com")
+        ```
+    """
     with contextlib.suppress(Exception):
         url = str(url)
         result = parse.urlparse(url)
@@ -59,7 +75,6 @@ def delete_dsstore(path, files_to_delete=('.DS_Store', '__MACOSX')):
         ".DS_store" files are created by the Apple operating system and contain metadata about folders and files. They
         are hidden system files and can cause issues when transferring files between different operating systems.
     """
-    # Delete Apple .DS_store files
     for file in files_to_delete:
         matches = list(Path(path).rglob(file))
         LOGGER.info(f'Deleting {file} files: {matches}')
@@ -275,9 +290,7 @@ def safe_download(url,
         path = safe_download(link)
         ```
     """
-
-    # Check if the URL is a Google Drive link
-    gdrive = url.startswith('https://drive.google.com/')
+    gdrive = url.startswith('https://drive.google.com/')  # check if the URL is a Google Drive link
     if gdrive:
         url, file = get_google_drive_file_info(url)
 
@@ -337,7 +350,24 @@ def safe_download(url,
 
 
 def get_github_assets(repo='ultralytics/assets', version='latest', retry=False):
-    """Return GitHub repo tag and assets (i.e. ['yolov8n.pt', 'yolov8s.pt', ...])."""
+    """
+    Retrieve the specified version's tag and assets from a GitHub repository. If the version is not specified, the
+    function fetches the latest release assets.
+
+    Args:
+        repo (str, optional): The GitHub repository in the format 'owner/repo'. Defaults to 'ultralytics/assets'.
+        version (str, optional): The release version to fetch assets from. Defaults to 'latest'.
+        retry (bool, optional): Flag to retry the request in case of a failure. Defaults to False.
+
+    Returns:
+        tuple: A tuple containing the release tag and a list of asset names.
+
+    Example:
+        ```python
+        tag, assets = get_github_assets(repo='ultralytics/assets', version='latest')
+        ```
+    """
+
     if version != 'latest':
         version = f'tags/{version}'  # i.e. tags/v6.2
     url = f'https://api.github.com/repos/{repo}/releases/{version}'
@@ -348,14 +378,27 @@ def get_github_assets(repo='ultralytics/assets', version='latest', retry=False):
         LOGGER.warning(f'⚠️ GitHub assets check failure for {url}: {r.status_code} {r.reason}')
         return '', []
     data = r.json()
-    return data['tag_name'], [x['name'] for x in data['assets']]  # tag, assets
+    return data['tag_name'], [x['name'] for x in data['assets']]  # tag, assets i.e. ['yolov8n.pt', 'yolov8s.pt', ...]
 
 
-def attempt_download_asset(file, repo='ultralytics/assets', release='v0.0.0'):
+def attempt_download_asset(file, repo='ultralytics/assets', release='v0.0.0', **kwargs):
     """
-    Attempt file download from GitHub release assets if not found locally.
+    Attempt to download a file from GitHub release assets if it is not found locally. The function checks for the file
+    locally first, then tries to download it from the specified GitHub repository release.
+
+    Args:
+        file (str | Path): The filename or file path to be downloaded.
+        repo (str, optional): The GitHub repository in the format 'owner/repo'. Defaults to 'ultralytics/assets'.
+        release (str, optional): The specific release version to be downloaded. Defaults to 'v0.0.0'.
+        **kwargs: Additional keyword arguments for the download process.
 
-    release = 'latest', 'v6.2', etc.
+    Returns:
+        str: The path to the downloaded file.
+
+    Example:
+        ```python
+        file_path = attempt_download_asset('yolov5s.pt', repo='ultralytics/assets', release='latest')
+        ```
     """
     from ultralytics.utils import SETTINGS  # scoped for circular import
 
@@ -370,29 +413,47 @@ def attempt_download_asset(file, repo='ultralytics/assets', release='v0.0.0'):
     else:
         # URL specified
         name = Path(parse.unquote(str(file))).name  # decode '%2F' to '/' etc.
+        download_url = f'https://github.com/{repo}/releases/download'
         if str(file).startswith(('http:/', 'https:/')):  # download
             url = str(file).replace(':/', '://')  # Pathlib turns :// -> :/
             file = url2file(name)  # parse authentication https://url.com/file.txt?auth...
             if Path(file).is_file():
                 LOGGER.info(f'Found {clean_url(url)} locally at {file}')  # file already exists
             else:
-                safe_download(url=url, file=file, min_bytes=1E5)
+                safe_download(url=url, file=file, min_bytes=1E5, **kwargs)
 
         elif repo == GITHUB_ASSETS_REPO and name in GITHUB_ASSETS_NAMES:
-            safe_download(url=f'https://github.com/{repo}/releases/download/{release}/{name}', file=file, min_bytes=1E5)
+            safe_download(url=f'{download_url}/{release}/{name}', file=file, min_bytes=1E5, **kwargs)
 
         else:
             tag, assets = get_github_assets(repo, release)
             if not assets:
                 tag, assets = get_github_assets(repo)  # latest release
             if name in assets:
-                safe_download(url=f'https://github.com/{repo}/releases/download/{tag}/{name}', file=file, min_bytes=1E5)
+                safe_download(url=f'{download_url}/{tag}/{name}', file=file, min_bytes=1E5, **kwargs)
 
         return str(file)
 
 
 def download(url, dir=Path.cwd(), unzip=True, delete=False, curl=False, threads=1, retry=3):
-    """Downloads and unzips files concurrently if threads > 1, else sequentially."""
+    """
+    Downloads files from specified URLs to a given directory. Supports concurrent downloads if multiple threads are
+    specified.
+
+    Args:
+        url (str | list): The URL or list of URLs of the files to be downloaded.
+        dir (Path, optional): The directory where the files will be saved. Defaults to the current working directory.
+        unzip (bool, optional): Flag to unzip the files after downloading. Defaults to True.
+        delete (bool, optional): Flag to delete the zip files after extraction. Defaults to False.
+        curl (bool, optional): Flag to use curl for downloading. Defaults to False.
+        threads (int, optional): Number of threads to use for concurrent downloads. Defaults to 1.
+        retry (int, optional): Number of retries in case of download failure. Defaults to 3.
+
+    Example:
+        ```python
+        download('https://ultralytics.com/assets/example.zip', dir='path/to/dir', unzip=True)
+        ```
+    """
     dir = Path(dir)
     dir.mkdir(parents=True, exist_ok=True)  # make directory
     if threads > 1: