|
|
|
@ -1,5 +1,6 @@ |
|
|
|
|
# Ultralytics YOLO 🚀, AGPL-3.0 license |
|
|
|
|
|
|
|
|
|
import shutil |
|
|
|
|
import threading |
|
|
|
|
import time |
|
|
|
|
from http import HTTPStatus |
|
|
|
@ -344,23 +345,34 @@ class HUBTrainingSession: |
|
|
|
|
map (float): Mean average precision of the model. |
|
|
|
|
final (bool): Indicates if the model is the final model after training. |
|
|
|
|
""" |
|
|
|
|
if Path(weights).is_file(): |
|
|
|
|
progress_total = Path(weights).stat().st_size if final else None # Only show progress if final |
|
|
|
|
self.request_queue( |
|
|
|
|
self.model.upload_model, |
|
|
|
|
epoch=epoch, |
|
|
|
|
weights=weights, |
|
|
|
|
is_best=is_best, |
|
|
|
|
map=map, |
|
|
|
|
final=final, |
|
|
|
|
retry=10, |
|
|
|
|
timeout=3600, |
|
|
|
|
thread=not final, |
|
|
|
|
progress_total=progress_total, |
|
|
|
|
stream_response=True, |
|
|
|
|
) |
|
|
|
|
else: |
|
|
|
|
LOGGER.warning(f"{PREFIX}WARNING ⚠️ Model upload issue. Missing model {weights}.") |
|
|
|
|
weights = Path(weights) |
|
|
|
|
if not weights.is_file(): |
|
|
|
|
last = weights.with_name("last" + weights.suffix) |
|
|
|
|
if final and last.is_file(): |
|
|
|
|
LOGGER.warning( |
|
|
|
|
f"{PREFIX} ARNING ⚠️ Model 'best.pt' not found, copying 'last.pt' to 'best.pt' and uploading. " |
|
|
|
|
"This often happens when resuming training in transient environments like Google Colab. " |
|
|
|
|
"For more reliable training, consider using Ultralytics HUB Cloud. " |
|
|
|
|
"Learn more at https://docs.ultralytics.com/hub/cloud-training/." |
|
|
|
|
) |
|
|
|
|
shutil.copy(last, weights) # copy last.pt to best.pt |
|
|
|
|
else: |
|
|
|
|
LOGGER.warning(f"{PREFIX} WARNING ⚠️ Model upload issue. Missing model {weights}.") |
|
|
|
|
return |
|
|
|
|
|
|
|
|
|
self.request_queue( |
|
|
|
|
self.model.upload_model, |
|
|
|
|
epoch=epoch, |
|
|
|
|
weights=str(weights), |
|
|
|
|
is_best=is_best, |
|
|
|
|
map=map, |
|
|
|
|
final=final, |
|
|
|
|
retry=10, |
|
|
|
|
timeout=3600, |
|
|
|
|
thread=not final, |
|
|
|
|
progress_total=weights.stat().st_size if final else None, # only show progress if final |
|
|
|
|
stream_response=True, |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
@staticmethod |
|
|
|
|
def _show_upload_progress(content_length: int, response: requests.Response) -> None: |
|
|
|
|