You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
105 lines
3.5 KiB
105 lines
3.5 KiB
# Ultralytics YOLO 🚀, AGPL-3.0 license |
|
|
|
import json |
|
from time import time |
|
|
|
from ultralytics.hub.utils import HUB_WEB_ROOT, PREFIX, events |
|
from ultralytics.utils import LOGGER, SETTINGS |
|
|
|
|
|
def on_pretrain_routine_end(trainer): |
|
"""Logs info before starting timer for upload rate limit.""" |
|
session = getattr(trainer, "hub_session", None) |
|
if session: |
|
# Start timer for upload rate limit |
|
session.timers = {"metrics": time(), "ckpt": time()} # start timer on session.rate_limit |
|
|
|
|
|
def on_fit_epoch_end(trainer): |
|
"""Uploads training progress metrics at the end of each epoch.""" |
|
session = getattr(trainer, "hub_session", None) |
|
if session: |
|
# Upload metrics after val end |
|
all_plots = { |
|
**trainer.label_loss_items(trainer.tloss, prefix="train"), |
|
**trainer.metrics, |
|
} |
|
if trainer.epoch == 0: |
|
from ultralytics.utils.torch_utils import model_info_for_loggers |
|
|
|
all_plots = {**all_plots, **model_info_for_loggers(trainer)} |
|
|
|
session.metrics_queue[trainer.epoch] = json.dumps(all_plots) |
|
|
|
# If any metrics fail to upload, add them to the queue to attempt uploading again. |
|
if session.metrics_upload_failed_queue: |
|
session.metrics_queue.update(session.metrics_upload_failed_queue) |
|
|
|
if time() - session.timers["metrics"] > session.rate_limits["metrics"]: |
|
session.upload_metrics() |
|
session.timers["metrics"] = time() # reset timer |
|
session.metrics_queue = {} # reset queue |
|
|
|
|
|
def on_model_save(trainer): |
|
"""Saves checkpoints to Ultralytics HUB with rate limiting.""" |
|
session = getattr(trainer, "hub_session", None) |
|
if session: |
|
# Upload checkpoints with rate limiting |
|
is_best = trainer.best_fitness == trainer.fitness |
|
if time() - session.timers["ckpt"] > session.rate_limits["ckpt"]: |
|
LOGGER.info(f"{PREFIX}Uploading checkpoint {HUB_WEB_ROOT}/models/{session.model.id}") |
|
session.upload_model(trainer.epoch, trainer.last, is_best) |
|
session.timers["ckpt"] = time() # reset timer |
|
|
|
|
|
def on_train_end(trainer): |
|
"""Upload final model and metrics to Ultralytics HUB at the end of training.""" |
|
session = getattr(trainer, "hub_session", None) |
|
if session: |
|
# Upload final model and metrics with exponential standoff |
|
LOGGER.info(f"{PREFIX}Syncing final model...") |
|
session.upload_model( |
|
trainer.epoch, |
|
trainer.best, |
|
map=trainer.metrics.get("metrics/mAP50-95(B)", 0), |
|
final=True, |
|
) |
|
session.alive = False # stop heartbeats |
|
LOGGER.info(f"{PREFIX}Done ✅\n" f"{PREFIX}View model at {session.model_url} 🚀") |
|
|
|
|
|
def on_train_start(trainer): |
|
"""Run events on train start.""" |
|
events(trainer.args) |
|
|
|
|
|
def on_val_start(validator): |
|
"""Runs events on validation start.""" |
|
events(validator.args) |
|
|
|
|
|
def on_predict_start(predictor): |
|
"""Run events on predict start.""" |
|
events(predictor.args) |
|
|
|
|
|
def on_export_start(exporter): |
|
"""Run events on export start.""" |
|
events(exporter.args) |
|
|
|
|
|
callbacks = ( |
|
{ |
|
"on_pretrain_routine_end": on_pretrain_routine_end, |
|
"on_fit_epoch_end": on_fit_epoch_end, |
|
"on_model_save": on_model_save, |
|
"on_train_end": on_train_end, |
|
"on_train_start": on_train_start, |
|
"on_val_start": on_val_start, |
|
"on_predict_start": on_predict_start, |
|
"on_export_start": on_export_start, |
|
} |
|
if SETTINGS["hub"] is True |
|
else {} |
|
) # verify enabled
|
|
|