|
|
|
@ -469,11 +469,9 @@ class BaseTrainer: |
|
|
|
|
|
|
|
|
|
if RANK in {-1, 0}: |
|
|
|
|
# Do final val with best.pt |
|
|
|
|
epochs = epoch - self.start_epoch + 1 # total training epochs |
|
|
|
|
seconds = time.time() - self.train_time_start # total training seconds |
|
|
|
|
LOGGER.info(f"\n{epochs} epochs completed in {seconds / 3600:.3f} hours.") |
|
|
|
|
seconds = time.time() - self.train_time_start |
|
|
|
|
LOGGER.info(f"\n{epoch - self.start_epoch + 1} epochs completed in {seconds / 3600:.3f} hours.") |
|
|
|
|
self.final_eval() |
|
|
|
|
self.validator.metrics.training = {"epochs": epochs, "seconds": seconds} # add training speed |
|
|
|
|
if self.args.plots: |
|
|
|
|
self.plot_metrics() |
|
|
|
|
self.run_callbacks("on_train_end") |
|
|
|
@ -504,7 +502,7 @@ class BaseTrainer: |
|
|
|
|
"""Read results.csv into a dict using pandas.""" |
|
|
|
|
import pandas as pd # scope for faster 'import ultralytics' |
|
|
|
|
|
|
|
|
|
return {k.strip(): v for k, v in pd.read_csv(self.csv).to_dict(orient="list").items()} |
|
|
|
|
return pd.read_csv(self.csv).to_dict(orient="list") |
|
|
|
|
|
|
|
|
|
def save_model(self): |
|
|
|
|
"""Save model training checkpoints with additional metadata.""" |
|
|
|
@ -654,10 +652,11 @@ class BaseTrainer: |
|
|
|
|
def save_metrics(self, metrics): |
|
|
|
|
"""Saves training metrics to a CSV file.""" |
|
|
|
|
keys, vals = list(metrics.keys()), list(metrics.values()) |
|
|
|
|
n = len(metrics) + 1 # number of cols |
|
|
|
|
s = "" if self.csv.exists() else (("%23s," * n % tuple(["epoch"] + keys)).rstrip(",") + "\n") # header |
|
|
|
|
n = len(metrics) + 2 # number of cols |
|
|
|
|
s = "" if self.csv.exists() else (("%s," * n % tuple(["epoch", "time"] + keys)).rstrip(",") + "\n") # header |
|
|
|
|
t = time.time() - self.train_time_start |
|
|
|
|
with open(self.csv, "a") as f: |
|
|
|
|
f.write(s + ("%23.5g," * n % tuple([self.epoch + 1] + vals)).rstrip(",") + "\n") |
|
|
|
|
f.write(s + ("%.6g," * n % tuple([self.epoch + 1, t] + vals)).rstrip(",") + "\n") |
|
|
|
|
|
|
|
|
|
def plot_metrics(self): |
|
|
|
|
"""Plot and display metrics visually.""" |
|
|
|
|