From 43726d699f1984812f0baf540bfdeb453892ff9b Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sun, 22 Sep 2024 22:38:35 +0200 Subject: [PATCH] `ultralytics 8.2.99` faster `JSONDict` settings (#16427) Signed-off-by: UltralyticsAssistant Co-authored-by: UltralyticsAssistant --- docs/en/quickstart.md | 2 +- tests/test_integrations.py | 2 + ultralytics/__init__.py | 2 +- ultralytics/cfg/__init__.py | 8 +-- ultralytics/data/utils.py | 4 +- ultralytics/hub/__init__.py | 1 - ultralytics/utils/__init__.py | 98 ++++++++++++++++++++--------------- 7 files changed, 65 insertions(+), 52 deletions(-) diff --git a/docs/en/quickstart.md b/docs/en/quickstart.md index a844dbbe70..4a339d9ad7 100644 --- a/docs/en/quickstart.md +++ b/docs/en/quickstart.md @@ -253,7 +253,7 @@ For example, users can load a model, train it, evaluate its performance on a val ## Ultralytics Settings -The Ultralytics library provides a powerful settings management system to enable fine-grained control over your experiments. By making use of the `SettingsManager` housed within the `ultralytics.utils` module, users can readily access and alter their settings. These are stored in a YAML file and can be viewed or modified either directly within the Python environment or via the Command-Line Interface (CLI). +The Ultralytics library provides a powerful settings management system to enable fine-grained control over your experiments. By making use of the `SettingsManager` housed within the `ultralytics.utils` module, users can readily access and alter their settings. These are stored in a JSON file in the environment user configuration directory, and can be viewed or modified directly within the Python environment or via the Command-Line Interface (CLI). ### Inspecting Settings diff --git a/tests/test_integrations.py b/tests/test_integrations.py index aaa8330b37..3a0d1b48a7 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -27,6 +27,7 @@ def test_mlflow(): """Test training with MLflow tracking enabled (see https://mlflow.org/ for details).""" SETTINGS["mlflow"] = True YOLO("yolov8n-cls.yaml").train(data="imagenet10", imgsz=32, epochs=3, plots=False, device="cpu") + SETTINGS["mlflow"] = False @pytest.mark.skipif(True, reason="Test failing in scheduled CI https://github.com/ultralytics/ultralytics/pull/8868") @@ -58,6 +59,7 @@ def test_mlflow_keep_run_active(): YOLO("yolov8n-cls.yaml").train(data="imagenet10", imgsz=32, epochs=1, plots=False, device="cpu") status = mlflow.get_run(run_id=run_id).info.status assert status == "FINISHED", "MLflow run should be ended by default when MLFLOW_KEEP_RUN_ACTIVE is not set" + SETTINGS["mlflow"] = False @pytest.mark.skipif(not check_requirements("tritonclient", install=False), reason="tritonclient[all] not installed") diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index b3e0acfb60..a1754a4875 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics YOLO 🚀, AGPL-3.0 license -__version__ = "8.2.98" +__version__ = "8.2.99" import os diff --git a/ultralytics/cfg/__init__.py b/ultralytics/cfg/__init__.py index 9ae1214ecb..06356e7589 100644 --- a/ultralytics/cfg/__init__.py +++ b/ultralytics/cfg/__init__.py @@ -19,7 +19,7 @@ from ultralytics.utils import ( ROOT, RUNS_DIR, SETTINGS, - SETTINGS_YAML, + SETTINGS_FILE, TESTS_RUNNING, IterableSimpleNamespace, __version__, @@ -532,7 +532,7 @@ def handle_yolo_settings(args: List[str]) -> None: try: if any(args): if args[0] == "reset": - SETTINGS_YAML.unlink() # delete the settings file + SETTINGS_FILE.unlink() # delete the settings file SETTINGS.reset() # create new settings LOGGER.info("Settings reset successfully") # inform the user that settings have been reset else: # save a new setting @@ -540,8 +540,8 @@ def handle_yolo_settings(args: List[str]) -> None: check_dict_alignment(SETTINGS, new) SETTINGS.update(new) - LOGGER.info(f"💡 Learn about settings at {url}") - yaml_print(SETTINGS_YAML) # print the current settings + print(SETTINGS) # print the current settings + LOGGER.info(f"💡 Learn more about Ultralytics Settings at {url}") except Exception as e: LOGGER.warning(f"WARNING ⚠️ settings error: '{e}'. Please see {url} for help.") diff --git a/ultralytics/data/utils.py b/ultralytics/data/utils.py index 600f62e574..e82d8bb759 100644 --- a/ultralytics/data/utils.py +++ b/ultralytics/data/utils.py @@ -22,7 +22,7 @@ from ultralytics.utils import ( LOGGER, NUM_THREADS, ROOT, - SETTINGS_YAML, + SETTINGS_FILE, TQDM, clean_url, colorstr, @@ -324,7 +324,7 @@ def check_det_dataset(dataset, autodownload=True): if s and autodownload: LOGGER.warning(m) else: - m += f"\nNote dataset download directory is '{DATASETS_DIR}'. You can update this in '{SETTINGS_YAML}'" + m += f"\nNote dataset download directory is '{DATASETS_DIR}'. You can update this in '{SETTINGS_FILE}'" raise FileNotFoundError(m) t = time.time() r = None # success diff --git a/ultralytics/hub/__init__.py b/ultralytics/hub/__init__.py index d8d7b4a274..33b0c3748d 100644 --- a/ultralytics/hub/__init__.py +++ b/ultralytics/hub/__init__.py @@ -79,7 +79,6 @@ def logout(): ``` """ SETTINGS["api_key"] = "" - SETTINGS.save() LOGGER.info(f"{PREFIX}logged out ✅. To log in again, use 'yolo hub login'.") diff --git a/ultralytics/utils/__init__.py b/ultralytics/utils/__init__.py index a9ada7cd54..b4cc312d85 100644 --- a/ultralytics/utils/__init__.py +++ b/ultralytics/utils/__init__.py @@ -802,7 +802,7 @@ IS_RASPBERRYPI = is_raspberrypi() GIT_DIR = get_git_dir() IS_GIT_DIR = is_git_dir() USER_CONFIG_DIR = Path(os.getenv("YOLO_CONFIG_DIR") or get_user_config_dir()) # Ultralytics settings dir -SETTINGS_YAML = USER_CONFIG_DIR / "settings.yaml" +SETTINGS_FILE = USER_CONFIG_DIR / "settings.json" def colorstr(*input): @@ -1108,6 +1108,10 @@ class JSONDict(dict): super().__delitem__(key) self._save() + def __str__(self): + """Return a pretty-printed JSON string representation of the dictionary.""" + return f'JSONDict("{self.file_path}"):\n{json.dumps(dict(self), indent=2, ensure_ascii=False)}' + def update(self, *args, **kwargs): """Update the dictionary and persist changes.""" with self.lock: @@ -1121,21 +1125,36 @@ class JSONDict(dict): self._save() -class SettingsManager(dict): +class SettingsManager(JSONDict): """ - Manages Ultralytics settings stored in a YAML file. + SettingsManager class for managing and persisting Ultralytics settings. - Args: - file (str | Path): Path to the Ultralytics settings YAML file. Default is USER_CONFIG_DIR / 'settings.yaml'. - version (str): Settings version. In case of local version mismatch, new default settings will be saved. + This class extends JSONDict to provide JSON persistence for settings, ensuring thread-safe operations and default + values. It validates settings on initialization and provides methods to update or reset settings. + + Attributes: + file (Path): The path to the JSON file used for persistence. + version (str): The version of the settings schema. + defaults (Dict): A dictionary containing default settings. + help_msg (str): A help message for users on how to view and update settings. + + Methods: + _validate_settings: Validates the current settings and resets if necessary. + update: Updates settings, validating keys and types. + reset: Resets the settings to default and saves them. + + Examples: + Initialize and update settings: + >>> settings = SettingsManager() + >>> settings.update(runs_dir="/new/runs/dir") + >>> print(settings["runs_dir"]) + /new/runs/dir """ - def __init__(self, file=SETTINGS_YAML, version="0.0.5"): + def __init__(self, file=SETTINGS_FILE, version="0.0.6"): """Initializes the SettingsManager with default settings and loads user settings.""" - import copy import hashlib - from ultralytics.utils.checks import check_version from ultralytics.utils.torch_utils import torch_distributed_zero_first root = GIT_DIR or Path() @@ -1164,45 +1183,42 @@ class SettingsManager(dict): "vscode_msg": True, } self.help_msg = ( - f"\nView settings with 'yolo settings' or at '{self.file}'" - "\nUpdate settings with 'yolo settings key=value', i.e. 'yolo settings runs_dir=path/to/dir'. " + f"\nView Ultralytics Settings with 'yolo settings' or at '{self.file}'" + "\nUpdate Settings with 'yolo settings key=value', i.e. 'yolo settings runs_dir=path/to/dir'. " "For help see https://docs.ultralytics.com/quickstart/#ultralytics-settings." ) - super().__init__(copy.deepcopy(self.defaults)) - with torch_distributed_zero_first(RANK): - if not self.file.exists(): - self.save() - - self.load() - correct_keys = self.keys() == self.defaults.keys() - correct_types = all(type(a) is type(b) for a, b in zip(self.values(), self.defaults.values())) - correct_version = check_version(self["settings_version"], self.version) - if not (correct_keys and correct_types and correct_version): - LOGGER.warning( - "WARNING ⚠️ Ultralytics settings reset to default values. This may be due to a possible problem " - f"with your settings or a recent ultralytics package update. {self.help_msg}" - ) + super().__init__(self.file) + + if not self.file.exists() or not self: # Check if file doesn't exist or is empty + LOGGER.info(f"Creating new Ultralytics Settings v{version} file ✅ {self.help_msg}") self.reset() - if self.get("datasets_dir") == self.get("runs_dir"): - LOGGER.warning( - f"WARNING ⚠️ Ultralytics setting 'datasets_dir: {self.get('datasets_dir')}' " - f"must be different than 'runs_dir: {self.get('runs_dir')}'. " - f"Please change one to avoid possible issues during training. {self.help_msg}" - ) + self._validate_settings() + + def _validate_settings(self): + """Validate the current settings and reset if necessary.""" + correct_keys = set(self.keys()) == set(self.defaults.keys()) + correct_types = all(isinstance(self.get(k), type(v)) for k, v in self.defaults.items()) + correct_version = self.get("settings_version", "") == self.version - def load(self): - """Loads settings from the YAML file.""" - super().update(yaml_load(self.file)) + if not (correct_keys and correct_types and correct_version): + LOGGER.warning( + "WARNING ⚠️ Ultralytics settings reset to default values. This may be due to a possible problem " + f"with your settings or a recent ultralytics package update. {self.help_msg}" + ) + self.reset() - def save(self): - """Saves the current settings to the YAML file.""" - yaml_save(self.file, dict(self)) + if self.get("datasets_dir") == self.get("runs_dir"): + LOGGER.warning( + f"WARNING ⚠️ Ultralytics setting 'datasets_dir: {self.get('datasets_dir')}' " + f"must be different than 'runs_dir: {self.get('runs_dir')}'. " + f"Please change one to avoid possible issues during training. {self.help_msg}" + ) def update(self, *args, **kwargs): - """Updates a setting value in the current settings.""" + """Updates settings, validating keys and types.""" for k, v in kwargs.items(): if k not in self.defaults: raise KeyError(f"No Ultralytics setting '{k}'. {self.help_msg}") @@ -1210,20 +1226,16 @@ class SettingsManager(dict): if not isinstance(v, t): raise TypeError(f"Ultralytics setting '{k}' must be of type '{t}', not '{type(v)}'. {self.help_msg}") super().update(*args, **kwargs) - self.save() def reset(self): """Resets the settings to default and saves them.""" self.clear() self.update(self.defaults) - self.save() def deprecation_warn(arg, new_arg): """Issue a deprecation warning when a deprecated argument is used, suggesting an updated argument.""" - LOGGER.warning( - f"WARNING ⚠️ '{arg}' is deprecated and will be removed in in the future. " f"Please use '{new_arg}' instead." - ) + LOGGER.warning(f"WARNING ⚠️ '{arg}' is deprecated and will be removed in in the future. Use '{new_arg}' instead.") def clean_url(url):