From 1e70710f3eb3779bcbaa38fb428b1be53fc913eb Mon Sep 17 00:00:00 2001 From: Muhammad Rizwan Munawar Date: Thu, 31 Oct 2024 22:18:28 +0500 Subject: [PATCH] Add model comparison graphs in `benchmark.md` (#17212) Co-authored-by: UltralyticsAssistant Co-authored-by: Glenn Jocher --- docs/en/modes/benchmark.md | 20 +++++++ docs/overrides/javascript/extra.js | 88 ++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/docs/en/modes/benchmark.md b/docs/en/modes/benchmark.md index 3086e98ec..00d851bea 100644 --- a/docs/en/modes/benchmark.md +++ b/docs/en/modes/benchmark.md @@ -8,6 +8,26 @@ keywords: model benchmarking, YOLO11, Ultralytics, performance evaluation, expor Ultralytics YOLO ecosystem and integrations +## Benchmark Visualization + + + +!!! tip "Refresh Browser" + + You may need to refresh the page to view the graphs correctly due to potential cookie issues. + +
+ + + + + + + + +
+ + ## Introduction Once your model is trained and validated, the next logical step is to evaluate its performance in various real-world scenarios. Benchmark mode in Ultralytics YOLO11 serves this purpose by providing a robust framework for assessing the speed and [accuracy](https://www.ultralytics.com/glossary/accuracy) of your model across a range of export formats. diff --git a/docs/overrides/javascript/extra.js b/docs/overrides/javascript/extra.js index 5029ff489..13f07397e 100644 --- a/docs/overrides/javascript/extra.js +++ b/docs/overrides/javascript/extra.js @@ -147,3 +147,91 @@ document.addEventListener("DOMContentLoaded", () => { addInkeepWidget(); // initialize the widget }); }); + +// This object contains the benchmark data for various object detection models. +const data = { + 'YOLOv5': {s: {speed: 1.92, mAP: 37.4}, m: {speed: 4.03, mAP: 45.4}, l: {speed: 6.61, mAP: 49.0}, x: {speed: 11.89, mAP: 50.7}}, + 'YOLOv6': {n: {speed: 1.17, mAP: 37.5}, s: {speed: 2.66, mAP: 45.0}, m: {speed: 5.28, mAP: 50.0}, l: {speed: 8.95, mAP: 52.8}}, + 'YOLOv7': {l: {speed: 6.84, mAP: 51.4}, x: {speed: 11.57, mAP: 53.1}}, + 'YOLOv8': {n: {speed: 1.47, mAP: 37.3}, s: {speed: 2.66, mAP: 44.9}, m: {speed: 5.86, mAP: 50.2}, l: {speed: 9.06, mAP: 52.9}, x: {speed: 14.37, mAP: 53.9}}, + 'YOLOv9': {t: {speed: 2.30, mAP: 37.8}, s: {speed: 3.54, mAP: 46.5}, m: {speed: 6.43, mAP: 51.5}, c: {speed: 7.16, mAP: 52.8}, e: {speed: 16.77, mAP: 55.1}}, + 'YOLOv10': {n: {speed: 1.56, mAP: 39.5}, s: {speed: 2.66, mAP: 46.7}, m: {speed: 5.48, mAP: 51.3}, b: {speed: 6.54, mAP: 52.7}, l: {speed: 8.33, mAP: 53.3}, x: {speed: 12.2, mAP: 54.4}}, + 'PPYOLOE': {t: {speed: 2.84, mAP: 39.9}, s: {speed: 2.62, mAP: 43.7}, m: {speed: 5.56, mAP: 49.8}, l: {speed: 8.36, mAP: 52.9}, x: {speed: 14.3, mAP: 54.7}}, + 'YOLO11': {n: {speed: 1.55, mAP: 39.5}, s: {speed: 2.63, mAP: 47.0}, m: {speed: 5.27, mAP: 51.4}, l: {speed: 6.84, mAP: 53.2}, x: {speed: 12.49, mAP: 54.7}} +}; + +let chart = null; // chart variable will hold the reference to the current chart instance. + +// This function is responsible for updating the benchmarks chart. +function updateChart() { + // If a chart instance already exists, destroy it. + if (chart) { chart.destroy(); } + + // Get the selected algorithms from the checkboxes. + const selectedAlgorithms = [...document.querySelectorAll('input[name="algorithm"]:checked')].map(e => e.value); + + // Create the datasets for the selected algorithms. + const datasets = selectedAlgorithms.map((algorithm, index) => ({ + label: algorithm, // Label for the data points in the legend. + data: Object.entries(data[algorithm]).map(([version, point]) => ({ + x: point.speed, // Speed data points on the x-axis. + y: point.mAP, // mAP data points on the y-axis. + version: version.toUpperCase() // Store the version as additional data. + })), + fill: false, // Don't fill the chart. + borderColor: `hsl(${index * 90}, 70%, 50%)`, // Assign a unique color to each dataset. + tension: 0.3, // Smooth the line. + pointRadius: 5, // Increase the dot size. + pointHoverRadius: 10, // Increase the dot size on hover. + borderWidth: 2 // Set the line thickness. + })); + + // If there are no selected algorithms, return without creating a new chart. + if (datasets.length === 0) return; + + // Create a new chart instance. + chart = new Chart(document.getElementById('chart').getContext('2d'), { + type: 'line', // Set the chart type to line. + data: { datasets }, + options: { + plugins: { + legend: { display: true, position: 'top', labels: { color: '#111e68' } }, // Configure the legend. + tooltip: { + callbacks: { + label: (tooltipItem) => { + const { dataset, dataIndex } = tooltipItem; + const point = dataset.data[dataIndex]; + return `${dataset.label}${point.version.toLowerCase()}: Speed = ${point.x}, mAP = ${point.y}`; // Custom tooltip label. + } + }, + mode: 'nearest', + intersect: false + } // Configure the tooltip. + }, + interaction: { mode: 'nearest', axis: 'x', intersect: false }, // Configure the interaction mode. + scales: { + x: { + type: 'linear', position: 'bottom', + title: { display: true, text: 'Latency T4 TensorRT10 FP16 (ms/img)', color: '#111e68' }, // X-axis title. + grid: { color: '#e0e0e0' }, // Grid line color. + ticks: { color: '#111e68' } // Tick label color. + }, + y: { + title: { display: true, text: 'mAP', color: '#111e68' }, // Y-axis title. + grid: { color: '#e0e0e0' }, // Grid line color. + ticks: { color: '#111e68' } // Tick label color. + } + } + } + }); +} + +// Add event listeners to the checkboxes to trigger the chart update. +document.addEventListener("DOMContentLoaded", () => { + document.querySelectorAll('input[name="algorithm"]').forEach(checkbox => + checkbox.addEventListener('change', updateChart) + ); + // Call updateChart on initial load + updateChart(); + console.log("DOM loaded, initial chart render attempted"); +});