diff --git a/docs/index.md b/docs/index.md
index 6aec2886..4bce227d 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -103,48 +103,48 @@ You can install `supervision` in a
- **Detect and Annotate**
- ---
+ ***
- Annotate predictions from a range of object detection and segmentation models
+ Annotate predictions from a range of object detection and segmentation models
- [:octicons-arrow-right-24: Tutorial](how_to/detect_and_annotate.md)
+ [:octicons-arrow-right-24: Tutorial](how_to/detect_and_annotate.md)
- **Track Objects**
- ---
+ ***
- Discover how to enhance video analysis by implementing seamless object tracking
+ Discover how to enhance video analysis by implementing seamless object tracking
- [:octicons-arrow-right-24: Tutorial](how_to/track_objects.md)
+ [:octicons-arrow-right-24: Tutorial](how_to/track_objects.md)
- **Detect Small Objects**
- ---
+ ***
- Learn how to detect small objects in images
+ Learn how to detect small objects in images
- [:octicons-arrow-right-24: Tutorial](how_to/detect_small_objects.md)
+ [:octicons-arrow-right-24: Tutorial](how_to/detect_small_objects.md)
- **Count Objects Crossing Line**
- ---
+ ***
- Explore methods to accurately count and analyze objects crossing a predefined line
+ Explore methods to accurately count and analyze objects crossing a predefined line
- [:octicons-arrow-right-24: Notebook](https://supervision.roboflow.com/latest/notebooks/count-objects-crossing-the-line/)
+ [:octicons-arrow-right-24: Notebook](https://supervision.roboflow.com/latest/notebooks/count-objects-crossing-the-line/)
-- **Filter Objects in Zone**
+- > **Filter Objects in Zone**
- ---
+ ***
- Master the techniques to selectively filter and focus on objects within a specific zone
+ Master the techniques to selectively filter and focus on objects within a specific zone
- **Cheatsheet**
- ---
+ ***
- Access a quick reference guide to the most common `supervision` functions
+ Access a quick reference guide to the most common `supervision` functions
- [:octicons-arrow-right-24: Cheatsheet](https://roboflow.github.io/cheatsheet-supervision/)
+ [:octicons-arrow-right-24: Cheatsheet](https://roboflow.github.io/cheatsheet-supervision/)
diff --git a/docs/metrics/f1_score.md b/docs/metrics/f1_score.md
new file mode 100644
index 00000000..e31a9557
--- /dev/null
+++ b/docs/metrics/f1_score.md
@@ -0,0 +1,18 @@
+---
+comments: true
+status: new
+---
+
+# F1 Score
+
+
+
+:::supervision.metrics.f1_score.F1Score
+
+
+
+:::supervision.metrics.f1_score.F1ScoreResult
diff --git a/mkdocs.yml b/mkdocs.yml
index 097889a2..3cd86759 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -66,6 +66,7 @@ nav:
- Utils: datasets/utils.md
- Metrics:
- mAP: metrics/mean_average_precision.md
+ - F1 Score: metrics/f1_score.md
- Legacy Metrics: detection/metrics.md
- Utils:
- Video: utils/video.md
diff --git a/supervision/metrics/__init__.py b/supervision/metrics/__init__.py
index 5b4b30fc..17a6cd48 100644
--- a/supervision/metrics/__init__.py
+++ b/supervision/metrics/__init__.py
@@ -1,4 +1,10 @@
-from supervision.metrics.core import CLASS_ID_NONE, Metric, MetricTarget
+from supervision.metrics.core import (
+ CLASS_ID_NONE,
+ AveragingMethod,
+ Metric,
+ MetricTarget,
+)
+from supervision.metrics.f1_score import F1Score, F1ScoreResult
from supervision.metrics.mean_average_precision import (
MeanAveragePrecision,
MeanAveragePrecisionResult,
diff --git a/supervision/metrics/core.py b/supervision/metrics/core.py
index 4f84eb9d..1440fd43 100644
--- a/supervision/metrics/core.py
+++ b/supervision/metrics/core.py
@@ -48,3 +48,26 @@ class MetricTarget(Enum):
BOXES = "boxes"
MASKS = "masks"
ORIENTED_BOUNDING_BOXES = "obb"
+
+
+class AveragingMethod(Enum):
+ """
+ Defines different ways of averaging the metric results.
+
+ Suppose, before returning the final result, a metric is computed for each class.
+ How do you combine those to get the final number?
+
+ * MACRO: Calculate the metric for each class and average the results. The simplest
+ averaging method, but it does not take class imbalance into account.
+ * MICRO: Calculate the metric globally by counting the total true positives, false
+ positives, and false negatives. Micro averaging is useful when you want to give
+ more importance to classes with more samples. It's also more appropriate if you
+ have an imbalance in the number of instances per class.
+ * WEIGHTED: Calculate the metric for each class and average the results, weighted by
+ the number of true instances of each class. Use weighted averaging if you want
+ to take class imbalance into account.
+ """
+
+ MACRO = "macro"
+ MICRO = "micro"
+ WEIGHTED = "weighted"
diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py
new file mode 100644
index 00000000..2ca5bca5
--- /dev/null
+++ b/supervision/metrics/f1_score.py
@@ -0,0 +1,545 @@
+from __future__ import annotations
+
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+
+import numpy as np
+from matplotlib import pyplot as plt
+
+from supervision.config import ORIENTED_BOX_COORDINATES
+from supervision.detection.core import Detections
+from supervision.detection.utils import box_iou_batch, mask_iou_batch
+from supervision.draw.color import LEGACY_COLOR_PALETTE
+from supervision.metrics.core import AveragingMethod, Metric, MetricTarget
+from supervision.metrics.utils.object_size import (
+ ObjectSizeCategory,
+ get_detection_size_category,
+)
+from supervision.metrics.utils.utils import ensure_pandas_installed
+
+if TYPE_CHECKING:
+ import pandas as pd
+
+
+class F1Score(Metric):
+ def __init__(
+ self,
+ metric_target: MetricTarget = MetricTarget.BOXES,
+ averaging_method: AveragingMethod = AveragingMethod.WEIGHTED,
+ ):
+ self._metric_target = metric_target
+ if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
+ raise NotImplementedError(
+ "F1 score is not implemented for oriented bounding boxes."
+ )
+
+ self._metric_target = metric_target
+ self.averaging_method = averaging_method
+ self._predictions_list: List[Detections] = []
+ self._targets_list: List[Detections] = []
+
+ def reset(self) -> None:
+ self._predictions_list = []
+ self._targets_list = []
+
+ def update(
+ self,
+ predictions: Union[Detections, List[Detections]],
+ targets: Union[Detections, List[Detections]],
+ ) -> F1Score:
+ if not isinstance(predictions, list):
+ predictions = [predictions]
+ if not isinstance(targets, list):
+ targets = [targets]
+
+ if len(predictions) != len(targets):
+ raise ValueError(
+ f"The number of predictions ({len(predictions)}) and"
+ f" targets ({len(targets)}) during the update must be the same."
+ )
+
+ self._predictions_list.extend(predictions)
+ self._targets_list.extend(targets)
+
+ return self
+
+ def compute(self) -> F1ScoreResult:
+ result = self._compute(self._predictions_list, self._targets_list)
+
+ small_predictions, small_targets = self._filter_predictions_and_targets_by_size(
+ self._predictions_list, self._targets_list, ObjectSizeCategory.SMALL
+ )
+ result.small_objects = self._compute(small_predictions, small_targets)
+
+ medium_predictions, medium_targets = (
+ self._filter_predictions_and_targets_by_size(
+ self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM
+ )
+ )
+ result.medium_objects = self._compute(medium_predictions, medium_targets)
+
+ large_predictions, large_targets = self._filter_predictions_and_targets_by_size(
+ self._predictions_list, self._targets_list, ObjectSizeCategory.LARGE
+ )
+ result.large_objects = self._compute(large_predictions, large_targets)
+
+ return result
+
+ def _compute(
+ self, predictions_list: List[Detections], targets_list: List[Detections]
+ ) -> F1ScoreResult:
+ iou_thresholds = np.linspace(0.5, 0.95, 10)
+ stats = []
+
+ for predictions, targets in zip(predictions_list, targets_list):
+ prediction_contents = self._detections_content(predictions)
+ target_contents = self._detections_content(targets)
+
+ if len(targets) > 0:
+ if len(predictions) == 0:
+ stats.append(
+ (
+ np.zeros((0, iou_thresholds.size), dtype=bool),
+ np.zeros((0,), dtype=np.float32),
+ np.zeros((0,), dtype=int),
+ targets.class_id,
+ )
+ )
+
+ else:
+ if self._metric_target == MetricTarget.BOXES:
+ iou = box_iou_batch(target_contents, prediction_contents)
+ elif self._metric_target == MetricTarget.MASKS:
+ iou = mask_iou_batch(target_contents, prediction_contents)
+ else:
+ raise NotImplementedError(
+ "Unsupported metric target for IoU calculation"
+ )
+
+ matches = self._match_detection_batch(
+ predictions.class_id, targets.class_id, iou, iou_thresholds
+ )
+ stats.append(
+ (
+ matches,
+ predictions.confidence,
+ predictions.class_id,
+ targets.class_id,
+ )
+ )
+
+ if not stats:
+ return F1ScoreResult(
+ metric_target=self._metric_target,
+ averaging_method=self.averaging_method,
+ f1_scores=np.zeros(iou_thresholds.shape[0]),
+ f1_per_class=np.zeros((0, iou_thresholds.shape[0])),
+ iou_thresholds=iou_thresholds,
+ matched_classes=np.array([], dtype=int),
+ small_objects=None,
+ medium_objects=None,
+ large_objects=None,
+ )
+
+ concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)]
+ f1_scores, f1_per_class, unique_classes = self._compute_f1_for_classes(
+ *concatenated_stats
+ )
+
+ return F1ScoreResult(
+ metric_target=self._metric_target,
+ averaging_method=self.averaging_method,
+ f1_scores=f1_scores,
+ f1_per_class=f1_per_class,
+ iou_thresholds=iou_thresholds,
+ matched_classes=unique_classes,
+ small_objects=None,
+ medium_objects=None,
+ large_objects=None,
+ )
+
+ def _compute_f1_for_classes(
+ self,
+ matches: np.ndarray,
+ prediction_confidence: np.ndarray,
+ prediction_class_ids: np.ndarray,
+ true_class_ids: np.ndarray,
+ ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+ sorted_indices = np.argsort(-prediction_confidence)
+ matches = matches[sorted_indices]
+ prediction_class_ids = prediction_class_ids[sorted_indices]
+ unique_classes, class_counts = np.unique(true_class_ids, return_counts=True)
+
+ # Shape: PxTh,P,C,C -> CxThx3
+ confusion_matrix = self._compute_confusion_matrix(
+ matches, prediction_class_ids, unique_classes, class_counts
+ )
+
+ # Shape: CxThx3 -> CxTh
+ f1_per_class = self._compute_f1(confusion_matrix)
+
+ # Shape: CxTh -> Th
+ if self.averaging_method == AveragingMethod.MACRO:
+ f1_scores = np.mean(f1_per_class, axis=0)
+ elif self.averaging_method == AveragingMethod.MICRO:
+ confusion_matrix_merged = confusion_matrix.sum(0)
+ f1_scores = self._compute_f1(confusion_matrix_merged)
+ elif self.averaging_method == AveragingMethod.WEIGHTED:
+ class_counts = class_counts.astype(np.float32)
+ f1_scores = np.average(f1_per_class, axis=0, weights=class_counts)
+
+ return f1_scores, f1_per_class, unique_classes
+
+ @staticmethod
+ def _match_detection_batch(
+ predictions_classes: np.ndarray,
+ target_classes: np.ndarray,
+ iou: np.ndarray,
+ iou_thresholds: np.ndarray,
+ ) -> np.ndarray:
+ num_predictions, num_iou_levels = (
+ predictions_classes.shape[0],
+ iou_thresholds.shape[0],
+ )
+ correct = np.zeros((num_predictions, num_iou_levels), dtype=bool)
+ correct_class = target_classes[:, None] == predictions_classes
+
+ for i, iou_level in enumerate(iou_thresholds):
+ matched_indices = np.where((iou >= iou_level) & correct_class)
+
+ if matched_indices[0].shape[0]:
+ combined_indices = np.stack(matched_indices, axis=1)
+ iou_values = iou[matched_indices][:, None]
+ matches = np.hstack([combined_indices, iou_values])
+
+ if matched_indices[0].shape[0] > 1:
+ matches = matches[matches[:, 2].argsort()[::-1]]
+ matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
+ matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
+
+ correct[matches[:, 1].astype(int), i] = True
+
+ return correct
+
+ @staticmethod
+ def _compute_confusion_matrix(
+ sorted_matches: np.ndarray,
+ sorted_prediction_class_ids: np.ndarray,
+ unique_classes: np.ndarray,
+ class_counts: np.ndarray,
+ ) -> np.ndarray:
+ """
+ Compute the confusion matrix for each class and IoU threshold.
+
+ Assumes the matches and prediction_class_ids are sorted by confidence
+ in descending order.
+
+ Arguments:
+ sorted_matches: np.ndarray, bool, shape (P, Th), that is True
+ if the prediction is a true positive at the given IoU threshold.
+ sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing
+ the class id for each prediction.
+ unique_classes: np.ndarray, int, shape (C,), containing the unique
+ class ids.
+ class_counts: np.ndarray, int, shape (C,), containing the number
+ of true instances for each class.
+
+ Returns:
+ np.ndarray, shape (C, Th, 3), containing the true positives, false
+ positives, and false negatives for each class and IoU threshold.
+ """
+
+ num_thresholds = sorted_matches.shape[1]
+ num_classes = unique_classes.shape[0]
+
+ confusion_matrix = np.zeros((num_classes, num_thresholds, 3))
+ for class_idx, class_id in enumerate(unique_classes):
+ is_class = sorted_prediction_class_ids == class_id
+ num_true = class_counts[class_idx]
+ num_predictions = is_class.sum()
+
+ if num_predictions == 0:
+ true_positives = np.zeros(num_thresholds)
+ false_positives = np.zeros(num_thresholds)
+ false_negatives = np.full(num_thresholds, num_true)
+ elif num_true == 0:
+ true_positives = np.zeros(num_thresholds)
+ false_positives = np.full(num_thresholds, num_predictions)
+ false_negatives = np.zeros(num_thresholds)
+ else:
+ true_positives = sorted_matches[is_class].sum(0)
+ false_positives = (1 - sorted_matches[is_class]).sum(0)
+ false_negatives = num_true - true_positives
+ confusion_matrix[class_idx] = np.stack(
+ [true_positives, false_positives, false_negatives], axis=1
+ )
+
+ return confusion_matrix
+
+ @staticmethod
+ def _compute_f1(confusion_matrix: np.ndarray) -> np.ndarray:
+ """
+ Broadcastable function, computing the F1 score from the confusion matrix.
+
+ Arguments:
+ confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension
+ contains the true positives, false positives, and false negatives.
+
+ Returns:
+ np.ndarray, shape (N, ...), containing the F1 score for each element.
+ """
+ if not confusion_matrix.shape[-1] == 3:
+ raise ValueError(
+ f"Confusion matrix must have shape (..., 3), got "
+ f"{confusion_matrix.shape}"
+ )
+ true_positives = confusion_matrix[..., 0]
+ false_positives = confusion_matrix[..., 1]
+ false_negatives = confusion_matrix[..., 2]
+
+ # Alternate formula, avoids multiple zero division checks
+ denominator = 2 * true_positives + false_positives + false_negatives
+ f1_score = np.where(denominator == 0, 0, 2 * true_positives / denominator)
+
+ return f1_score
+
+ def _detections_content(self, detections: Detections) -> np.ndarray:
+ """Return boxes, masks or oriented bounding boxes from detections."""
+ if self._metric_target == MetricTarget.BOXES:
+ return detections.xyxy
+ if self._metric_target == MetricTarget.MASKS:
+ return (
+ detections.mask
+ if detections.mask is not None
+ else np.empty((0, 0, 0), dtype=bool)
+ )
+ if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
+ if obb := detections.data.get(ORIENTED_BOX_COORDINATES):
+ return np.ndarray(obb, dtype=np.float32)
+ return np.empty((0, 8), dtype=np.float32)
+ raise ValueError(f"Invalid metric target: {self._metric_target}")
+
+ def _filter_detections_by_size(
+ self, detections: Detections, size_category: ObjectSizeCategory
+ ) -> Detections:
+ """Return a copy of detections with contents filtered by object size."""
+ new_detections = deepcopy(detections)
+ if detections.is_empty() or size_category == ObjectSizeCategory.ANY:
+ return new_detections
+
+ sizes = get_detection_size_category(new_detections, self._metric_target)
+ size_mask = sizes == size_category.value
+
+ new_detections.xyxy = new_detections.xyxy[size_mask]
+ if new_detections.mask is not None:
+ new_detections.mask = new_detections.mask[size_mask]
+ if new_detections.class_id is not None:
+ new_detections.class_id = new_detections.class_id[size_mask]
+ if new_detections.confidence is not None:
+ new_detections.confidence = new_detections.confidence[size_mask]
+ if new_detections.tracker_id is not None:
+ new_detections.tracker_id = new_detections.tracker_id[size_mask]
+ if new_detections.data is not None:
+ for key, value in new_detections.data.items():
+ new_detections.data[key] = np.array(value)[size_mask]
+
+ return new_detections
+
+ def _filter_predictions_and_targets_by_size(
+ self,
+ predictions_list: List[Detections],
+ targets_list: List[Detections],
+ size_category: ObjectSizeCategory,
+ ) -> Tuple[List[Detections], List[Detections]]:
+ """
+ Filter predictions and targets by object size category.
+ """
+ new_predictions_list = []
+ new_targets_list = []
+ for predictions, targets in zip(predictions_list, targets_list):
+ new_predictions_list.append(
+ self._filter_detections_by_size(predictions, size_category)
+ )
+ new_targets_list.append(
+ self._filter_detections_by_size(targets, size_category)
+ )
+ return new_predictions_list, new_targets_list
+
+
+@dataclass
+class F1ScoreResult:
+ """
+ The results of the F1 score metric calculation.
+
+ Defaults to `0` if no detections or targets were provided.
+ Provides a custom `__str__` method for pretty printing.
+
+ Attributes:
+ metric_target (MetricTarget): the type of data used for the metric -
+ boxes, masks or oriented bounding boxes.
+ averaging_method (AveragingMethod): the averaging method used to compute the
+ F1 scores. Determines how the F1 scores are aggregated across classes.
+ f1_50 (float): the F1 score at IoU threshold of `0.5`.
+ f1_75 (float): the F1 score at IoU threshold of `0.75`.
+ f1_scores (np.ndarray): the F1 scores at each IoU threshold.
+ Shape: `(num_iou_thresholds,)`
+ f1_per_class (np.ndarray): the F1 scores per class and IoU threshold.
+ Shape: `(num_target_classes, num_iou_thresholds)`
+ iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
+ matched_classes (np.ndarray): the class IDs of all matched classes.
+ Corresponds to the rows of `f1_per_class`.
+ small_objects (Optional[F1ScoreResult]): the F1 metric results
+ for small objects.
+ medium_objects (Optional[F1ScoreResult]): the F1 metric results
+ for medium objects.
+ large_objects (Optional[F1ScoreResult]): the F1 metric results
+ for large objects.
+ """
+
+ metric_target: MetricTarget
+ averaging_method: AveragingMethod
+
+ @property
+ def f1_50(self) -> float:
+ return self.f1_scores[0]
+
+ @property
+ def f1_75(self) -> float:
+ return self.f1_scores[5]
+
+ f1_scores: np.ndarray
+ f1_per_class: np.ndarray
+ iou_thresholds: np.ndarray
+ matched_classes: np.ndarray
+
+ small_objects: Optional[F1ScoreResult]
+ medium_objects: Optional[F1ScoreResult]
+ large_objects: Optional[F1ScoreResult]
+
+ def __str__(self) -> str:
+ """
+ Format as a pretty string.
+
+ Example:
+ ```python
+ print(f1_result)
+ ```
+ """
+ out_str = (
+ f"{self.__class__.__name__}:\n"
+ f"Metric target: {self.metric_target}\n"
+ f"Averaging method: {self.averaging_method}\n"
+ f"F1 @ 50: {self.f1_50:.4f}\n"
+ f"F1 @ 75: {self.f1_75:.4f}\n"
+ f"F1 @ thresh: {self.f1_scores}\n"
+ f"IoU thresh: {self.iou_thresholds}\n"
+ f"F1 per class:\n"
+ )
+ if self.f1_per_class.size == 0:
+ out_str += " No results\n"
+ for class_id, f1_of_class in zip(self.matched_classes, self.f1_per_class):
+ out_str += f" {class_id}: {f1_of_class}\n"
+
+ indent = " "
+ if self.small_objects is not None:
+ indented = indent + str(self.small_objects).replace("\n", f"\n{indent}")
+ out_str += f"\nSmall objects:\n{indented}"
+ if self.medium_objects is not None:
+ indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}")
+ out_str += f"\nMedium objects:\n{indented}"
+ if self.large_objects is not None:
+ indented = indent + str(self.large_objects).replace("\n", f"\n{indent}")
+ out_str += f"\nLarge objects:\n{indented}"
+
+ return out_str
+
+ def to_pandas(self) -> "pd.DataFrame":
+ """
+ Convert the result to a pandas DataFrame.
+
+ Returns:
+ (pd.DataFrame): The result as a DataFrame.
+ """
+ ensure_pandas_installed()
+ import pandas as pd
+
+ pandas_data = {
+ "F1@50": self.f1_50,
+ "F1@75": self.f1_75,
+ }
+
+ if self.small_objects is not None:
+ small_objects_df = self.small_objects.to_pandas()
+ for key, value in small_objects_df.items():
+ pandas_data[f"small_objects_{key}"] = value
+ if self.medium_objects is not None:
+ medium_objects_df = self.medium_objects.to_pandas()
+ for key, value in medium_objects_df.items():
+ pandas_data[f"medium_objects_{key}"] = value
+ if self.large_objects is not None:
+ large_objects_df = self.large_objects.to_pandas()
+ for key, value in large_objects_df.items():
+ pandas_data[f"large_objects_{key}"] = value
+
+ return pd.DataFrame(pandas_data, index=[0])
+
+ def plot(self):
+ """
+ Plot the F1 results.
+ """
+
+ labels = ["F1@50", "F1@75"]
+ values = [self.f1_50, self.f1_75]
+ colors = [LEGACY_COLOR_PALETTE[0]] * 2
+
+ if self.small_objects is not None:
+ small_objects = self.small_objects
+ labels += ["Small: F1@50", "Small: F1@75"]
+ values += [small_objects.f1_50, small_objects.f1_75]
+ colors += [LEGACY_COLOR_PALETTE[3]] * 2
+
+ if self.medium_objects is not None:
+ medium_objects = self.medium_objects
+ labels += ["Medium: F1@50", "Medium: F1@75"]
+ values += [medium_objects.f1_50, medium_objects.f1_75]
+ colors += [LEGACY_COLOR_PALETTE[2]] * 2
+
+ if self.large_objects is not None:
+ large_objects = self.large_objects
+ labels += ["Large: F1@50", "Large: F1@75"]
+ values += [large_objects.f1_50, large_objects.f1_75]
+ colors += [LEGACY_COLOR_PALETTE[4]] * 2
+
+ plt.rcParams["font.family"] = "monospace"
+
+ _, ax = plt.subplots(figsize=(10, 6))
+ ax.set_ylim(0, 1)
+ ax.set_ylabel("Value", fontweight="bold")
+ title = (
+ f"F1 Score, by Object Size"
+ f"\n(target: {self.metric_target.value},"
+ f" averaging: {self.averaging_method.value})"
+ )
+ ax.set_title(title, fontweight="bold")
+
+ x_positions = range(len(labels))
+ bars = ax.bar(x_positions, values, color=colors, align="center")
+
+ ax.set_xticks(x_positions)
+ ax.set_xticklabels(labels, rotation=45, ha="right")
+
+ for bar in bars:
+ y_value = bar.get_height()
+ ax.text(
+ bar.get_x() + bar.get_width() / 2,
+ y_value + 0.02,
+ f"{y_value:.2f}",
+ ha="center",
+ va="bottom",
+ )
+
+ plt.rcParams["font.family"] = "sans-serif"
+
+ plt.tight_layout()
+ plt.show()
diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py
index 5e6b54bb..04a5fe9d 100644
--- a/supervision/metrics/mean_average_precision.py
+++ b/supervision/metrics/mean_average_precision.py
@@ -2,7 +2,7 @@ from __future__ import annotations
from copy import deepcopy
from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
import numpy as np
from matplotlib import pyplot as plt
@@ -192,21 +192,21 @@ class MeanAveragePrecision(Metric):
# Compute average precisions if any matches exist
if stats:
concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)]
- average_precisions = self._average_precisions_per_class(*concatenated_stats)
- map50 = average_precisions[:, 0].mean()
- map75 = average_precisions[:, 5].mean()
- map50_95 = average_precisions.mean()
+ average_precisions, unique_classes = self._average_precisions_per_class(
+ *concatenated_stats
+ )
+ mAP_scores = np.mean(average_precisions, axis=0)
else:
- map50, map75, map50_95 = 0, 0, 0
+ mAP_scores = np.zeros((10,), dtype=np.float32)
+ unique_classes = np.empty((0,), dtype=int)
average_precisions = np.empty((0, len(iou_thresholds)), dtype=np.float32)
return MeanAveragePrecisionResult(
- iou_thresholds=iou_thresholds,
- map50_95=map50_95,
- map50=map50,
- map75=map75,
- per_class_ap50_95=average_precisions,
metric_target=self._metric_target,
+ mAP_scores=mAP_scores,
+ iou_thresholds=iou_thresholds,
+ matched_classes=unique_classes,
+ ap_per_class=average_precisions,
)
@staticmethod
@@ -270,7 +270,7 @@ class MeanAveragePrecision(Metric):
prediction_confidence: np.ndarray,
prediction_class_ids: np.ndarray,
true_class_ids: np.ndarray,
- ) -> np.ndarray:
+ ) -> Tuple[np.ndarray, np.ndarray]:
"""
Compute the average precision, given the recall and precision curves.
Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
@@ -283,7 +283,8 @@ class MeanAveragePrecision(Metric):
eps (float, optional): Small value to prevent division by zero.
Returns:
- (np.ndarray): Average precision for different IoU levels.
+ (Tuple[np.ndarray, np.ndarray]): Average precision for different
+ IoU levels, and an array of class IDs that were matched.
"""
eps = 1e-16
@@ -318,7 +319,7 @@ class MeanAveragePrecision(Metric):
)
)
- return average_precisions
+ return average_precisions, unique_classes
def _detections_content(self, detections: Detections) -> np.ndarray:
"""Return boxes, masks or oriented bounding boxes from detections."""
@@ -374,38 +375,57 @@ class MeanAveragePrecision(Metric):
@dataclass
class MeanAveragePrecisionResult:
- iou_thresholds: np.ndarray
- """Array of IoU thresholds used in the calculations"""
- map50_95: float
- """Mean Average Precision over IoU thresholds from 0.5 to 0.95"""
+ """
+ The result of the Mean Average Precision calculation.
- map50: float
- """Mean Average Precision at IoU threshold of 0.5"""
+ Defaults to `0` when no detections or targets are present.
- map75: float
- """Mean Average Precision at IoU threshold of 0.75"""
-
- per_class_ap50_95: np.ndarray
- """Average precision for each class at different IoU thresholds"""
+ Attributes:
+ metric_target (MetricTarget): the type of data used for the metric -
+ boxes, masks or oriented bounding boxes.
+ mAP_map50_95 (float): the mAP score at IoU thresholds from `0.5` to `0.95`.
+ mAP_map50 (float): the mAP score at IoU threshold of `0.5`.
+ mAP_map75 (float): the mAP score at IoU threshold of `0.75`.
+ mAP_scores (np.ndarray): the mAP scores at each IoU threshold.
+ Shape: `(num_iou_thresholds,)`
+ ap_per_class (np.ndarray): the average precision scores per
+ class and IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)`
+ iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
+ matched_classes (np.ndarray): the class IDs of all matched classes.
+ Corresponds to the rows of `ap_per_class`.
+ small_objects (Optional[MeanAveragePrecisionResult]): the mAP results
+ for small objects.
+ medium_objects (Optional[MeanAveragePrecisionResult]): the mAP results
+ for medium objects.
+ large_objects (Optional[MeanAveragePrecisionResult]): the mAP results
+ for large objects.
+ """
metric_target: MetricTarget
- """
- Defines the type of data used for the metric - boxes, masks or
- oriented bounding boxes.
- """
+ @property
+ def map50_95(self) -> float:
+ return self.mAP_scores.mean()
+
+ @property
+ def map50(self) -> float:
+ return self.mAP_scores[0]
+
+ @property
+ def map75(self) -> float:
+ return self.mAP_scores[5]
+
+ mAP_scores: np.ndarray
+ ap_per_class: np.ndarray
+ iou_thresholds: np.ndarray
+ matched_classes: np.ndarray
small_objects: Optional[MeanAveragePrecisionResult] = None
- """Mean Average Precision results for small objects"""
-
medium_objects: Optional[MeanAveragePrecisionResult] = None
- """Mean Average Precision results for medium objects"""
-
large_objects: Optional[MeanAveragePrecisionResult] = None
- """Mean Average Precision results for large objects"""
def __str__(self) -> str:
"""
- Format the mAP results as a pretty string.
+ Format as a pretty string.
Example:
```python
@@ -415,28 +435,29 @@ class MeanAveragePrecisionResult:
out_str = (
f"{self.__class__.__name__}:\n"
- f"iou_thresholds: {self.iou_thresholds}\n"
- f"map50_95: {self.map50_95:.4f}\n"
- f"map50: {self.map50:.4f}\n"
- f"map75: {self.map75:.4f}\n"
- f"per_class_ap50_95:"
+ f"Metric target: {self.metric_target}\n"
+ f"mAP @ 50:95: {self.map50_95:.4f}\n"
+ f"mAP @ 50: {self.map50:.4f}\n"
+ f"mAP @ 75: {self.map75:.4f}\n"
+ f"mAP scores: {self.mAP_scores}\n"
+ f"IoU thresh: {self.iou_thresholds}\n"
+ f"AP per class:\n"
)
-
- for class_id, ap in enumerate(self.per_class_ap50_95):
- out_str += f"\n {class_id}: {ap}"
+ if self.ap_per_class.size == 0:
+ out_str += " No results\n"
+ for class_id, ap_of_class in zip(self.matched_classes, self.ap_per_class):
+ out_str += f" {class_id}: {ap_of_class}\n"
indent = " "
if self.small_objects is not None:
- indented_str = indent + str(self.small_objects).replace("\n", f"\n{indent}")
- out_str += f"\nSmall objects:\n{indented_str}"
+ indented = indent + str(self.small_objects).replace("\n", f"\n{indent}")
+ out_str += f"\nSmall objects:\n{indented}"
if self.medium_objects is not None:
- indented_str = indent + str(self.medium_objects).replace(
- "\n", f"\n{indent}"
- )
- out_str += f"\nMedium objects:\n{indented_str}"
+ indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}")
+ out_str += f"\nMedium objects:\n{indented}"
if self.large_objects is not None:
- indented_str = indent + str(self.large_objects).replace("\n", f"\n{indent}")
- out_str += f"\nLarge objects:\n{indented_str}"
+ indented = indent + str(self.large_objects).replace("\n", f"\n{indent}")
+ out_str += f"\nLarge objects:\n{indented}"
return out_str
@@ -451,10 +472,11 @@ class MeanAveragePrecisionResult:
import pandas as pd
pandas_data = {
- "mAP_50_95": self.map50_95,
- "mAP_50": self.map50,
- "mAP_75": self.map75,
+ "mAP@50:95": self.map50_95,
+ "mAP@50": self.map50,
+ "mAP@75": self.map75,
}
+
if self.small_objects is not None:
small_objects_df = self.small_objects.to_pandas()
for key, value in small_objects_df.items():
@@ -480,16 +502,12 @@ class MeanAveragePrecisionResult:
Plot the mAP results.
"""
- labels = ["mAP_50_95", "mAP_50", "mAP_75"]
+ labels = ["mAP@50:95", "mAP@50", "mAP@75"]
values = [self.map50_95, self.map50, self.map75]
colors = [LEGACY_COLOR_PALETTE[0]] * 3
if self.small_objects is not None:
- labels += [
- "small_objects_mAP_50_95",
- "small_objects_mAP_50",
- "small_objects_mAP_75",
- ]
+ labels += ["Small: mAP@50:95", "Small: mAP@50", "Small: mAP@75"]
values += [
self.small_objects.map50_95,
self.small_objects.map50,
@@ -498,11 +516,7 @@ class MeanAveragePrecisionResult:
colors += [LEGACY_COLOR_PALETTE[3]] * 3
if self.medium_objects is not None:
- labels += [
- "medium_objects_mAP_50_95",
- "medium_objects_mAP_50",
- "medium_objects_mAP_75",
- ]
+ labels += ["Medium: mAP@50:95", "Medium: mAP@50", "Medium: mAP@75"]
values += [
self.medium_objects.map50_95,
self.medium_objects.map50,
@@ -511,11 +525,7 @@ class MeanAveragePrecisionResult:
colors += [LEGACY_COLOR_PALETTE[2]] * 3
if self.large_objects is not None:
- labels += [
- "large_objects_mAP_50_95",
- "large_objects_mAP_50",
- "large_objects_mAP_75",
- ]
+ labels += ["Large: mAP@50:95", "Large: mAP@50", "Large: mAP@75"]
values += [
self.large_objects.map50_95,
self.large_objects.map50,
diff --git a/supervision/metrics/utils/internal_data_store.py b/supervision/metrics/utils/internal_data_store.py
deleted file mode 100644
index f069f506..00000000
--- a/supervision/metrics/utils/internal_data_store.py
+++ /dev/null
@@ -1,300 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, List, Optional, Set, Tuple
-
-import numpy as np
-import numpy.typing as npt
-
-from supervision.config import ORIENTED_BOX_COORDINATES
-from supervision.metrics.core import CLASS_ID_NONE, MetricTarget
-from supervision.metrics.utils.object_size import (
- ObjectSizeCategory,
- get_object_size_category,
-)
-from supervision.metrics.utils.utils import unify_pad_masks_shape
-
-if TYPE_CHECKING:
- from supervision.detection.core import Detections
-
-
-class MetricData:
- """
- A container for detection contents, decouple from `Detections`.
- """
-
- def __init__(self, metric_target: MetricTarget, class_agnostic: bool = False):
- self._metric_target = metric_target
- self._class_agnostic = class_agnostic
- self._content_list: List[npt.NDArray] = []
- self._confidence_list: List[npt.NDArray[np.float32]] = []
- self._class_id_list: List[npt.NDArray[np.int_]] = []
-
- def update(self, detections: Detections):
- """Add new detections to the store."""
- if detections.is_empty() or len(detections) == 0:
- return
-
- # Relies on Detections to ensure that member vars are equal in length
- # or None/empty.
- self._validate_new_entry(detections)
-
- new_content = self._detections_content(detections)
- self._validate_shape(new_content)
- self._content_list.append(new_content)
-
- if detections.confidence is not None:
- self._confidence_list.append(detections.confidence)
-
- if detections.class_id is not None:
- if self._class_agnostic:
- self._class_id_list.append(
- np.full(len(detections.class_id), CLASS_ID_NONE, dtype=int)
- )
- else:
- self._class_id_list.append(detections.class_id)
-
- def get_classes(self) -> Set[int]:
- """Return all class IDs."""
- if self._class_agnostic:
- return {CLASS_ID_NONE}
- return set.union(*[set(class_id) for class_id in self._class_id_list])
-
- def get(
- self, class_id: Optional[int] = None, size_category=ObjectSizeCategory.ANY
- ) -> Tuple[npt.NDArray, npt.NDArray, npt.NDArray]:
- """
- Get the contents, class_ids and confidences, optionally filtered by
- class_id and/or size.
-
- Args:
- class_id (Optional[int]): The class ID for the data to retrieve
- size_category (ObjectSizeCategory): The size of the objects to retrieve.
-
- Returns:
- (np.ndarray): Boxes, masks or obb, all stacked in a single array.
- """
- if self._metric_target in [
- MetricTarget.BOXES,
- MetricTarget.ORIENTED_BOUNDING_BOXES,
- ]:
- self._merge_boxes()
- if len(self._content_list) == 0:
- content = self._make_empty_content()
- else:
- content = self._content_list[0]
-
- elif self._metric_target == MetricTarget.MASKS:
- self._merge_masks()
- if len(self._content_list) == 0:
- content = self._make_empty_content()
- else:
- content = self._content_list[0]
-
- size_mask = np.full(len(content), True)
- if size_category != ObjectSizeCategory.ANY:
- sizes = get_object_size_category(content, self._metric_target)
- size_mask = sizes == size_category.value
-
- class_mask = np.full(len(content), True)
- if class_id is not None:
- self._merge_class_id()
- class_ids = self._class_id_list[0]
- class_mask = class_ids == class_id
-
- content = content[(size_mask & class_mask)]
-
- self._merge_class_id()
- if len(self._class_id_list) == 0:
- class_ids = np.array([], dtype=int)
- else:
- class_ids = self._class_id_list[0]
- if len(class_ids) > 0:
- class_ids = class_ids[(size_mask & class_mask)]
-
- self._merge_confidence()
- if len(self._confidence_list) == 0:
- confidences = np.array([], dtype=float)
- else:
- confidences = self._confidence_list[0]
- if len(confidences) > 0:
- confidences = confidences[(size_mask & class_mask)]
-
- return content, class_ids, confidences
-
- def get_class_id(self) -> npt.NDArray[np.int_]:
- self._merge_class_id()
- if len(self._class_id_list) == 0:
- return np.array([], dtype=int)
- return self._class_id_list[0]
-
- def get_confidence(self) -> npt.NDArray[np.float32]:
- self._merge_confidence()
- if len(self._confidence_list) == 0:
- return np.array([], dtype=float)
- return self._confidence_list[0]
-
- def _detections_content(self, detections: Detections) -> npt.NDArray:
- """Return boxes, masks or oriented bounding boxes from detections."""
- if self._metric_target == MetricTarget.BOXES:
- return detections.xyxy
- if self._metric_target == MetricTarget.MASKS:
- return (
- detections.mask
- if detections.mask is not None
- else self._make_empty_content()
- )
- if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
- if obb := detections.data.get(ORIENTED_BOX_COORDINATES):
- return np.ndarray(obb, dtype=np.float32)
- return self._make_empty_content()
- raise ValueError(f"Invalid metric target: {self._metric_target}")
-
- def _make_empty_content(self) -> npt.NDArray:
- if self._metric_target == MetricTarget.BOXES:
- return np.empty((0, 4), dtype=np.float32)
- if self._metric_target == MetricTarget.MASKS:
- return np.empty((0, 0, 0), dtype=bool)
- if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
- return np.empty((0, 8), dtype=np.float32)
- raise ValueError(f"Invalid metric target: {self._metric_target}")
-
- def _validate_new_entry(self, detections: Detections) -> None:
- if (
- len(self._content_list) == 0
- or detections.is_empty()
- or len(detections) == 0
- ):
- return
-
- is_self_class_empty = len(self._class_id_list) == 0
- is_self_confidence_empty = len(self._confidence_list) == 0
- is_detection_class_empty = (
- detections.class_id is None or len(detections.class_id) == 0
- )
- is_detection_confidence_empty = (
- detections.confidence is None or len(detections.confidence) == 0
- )
-
- if is_self_confidence_empty and not is_detection_confidence_empty:
- raise ValueError(
- "Previously stored detections without confidence,"
- " but new detections have it."
- )
- if is_self_class_empty and not is_detection_class_empty:
- raise ValueError(
- "Previously stored detections without class ID,"
- " but new detections have it."
- )
- if not is_self_class_empty and is_detection_class_empty:
- raise ValueError(
- "Started storing detections with class ID, but"
- " new detections do not have it."
- )
- if not is_self_confidence_empty and is_detection_confidence_empty:
- raise ValueError(
- "Started storing detections with confidence,"
- " but new detections do not have it."
- )
-
- def _validate_shape(self, data: npt.NDArray) -> None:
- if self._metric_target == MetricTarget.BOXES:
- if len(data.shape) != 2 or data.shape[1] != 4:
- raise ValueError(f"Invalid xyxy shape: {data.shape}. Expected: (N, 4)")
- elif self._metric_target == MetricTarget.MASKS:
- if len(data.shape) != 3:
- raise ValueError(
- f"Invalid mask shape: {data.shape}. Expected: (N, H, W)"
- )
- elif self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
- if len(data.shape) != 2 or data.shape[1] != 8:
- raise ValueError(f"Invalid obb shape: {data.shape}. Expected: (N, 8)")
- else:
- raise ValueError(f"Invalid metric target: {self._metric_target}")
-
- def _merge_boxes(self):
- """Merge all boxes into a single array."""
- if self._metric_target not in (
- MetricTarget.BOXES,
- MetricTarget.ORIENTED_BOUNDING_BOXES,
- ):
- raise ValueError("Invalid metric target for merging boxes")
- if len(self._content_list) < 2:
- return
- self._content_list = [np.vstack(self._content_list)]
-
- def _merge_masks(self):
- """Merge all masks into a single array."""
- if self._metric_target != MetricTarget.MASKS:
- raise ValueError("Invalid metric target for merging masks")
- if len(self._content_list) < 2:
- return
-
- mew_mask_list = unify_pad_masks_shape(*self._content_list)
- self._content_list = [np.vstack(mew_mask_list)]
-
- def _merge_class_id(self):
- """Merge all class IDs into a single array."""
- if len(self._class_id_list) < 2:
- return
- self._class_id_list = [np.hstack(self._class_id_list)]
-
- def _merge_confidence(self):
- """Merge all confidences into a single array."""
- if len(self._confidence_list) < 2:
- return
- self._confidence_list = [np.hstack(self._confidence_list)]
-
-
-class MetricDataStore:
- """
- Stores internal data for metrics.
-
- Provides a class-agnostic way to access it.
- """
-
- def __init__(self, metric_target: MetricTarget, class_agnostic: bool = False):
- self._metric_target = metric_target
- self._class_agnostic = class_agnostic
- self._data_1: MetricData
- self._data_2: MetricData
- self.reset()
-
- def reset(self) -> None:
- self._data_1 = MetricData(self._metric_target, self._class_agnostic)
- self._data_2 = MetricData(self._metric_target, self._class_agnostic)
-
- def update(self, data_1: Detections, data_2: Detections) -> None:
- """
- Add new data to the store.
-
- Use sv.Detections.empty() if only one set of data is available.
- """
- self._data_1.update(data_1)
- self._data_2.update(data_2)
-
- def get(
- self,
- class_id: Optional[int] = None,
- size_category: ObjectSizeCategory = ObjectSizeCategory.ANY,
- ) -> Tuple[
- Tuple[npt.NDArray, npt.NDArray, npt.NDArray],
- Tuple[npt.NDArray, npt.NDArray, npt.NDArray],
- ]:
- """
- Get the data for both sets, optionally filtered by class_id and/or size.
- """
-
- data_1, class_id_1, confidence_1 = self._data_1.get(class_id, size_category)
- data_2, class_id_2, confidence_2 = self._data_2.get(class_id, size_category)
-
- if self._metric_target == MetricTarget.MASKS:
- data_1, data_2 = unify_pad_masks_shape(data_1, data_2)
-
- return (
- (data_1, class_id_1, confidence_1),
- (data_2, class_id_2, confidence_2),
- )
-
- def get_classes(self) -> Set[int]:
- """Return all class IDs."""
- return self._data_1.get_classes() | self._data_2.get_classes()
diff --git a/supervision/metrics/utils/utils.py b/supervision/metrics/utils/utils.py
index 72065fe5..6ae41a5f 100644
--- a/supervision/metrics/utils/utils.py
+++ b/supervision/metrics/utils/utils.py
@@ -1,9 +1,3 @@
-from typing import List, Tuple
-
-import numpy as np
-import numpy.typing as npt
-
-
def ensure_pandas_installed():
try:
import pandas # noqa
@@ -13,58 +7,3 @@ def ensure_pandas_installed():
" Run `pip install 'supervision[metrics]'` or"
" `poetry add supervision -E metrics`"
)
-
-
-def pad_mask(mask: npt.NDArray, new_shape: Tuple[int, int]) -> npt.NDArray:
- """Pad a mask to a new shape, inserting zeros on the right and bottom."""
- if len(mask.shape) != 3:
- raise ValueError(f"Invalid mask shape: {mask.shape}. Expected: (N, H, W)")
-
- new_mask = np.pad(
- mask,
- (
- (0, 0),
- (0, new_shape[0] - mask.shape[1]),
- (0, new_shape[1] - mask.shape[2]),
- ),
- mode="constant",
- constant_values=0,
- )
-
- return new_mask
-
-
-def unify_pad_masks_shape(
- *masks: npt.NDArray[np.bool_],
-) -> List[npt.NDArray[np.bool_]]:
- """
- Given any number of (N, H, W) mask objects, return copies of the
- same (H, W), padded to the largest dimensions.
-
- Args:
- *masks (np.ndarray): The mask arrays to unify. Each shaped (_, H, W),
-
- Returns:
- List[np.ndarray]: The masks, padded to the largest dimensions.
- Each list element shaped (_, H_max, W_max)
- """
- new_h = 0
- new_w = 0
- for mask in masks:
- if len(mask.shape) != 3:
- raise ValueError(f"Invalid mask shape: {mask.shape}. Expected: (N, H, W)")
-
- _, h, w = mask.shape
- new_h = max(new_h, h)
- new_w = max(new_w, w)
-
- results = []
- for mask in masks:
- results.append(pad_mask(mask, (new_h, new_w)))
-
- return results
-
-
-def len0_like(data: npt.NDArray) -> npt.NDArray:
- """Create an empty array with the same shape as input, but with 0 rows."""
- return np.empty((0, *data.shape[1:]), dtype=data.dtype)