From 3cf96349071b5459ed2bc4dd366e9c16ca639e1b Mon Sep 17 00:00:00 2001 From: LinasKo Date: Mon, 16 Sep 2024 15:14:40 +0300 Subject: [PATCH 1/5] WIP: F1 score --- supervision/metrics/__init__.py | 1 + supervision/metrics/f1_score.py | 830 ++++++++++++++++++++++++++++++++ 2 files changed, 831 insertions(+) create mode 100644 supervision/metrics/f1_score.py diff --git a/supervision/metrics/__init__.py b/supervision/metrics/__init__.py index 5b4b30fc..6696b2eb 100644 --- a/supervision/metrics/__init__.py +++ b/supervision/metrics/__init__.py @@ -3,6 +3,7 @@ from supervision.metrics.mean_average_precision import ( MeanAveragePrecision, MeanAveragePrecisionResult, ) +from supervision.metrics.f1_score import F1Score, F1ScoreResult from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py new file mode 100644 index 00000000..26ae30d2 --- /dev/null +++ b/supervision/metrics/f1_score.py @@ -0,0 +1,830 @@ +from __future__ import annotations + +from copy import deepcopy +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional, Union + +import numpy as np +from matplotlib import pyplot as plt + +from supervision.config import ORIENTED_BOX_COORDINATES +from supervision.detection.core import Detections +from supervision.detection.utils import box_iou_batch, mask_iou_batch +from supervision.draw.color import LEGACY_COLOR_PALETTE +from supervision.metrics.core import Metric, MetricTarget +from supervision.metrics.utils.object_size import ( + ObjectSizeCategory, + get_detection_size_category, +) +from supervision.metrics.utils.utils import ensure_pandas_installed + +if TYPE_CHECKING: + import pandas as pd + + +class F1Score(Metric): + def __init__( + self, + metric_target: MetricTarget = MetricTarget.BOXES + ): + self._metric_target = metric_target + if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + raise NotImplementedError( + "Mean Average Precision is not implemented for oriented bounding boxes." + ) + + self._metric_target = metric_target + self._predictions_list: List[Detections] = [] + self._targets_list: List[Detections] = [] + + def reset(self) -> None: + self._predictions_list = [] + self._targets_list = [] + + def update( + self, + predictions: Union[Detections, List[Detections]], + targets: Union[Detections, List[Detections]], + ) -> F1Score: + if not isinstance(predictions, list): + predictions = [predictions] + if not isinstance(targets, list): + targets = [targets] + + if len(predictions) != len(targets): + raise ValueError( + f"The number of predictions ({len(predictions)}) and" + f" targets ({len(targets)}) during the update must be the same." + ) + + self._predictions_list.extend(predictions) + self._targets_list.extend(targets) + + return self + + def compute(self) -> F1ScoreResult: + result = self._compute(self._predictions_list, self._targets_list) + + small_predictions = [] + small_targets = [] + for predictions, targets in zip(self._predictions_list, self._targets_list): + small_predictions.append( + self._filter_detections_by_size(predictions, ObjectSizeCategory.SMALL) + ) + small_targets.append( + self._filter_detections_by_size(targets, ObjectSizeCategory.SMALL) + ) + result.small_objects = self._compute(small_predictions, small_targets) + + medium_predictions = [] + medium_targets = [] + for predictions, targets in zip(self._predictions_list, self._targets_list): + medium_predictions.append( + self._filter_detections_by_size(predictions, ObjectSizeCategory.MEDIUM) + ) + medium_targets.append( + self._filter_detections_by_size(targets, ObjectSizeCategory.MEDIUM) + ) + result.medium_objects = self._compute(medium_predictions, medium_targets) + + large_predictions = [] + large_targets = [] + for predictions, targets in zip(self._predictions_list, self._targets_list): + large_predictions.append( + self._filter_detections_by_size(predictions, ObjectSizeCategory.LARGE) + ) + large_targets.append( + self._filter_detections_by_size(targets, ObjectSizeCategory.LARGE) + ) + result.large_objects = self._compute(large_predictions, large_targets) + + return result + + def _compute(self, predictions_list: List[Detections], targets_list: List[Detections]) -> F1ScoreResult: + iou_thresholds = np.linspace(0.5, 0.95, 10) + stats = [] + + for predictions, targets in zip(predictions_list, targets_list): + prediction_contents = self._detections_content(predictions) + target_contents = self._detections_content(targets) + + if len(targets) > 0: + if len(predictions) == 0: + stats.append( + ( + np.zeros((0, iou_thresholds.size), dtype=bool), + np.zeros((0,), dtype=np.float32), + np.zeros((0,), dtype=int), + targets.class_id, + ) + ) + + else: + if self._metric_target == MetricTarget.BOXES: + iou = box_iou_batch(target_contents, prediction_contents) + elif self._metric_target == MetricTarget.MASKS: + iou = mask_iou_batch(target_contents, prediction_contents) + else: + raise NotImplementedError( + "Unsupported metric target for IoU calculation" + ) + + matches = self._match_detection_batch( + predictions.class_id, targets.class_id, iou, iou_thresholds + ) + stats.append( + ( + matches, + predictions.confidence, + predictions.class_id, + targets.class_id, + ) + ) + + # Compute average precisions if any matches exist + if stats: + concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] + f1_scores = self._f1_score_per_class(*concatenated_stats) + else: + f1_scores = np.empty((0, len(iou_thresholds)), dtype=np.float32) + + return F1ScoreResult( + f1_scores=f1_scores, + iou_thresholds=iou_thresholds, + # classes=targets.class_id, + f1_per_class=f1_scores, + small_objects=None, + medium_objects=None, + large_objects=None + ) + + @staticmethod + def _f1_score_per_class( + matches: np.ndarray, + prediction_confidence: np.ndarray, + prediction_class_ids: np.ndarray, + true_class_ids: np.ndarray, + ) -> np.ndarray: + eps = 1e-16 + + sorted_indices = np.argsort(-prediction_confidence) + matches = matches[sorted_indices] + prediction_class_ids = prediction_class_ids[sorted_indices] + + unique_classes, class_counts = np.unique(true_class_ids, return_counts=True) + num_classes = unique_classes.shape[0] + + f1_scores = np.zeros((num_classes, matches.shape[1])) + print(f1_scores.shape) + + for class_idx, class_id in enumerate(unique_classes): + is_class = prediction_class_ids == class_id + total_true = class_counts[class_idx] + total_prediction = is_class.sum() + + if total_prediction == 0 or total_true == 0: + continue + + true_positives = matches[is_class].sum(0) + false_positives = (1 - matches[is_class]).sum(0) + false_negatives = total_true - true_positives + + precision = true_positives / (true_positives + false_positives) + recall = true_positives / (true_positives + false_negatives + eps) + print(true_positives) + print(false_positives) + print(false_negatives) + # print(true_positives, false_positives, false_negatives) + + for iou_level_idx in range(matches.shape[1]): + prec = precision[iou_level_idx] + rec = recall[iou_level_idx] + f1 = ( + 2 * (prec * rec) / (prec + rec + eps) + ) + f1_scores[class_idx, iou_level_idx] = f1 + + return f1_scores + + @staticmethod + def _match_detection_batch( + predictions_classes: np.ndarray, + target_classes: np.ndarray, + iou: np.ndarray, + iou_thresholds: np.ndarray, + ) -> np.ndarray: + num_predictions, num_iou_levels = ( + predictions_classes.shape[0], + iou_thresholds.shape[0], + ) + correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) + correct_class = target_classes[:, None] == predictions_classes + + for i, iou_level in enumerate(iou_thresholds): + matched_indices = np.where((iou >= iou_level) & correct_class) + + if matched_indices[0].shape[0]: + combined_indices = np.stack(matched_indices, axis=1) + iou_values = iou[matched_indices][:, None] + matches = np.hstack([combined_indices, iou_values]) + + if matched_indices[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + + correct[matches[:, 1].astype(int), i] = True + + return correct + + def _detections_content(self, detections: Detections) -> np.ndarray: + """Return boxes, masks or oriented bounding boxes from detections.""" + if self._metric_target == MetricTarget.BOXES: + return detections.xyxy + if self._metric_target == MetricTarget.MASKS: + return ( + detections.mask + if detections.mask is not None + else self._make_empty_content() + ) + if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + if obb := detections.data.get(ORIENTED_BOX_COORDINATES): + return np.ndarray(obb, dtype=np.float32) + return self._make_empty_content() + raise ValueError(f"Invalid metric target: {self._metric_target}") + + def _make_empty_content(self) -> np.ndarray: + if self._metric_target == MetricTarget.BOXES: + return np.empty((0, 4), dtype=np.float32) + if self._metric_target == MetricTarget.MASKS: + return np.empty((0, 0, 0), dtype=bool) + if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + return np.empty((0, 8), dtype=np.float32) + raise ValueError(f"Invalid metric target: {self._metric_target}") + + def _filter_detections_by_size( + self, detections: Detections, size_category: ObjectSizeCategory + ) -> Detections: + """Return a copy of detections with contents filtered by object size.""" + new_detections = deepcopy(detections) + if detections.is_empty() or size_category == ObjectSizeCategory.ANY: + return new_detections + + sizes = get_detection_size_category(new_detections, self._metric_target) + size_mask = sizes == size_category.value + + new_detections.xyxy = new_detections.xyxy[size_mask] + if new_detections.mask is not None: + new_detections.mask = new_detections.mask[size_mask] + if new_detections.class_id is not None: + new_detections.class_id = new_detections.class_id[size_mask] + if new_detections.confidence is not None: + new_detections.confidence = new_detections.confidence[size_mask] + if new_detections.tracker_id is not None: + new_detections.tracker_id = new_detections.tracker_id[size_mask] + if new_detections.data is not None: + for key, value in new_detections.data.items(): + new_detections.data[key] = np.array(value)[size_mask] + + return new_detections + + +@dataclass +class F1ScoreResult: + f1_scores: np.ndarray + f1_per_class: np.ndarray + iou_thresholds: np.ndarray + # classes: np.ndarray + + small_objects: Optional[F1ScoreResult] + medium_objects: Optional[F1ScoreResult] + large_objects: Optional[F1ScoreResult] + +# class MeanAveragePrecision(Metric): +# def __init__( +# self, +# metric_target: MetricTarget = MetricTarget.BOXES, +# class_agnostic: bool = False, +# ): +# """ +# Initialize the Mean Average Precision metric. + +# Args: +# metric_target (MetricTarget): The type of detection data to use. +# class_agnostic (bool): Whether to treat all data as a single class. +# """ +# self._metric_target = metric_target +# if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: +# raise NotImplementedError( +# "Mean Average Precision is not implemented for oriented bounding boxes." +# ) + +# self._class_agnostic = class_agnostic + +# self._predictions_list: List[Detections] = [] +# self._targets_list: List[Detections] = [] + +# def reset(self) -> None: +# self._predictions_list = [] +# self._targets_list = [] + +# def update( +# self, +# predictions: Union[Detections, List[Detections]], +# targets: Union[Detections, List[Detections]], +# ) -> MeanAveragePrecision: +# """ +# Add new predictions and targets to the metric, but do not compute the result. + +# Args: +# predictions (Union[Detections, List[Detections]]): The predicted detections. +# targets (Union[Detections, List[Detections]]): The ground-truth detections. + +# Returns: +# (MeanAveragePrecision): The updated metric instance. +# """ +# if not isinstance(predictions, list): +# predictions = [predictions] +# if not isinstance(targets, list): +# targets = [targets] + +# if len(predictions) != len(targets): +# raise ValueError( +# f"The number of predictions ({len(predictions)}) and" +# f" targets ({len(targets)}) during the update must be the same." +# ) + +# self._predictions_list.extend(predictions) +# self._targets_list.extend(targets) + +# return self + +# def compute( +# self, +# ) -> MeanAveragePrecisionResult: +# """ +# Calculate Mean Average Precision based on predicted and ground-truth +# detections at different thresholds. + +# Returns: +# (MeanAveragePrecisionResult): New instance of MeanAveragePrecision. + +# Example: +# ```python +# import supervision as sv +# from supervision.metrics import MeanAveragePrecision + +# predictions = sv.Detections(...) +# targets = sv.Detections(...) + +# map_metric = MeanAveragePrecision() +# map_result = map_metric.update(predictions, targets).compute() + +# print(map_result) +# print(map_result.map50_95) +# map_result.plot() +# ``` +# """ +# result = self._compute(self._predictions_list, self._targets_list) + +# small_predictions = [] +# small_targets = [] +# for predictions, targets in zip(self._predictions_list, self._targets_list): +# small_predictions.append( +# self._filter_detections_by_size(predictions, ObjectSizeCategory.SMALL) +# ) +# small_targets.append( +# self._filter_detections_by_size(targets, ObjectSizeCategory.SMALL) +# ) +# result.small_objects = self._compute(small_predictions, small_targets) + +# medium_predictions = [] +# medium_targets = [] +# for predictions, targets in zip(self._predictions_list, self._targets_list): +# medium_predictions.append( +# self._filter_detections_by_size(predictions, ObjectSizeCategory.MEDIUM) +# ) +# medium_targets.append( +# self._filter_detections_by_size(targets, ObjectSizeCategory.MEDIUM) +# ) +# result.medium_objects = self._compute(medium_predictions, medium_targets) + +# large_predictions = [] +# large_targets = [] +# for predictions, targets in zip(self._predictions_list, self._targets_list): +# large_predictions.append( +# self._filter_detections_by_size(predictions, ObjectSizeCategory.LARGE) +# ) +# large_targets.append( +# self._filter_detections_by_size(targets, ObjectSizeCategory.LARGE) +# ) +# result.large_objects = self._compute(large_predictions, large_targets) + +# return result + +# def _compute( +# self, +# predictions_list: List[Detections], +# targets_list: List[Detections], +# ) -> MeanAveragePrecisionResult: +# iou_thresholds = np.linspace(0.5, 0.95, 10) +# stats = [] + +# for predictions, targets in zip(predictions_list, targets_list): +# prediction_contents = self._detections_content(predictions) +# target_contents = self._detections_content(targets) + +# if len(targets) > 0: +# if len(predictions) == 0: +# stats.append( +# ( +# np.zeros((0, iou_thresholds.size), dtype=bool), +# np.zeros((0,), dtype=np.float32), +# np.zeros((0,), dtype=int), +# targets.class_id, +# ) +# ) + +# else: +# if self._metric_target == MetricTarget.BOXES: +# iou = box_iou_batch(target_contents, prediction_contents) +# elif self._metric_target == MetricTarget.MASKS: +# iou = mask_iou_batch(target_contents, prediction_contents) +# else: +# raise NotImplementedError( +# "Unsupported metric target for IoU calculation" +# ) + +# matches = self._match_detection_batch( +# predictions.class_id, targets.class_id, iou, iou_thresholds +# ) +# stats.append( +# ( +# matches, +# predictions.confidence, +# predictions.class_id, +# targets.class_id, +# ) +# ) + +# # Compute average precisions if any matches exist +# if stats: +# concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] +# average_precisions = self._average_precisions_per_class(*concatenated_stats) +# map50 = average_precisions[:, 0].mean() +# map75 = average_precisions[:, 5].mean() +# map50_95 = average_precisions.mean() +# else: +# map50, map75, map50_95 = 0, 0, 0 +# average_precisions = np.empty((0, len(iou_thresholds)), dtype=np.float32) + +# return MeanAveragePrecisionResult( +# iou_thresholds=iou_thresholds, +# map50_95=map50_95, +# map50=map50, +# map75=map75, +# per_class_ap50_95=average_precisions, +# metric_target=self._metric_target, +# ) + +# @staticmethod +# def _compute_average_precision(recall: np.ndarray, precision: np.ndarray) -> float: +# """ +# Compute the average precision using 101-point interpolation (COCO), given +# the recall and precision curves. + +# Args: +# recall (np.ndarray): The recall curve. +# precision (np.ndarray): The precision curve. + +# Returns: +# (float): Average precision. +# """ +# if len(recall) == 0 and len(precision) == 0: +# return 0.0 + +# recall_levels = np.linspace(0, 1, 101) +# precision_levels = np.zeros_like(recall_levels) +# for r, p in zip(recall[::-1], precision[::-1]): +# precision_levels[recall_levels <= r] = p + +# average_precision = (1 / 100 * precision_levels).sum() +# return average_precision + +# @staticmethod +# def _match_detection_batch( +# predictions_classes: np.ndarray, +# target_classes: np.ndarray, +# iou: np.ndarray, +# iou_thresholds: np.ndarray, +# ) -> np.ndarray: +# num_predictions, num_iou_levels = ( +# predictions_classes.shape[0], +# iou_thresholds.shape[0], +# ) +# correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) +# correct_class = target_classes[:, None] == predictions_classes + +# for i, iou_level in enumerate(iou_thresholds): +# matched_indices = np.where((iou >= iou_level) & correct_class) + +# if matched_indices[0].shape[0]: +# combined_indices = np.stack(matched_indices, axis=1) +# iou_values = iou[matched_indices][:, None] +# matches = np.hstack([combined_indices, iou_values]) + +# if matched_indices[0].shape[0] > 1: +# matches = matches[matches[:, 2].argsort()[::-1]] +# matches = matches[np.unique(matches[:, 1], return_index=True)[1]] +# matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + +# correct[matches[:, 1].astype(int), i] = True + +# return correct + +# @staticmethod +# def _average_precisions_per_class( +# matches: np.ndarray, +# prediction_confidence: np.ndarray, +# prediction_class_ids: np.ndarray, +# true_class_ids: np.ndarray, +# ) -> np.ndarray: +# """ +# Compute the average precision, given the recall and precision curves. +# Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. + +# Args: +# matches (np.ndarray): True positives. +# prediction_confidence (np.ndarray): Objectness value from 0-1. +# prediction_class_ids (np.ndarray): Predicted object classes. +# true_class_ids (np.ndarray): True object classes. +# eps (float, optional): Small value to prevent division by zero. + +# Returns: +# (np.ndarray): Average precision for different IoU levels. +# """ +# eps = 1e-16 + +# sorted_indices = np.argsort(-prediction_confidence) +# matches = matches[sorted_indices] +# prediction_class_ids = prediction_class_ids[sorted_indices] + +# unique_classes, class_counts = np.unique(true_class_ids, return_counts=True) +# num_classes = unique_classes.shape[0] + +# average_precisions = np.zeros((num_classes, matches.shape[1])) + +# for class_idx, class_id in enumerate(unique_classes): +# is_class = prediction_class_ids == class_id +# total_true = class_counts[class_idx] +# total_prediction = is_class.sum() + +# if total_prediction == 0 or total_true == 0: +# continue + +# false_positives = (1 - matches[is_class]).cumsum(0) +# true_positives = matches[is_class].cumsum(0) +# false_negatives = total_true - true_positives + +# recall = true_positives / (true_positives + false_negatives + eps) +# precision = true_positives / (true_positives + false_positives) + +# for iou_level_idx in range(matches.shape[1]): +# average_precisions[class_idx, iou_level_idx] = ( +# MeanAveragePrecision._compute_average_precision( +# recall[:, iou_level_idx], precision[:, iou_level_idx] +# ) +# ) + +# return average_precisions + +# def _detections_content(self, detections: Detections) -> np.ndarray: +# """Return boxes, masks or oriented bounding boxes from detections.""" +# if self._metric_target == MetricTarget.BOXES: +# return detections.xyxy +# if self._metric_target == MetricTarget.MASKS: +# return ( +# detections.mask +# if detections.mask is not None +# else self._make_empty_content() +# ) +# if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: +# if obb := detections.data.get(ORIENTED_BOX_COORDINATES): +# return np.ndarray(obb, dtype=np.float32) +# return self._make_empty_content() +# raise ValueError(f"Invalid metric target: {self._metric_target}") + +# def _make_empty_content(self) -> np.ndarray: +# if self._metric_target == MetricTarget.BOXES: +# return np.empty((0, 4), dtype=np.float32) +# if self._metric_target == MetricTarget.MASKS: +# return np.empty((0, 0, 0), dtype=bool) +# if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: +# return np.empty((0, 8), dtype=np.float32) +# raise ValueError(f"Invalid metric target: {self._metric_target}") + +# def _filter_detections_by_size( +# self, detections: Detections, size_category: ObjectSizeCategory +# ) -> Detections: +# """Return a copy of detections with contents filtered by object size.""" +# new_detections = deepcopy(detections) +# if detections.is_empty() or size_category == ObjectSizeCategory.ANY: +# return new_detections + +# sizes = get_detection_size_category(new_detections, self._metric_target) +# size_mask = sizes == size_category.value + +# new_detections.xyxy = new_detections.xyxy[size_mask] +# if new_detections.mask is not None: +# new_detections.mask = new_detections.mask[size_mask] +# if new_detections.class_id is not None: +# new_detections.class_id = new_detections.class_id[size_mask] +# if new_detections.confidence is not None: +# new_detections.confidence = new_detections.confidence[size_mask] +# if new_detections.tracker_id is not None: +# new_detections.tracker_id = new_detections.tracker_id[size_mask] +# if new_detections.data is not None: +# for key, value in new_detections.data.items(): +# new_detections.data[key] = np.array(value)[size_mask] + +# return new_detections + + +# @dataclass +# class MeanAveragePrecisionResult: +# iou_thresholds: np.ndarray +# """Array of IoU thresholds used in the calculations""" +# map50_95: float +# """Mean Average Precision over IoU thresholds from 0.5 to 0.95""" + +# map50: float +# """Mean Average Precision at IoU threshold of 0.5""" + +# map75: float +# """Mean Average Precision at IoU threshold of 0.75""" + +# per_class_ap50_95: np.ndarray +# """Average precision for each class at different IoU thresholds""" + +# metric_target: MetricTarget +# """ +# Defines the type of data used for the metric - boxes, masks or +# oriented bounding boxes. +# """ + +# small_objects: Optional[MeanAveragePrecisionResult] = None +# """Mean Average Precision results for small objects""" + +# medium_objects: Optional[MeanAveragePrecisionResult] = None +# """Mean Average Precision results for medium objects""" + +# large_objects: Optional[MeanAveragePrecisionResult] = None +# """Mean Average Precision results for large objects""" + +# def __str__(self) -> str: +# """ +# Format the mAP results as a pretty string. + +# Example: +# ```python +# print(map_result) +# ``` +# """ + +# out_str = ( +# f"{self.__class__.__name__}:\n" +# f"iou_thresholds: {self.iou_thresholds}\n" +# f"map50_95: {self.map50_95:.4f}\n" +# f"map50: {self.map50:.4f}\n" +# f"map75: {self.map75:.4f}\n" +# f"per_class_ap50_95:" +# ) + +# for class_id, ap in enumerate(self.per_class_ap50_95): +# out_str += f"\n {class_id}: {ap}" + +# indent = " " +# if self.small_objects is not None: +# indented_str = indent + str(self.small_objects).replace("\n", f"\n{indent}") +# out_str += f"\nSmall objects:\n{indented_str}" +# if self.medium_objects is not None: +# indented_str = indent + str(self.medium_objects).replace( +# "\n", f"\n{indent}" +# ) +# out_str += f"\nMedium objects:\n{indented_str}" +# if self.large_objects is not None: +# indented_str = indent + str(self.large_objects).replace("\n", f"\n{indent}") +# out_str += f"\nLarge objects:\n{indented_str}" + +# return out_str + +# def to_pandas(self) -> "pd.DataFrame": +# """ +# Convert the result to a pandas DataFrame. + +# Returns: +# (pd.DataFrame): The result as a DataFrame. +# """ +# ensure_pandas_installed() +# import pandas as pd + +# pandas_data = { +# "mAP_50_95": self.map50_95, +# "mAP_50": self.map50, +# "mAP_75": self.map75, +# } +# if self.small_objects is not None: +# small_objects_df = self.small_objects.to_pandas() +# for key, value in small_objects_df.items(): +# pandas_data[f"small_objects_{key}"] = value +# if self.medium_objects is not None: +# medium_objects_df = self.medium_objects.to_pandas() +# for key, value in medium_objects_df.items(): +# pandas_data[f"medium_objects_{key}"] = value +# if self.large_objects is not None: +# large_objects_df = self.large_objects.to_pandas() +# for key, value in large_objects_df.items(): +# pandas_data[f"large_objects_{key}"] = value + +# # Average precisions are currently not included in the DataFrame. + +# return pd.DataFrame( +# pandas_data, +# index=[0], +# ) + +# def plot(self): +# """ +# Plot the mAP results. +# """ + +# labels = ["mAP_50_95", "mAP_50", "mAP_75"] +# values = [self.map50_95, self.map50, self.map75] +# colors = [LEGACY_COLOR_PALETTE[0]] * 3 + +# if self.small_objects is not None: +# labels += [ +# "small_objects_mAP_50_95", +# "small_objects_mAP_50", +# "small_objects_mAP_75", +# ] +# values += [ +# self.small_objects.map50_95, +# self.small_objects.map50, +# self.small_objects.map75, +# ] +# colors += [LEGACY_COLOR_PALETTE[3]] * 3 + +# if self.medium_objects is not None: +# labels += [ +# "medium_objects_mAP_50_95", +# "medium_objects_mAP_50", +# "medium_objects_mAP_75", +# ] +# values += [ +# self.medium_objects.map50_95, +# self.medium_objects.map50, +# self.medium_objects.map75, +# ] +# colors += [LEGACY_COLOR_PALETTE[2]] * 3 + +# if self.large_objects is not None: +# labels += [ +# "large_objects_mAP_50_95", +# "large_objects_mAP_50", +# "large_objects_mAP_75", +# ] +# values += [ +# self.large_objects.map50_95, +# self.large_objects.map50, +# self.large_objects.map75, +# ] +# colors += [LEGACY_COLOR_PALETTE[4]] * 3 + +# plt.rcParams["font.family"] = "monospace" + +# _, ax = plt.subplots(figsize=(10, 6)) +# ax.set_ylim(0, 1) +# ax.set_ylabel("Value", fontweight="bold") +# ax.set_title("Mean Average Precision", fontweight="bold") + +# x_positions = range(len(labels)) +# bars = ax.bar(x_positions, values, color=colors, align="center") + +# ax.set_xticks(x_positions) +# ax.set_xticklabels(labels, rotation=45, ha="right") + +# for bar in bars: +# y_value = bar.get_height() +# ax.text( +# bar.get_x() + bar.get_width() / 2, +# y_value + 0.02, +# f"{y_value:.2f}", +# ha="center", +# va="bottom", +# ) + +# plt.rcParams["font.family"] = "sans-serif" + +# plt.tight_layout() +# plt.show() From b9e9adaab35cbe038e5736a25db0a41a08838f64 Mon Sep 17 00:00:00 2001 From: LinasKo Date: Wed, 18 Sep 2024 09:50:52 +0300 Subject: [PATCH 2/5] Add F1 Score, remove dead code, list TODOs for mAP --- supervision/metrics/__init__.py | 9 +- supervision/metrics/core.py | 23 + supervision/metrics/f1_score.py | 941 ++++++------------ supervision/metrics/mean_average_precision.py | 23 +- .../metrics/utils/internal_data_store.py | 300 ------ supervision/metrics/utils/utils.py | 61 -- 6 files changed, 369 insertions(+), 988 deletions(-) delete mode 100644 supervision/metrics/utils/internal_data_store.py diff --git a/supervision/metrics/__init__.py b/supervision/metrics/__init__.py index 6696b2eb..17a6cd48 100644 --- a/supervision/metrics/__init__.py +++ b/supervision/metrics/__init__.py @@ -1,9 +1,14 @@ -from supervision.metrics.core import CLASS_ID_NONE, Metric, MetricTarget +from supervision.metrics.core import ( + CLASS_ID_NONE, + AveragingMethod, + Metric, + MetricTarget, +) +from supervision.metrics.f1_score import F1Score, F1ScoreResult from supervision.metrics.mean_average_precision import ( MeanAveragePrecision, MeanAveragePrecisionResult, ) -from supervision.metrics.f1_score import F1Score, F1ScoreResult from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, diff --git a/supervision/metrics/core.py b/supervision/metrics/core.py index 4f84eb9d..1440fd43 100644 --- a/supervision/metrics/core.py +++ b/supervision/metrics/core.py @@ -48,3 +48,26 @@ class MetricTarget(Enum): BOXES = "boxes" MASKS = "masks" ORIENTED_BOUNDING_BOXES = "obb" + + +class AveragingMethod(Enum): + """ + Defines different ways of averaging the metric results. + + Suppose, before returning the final result, a metric is computed for each class. + How do you combine those to get the final number? + + * MACRO: Calculate the metric for each class and average the results. The simplest + averaging method, but it does not take class imbalance into account. + * MICRO: Calculate the metric globally by counting the total true positives, false + positives, and false negatives. Micro averaging is useful when you want to give + more importance to classes with more samples. It's also more appropriate if you + have an imbalance in the number of instances per class. + * WEIGHTED: Calculate the metric for each class and average the results, weighted by + the number of true instances of each class. Use weighted averaging if you want + to take class imbalance into account. + """ + + MACRO = "macro" + MICRO = "micro" + WEIGHTED = "weighted" diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py index 26ae30d2..e5ecba44 100644 --- a/supervision/metrics/f1_score.py +++ b/supervision/metrics/f1_score.py @@ -2,7 +2,7 @@ from __future__ import annotations from copy import deepcopy from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Tuple, Union import numpy as np from matplotlib import pyplot as plt @@ -11,7 +11,7 @@ from supervision.config import ORIENTED_BOX_COORDINATES from supervision.detection.core import Detections from supervision.detection.utils import box_iou_batch, mask_iou_batch from supervision.draw.color import LEGACY_COLOR_PALETTE -from supervision.metrics.core import Metric, MetricTarget +from supervision.metrics.core import AveragingMethod, Metric, MetricTarget from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -25,7 +25,8 @@ if TYPE_CHECKING: class F1Score(Metric): def __init__( self, - metric_target: MetricTarget = MetricTarget.BOXES + metric_target: MetricTarget = MetricTarget.BOXES, + averaging_method: AveragingMethod = AveragingMethod.WEIGHTED, ): self._metric_target = metric_target if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: @@ -34,6 +35,7 @@ class F1Score(Metric): ) self._metric_target = metric_target + self.averaging_method = averaging_method self._predictions_list: List[Detections] = [] self._targets_list: List[Detections] = [] @@ -61,46 +63,32 @@ class F1Score(Metric): self._targets_list.extend(targets) return self - + def compute(self) -> F1ScoreResult: result = self._compute(self._predictions_list, self._targets_list) - small_predictions = [] - small_targets = [] - for predictions, targets in zip(self._predictions_list, self._targets_list): - small_predictions.append( - self._filter_detections_by_size(predictions, ObjectSizeCategory.SMALL) - ) - small_targets.append( - self._filter_detections_by_size(targets, ObjectSizeCategory.SMALL) - ) + small_predictions, small_targets = self._filter_predictions_and_targets_by_size( + self._predictions_list, self._targets_list, ObjectSizeCategory.SMALL + ) result.small_objects = self._compute(small_predictions, small_targets) - medium_predictions = [] - medium_targets = [] - for predictions, targets in zip(self._predictions_list, self._targets_list): - medium_predictions.append( - self._filter_detections_by_size(predictions, ObjectSizeCategory.MEDIUM) - ) - medium_targets.append( - self._filter_detections_by_size(targets, ObjectSizeCategory.MEDIUM) + medium_predictions, medium_targets = ( + self._filter_predictions_and_targets_by_size( + self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM ) + ) result.medium_objects = self._compute(medium_predictions, medium_targets) - large_predictions = [] - large_targets = [] - for predictions, targets in zip(self._predictions_list, self._targets_list): - large_predictions.append( - self._filter_detections_by_size(predictions, ObjectSizeCategory.LARGE) - ) - large_targets.append( - self._filter_detections_by_size(targets, ObjectSizeCategory.LARGE) - ) + large_predictions, large_targets = self._filter_predictions_and_targets_by_size( + self._predictions_list, self._targets_list, ObjectSizeCategory.LARGE + ) result.large_objects = self._compute(large_predictions, large_targets) return result - - def _compute(self, predictions_list: List[Detections], targets_list: List[Detections]) -> F1ScoreResult: + + def _compute( + self, predictions_list: List[Detections], targets_list: List[Detections] + ) -> F1ScoreResult: iou_thresholds = np.linspace(0.5, 0.95, 10) stats = [] @@ -141,71 +129,68 @@ class F1Score(Metric): ) ) - # Compute average precisions if any matches exist - if stats: - concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] - f1_scores = self._f1_score_per_class(*concatenated_stats) - else: - f1_scores = np.empty((0, len(iou_thresholds)), dtype=np.float32) + if not stats: + return F1ScoreResult( + metric_target=self._metric_target, + averaging_method=self.averaging_method, + f1_scores=np.zeros(iou_thresholds.shape[0]), + f1_per_class=np.zeros((0, iou_thresholds.shape[0])), + iou_thresholds=iou_thresholds, + matched_classes=np.array([], dtype=int), + small_objects=None, + medium_objects=None, + large_objects=None, + ) + + concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] + f1_scores, f1_per_class, unique_classes = self._compute_f1_for_classes( + *concatenated_stats + ) return F1ScoreResult( + metric_target=self._metric_target, + averaging_method=self.averaging_method, f1_scores=f1_scores, + f1_per_class=f1_per_class, iou_thresholds=iou_thresholds, - # classes=targets.class_id, - f1_per_class=f1_scores, + matched_classes=unique_classes, small_objects=None, medium_objects=None, - large_objects=None + large_objects=None, ) - - @staticmethod - def _f1_score_per_class( + + def _compute_f1_for_classes( + self, matches: np.ndarray, prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_class_ids: np.ndarray, - ) -> np.ndarray: - eps = 1e-16 - + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: sorted_indices = np.argsort(-prediction_confidence) matches = matches[sorted_indices] prediction_class_ids = prediction_class_ids[sorted_indices] - unique_classes, class_counts = np.unique(true_class_ids, return_counts=True) - num_classes = unique_classes.shape[0] - f1_scores = np.zeros((num_classes, matches.shape[1])) - print(f1_scores.shape) + # Shape: PxTh,P,C,C -> CxThx3 + confusion_matrix = self._compute_confusion_matrix( + matches, prediction_class_ids, unique_classes, class_counts + ) - for class_idx, class_id in enumerate(unique_classes): - is_class = prediction_class_ids == class_id - total_true = class_counts[class_idx] - total_prediction = is_class.sum() + # Shape: CxThx3 -> CxTh + f1_per_class = self._compute_f1(confusion_matrix) - if total_prediction == 0 or total_true == 0: - continue + # Shape: CxTh -> Th + if self.averaging_method == AveragingMethod.MACRO: + f1_scores = np.mean(f1_per_class, axis=0) + elif self.averaging_method == AveragingMethod.MICRO: + confusion_matrix_merged = confusion_matrix.sum(0) + f1_scores = self._compute_f1(confusion_matrix_merged) + elif self.averaging_method == AveragingMethod.WEIGHTED: + class_counts = class_counts.astype(np.float32) + f1_scores = np.average(f1_per_class, axis=0, weights=class_counts) - true_positives = matches[is_class].sum(0) - false_positives = (1 - matches[is_class]).sum(0) - false_negatives = total_true - true_positives + return f1_scores, f1_per_class, unique_classes - precision = true_positives / (true_positives + false_positives) - recall = true_positives / (true_positives + false_negatives + eps) - print(true_positives) - print(false_positives) - print(false_negatives) - # print(true_positives, false_positives, false_negatives) - - for iou_level_idx in range(matches.shape[1]): - prec = precision[iou_level_idx] - rec = recall[iou_level_idx] - f1 = ( - 2 * (prec * rec) / (prec + rec + eps) - ) - f1_scores[class_idx, iou_level_idx] = f1 - - return f1_scores - @staticmethod def _match_detection_batch( predictions_classes: np.ndarray, @@ -237,6 +222,88 @@ class F1Score(Metric): return correct + @staticmethod + def _compute_confusion_matrix( + sorted_matches: np.ndarray, + sorted_prediction_class_ids: np.ndarray, + unique_classes: np.ndarray, + class_counts: np.ndarray, + ) -> np.ndarray: + """ + Compute the confusion matrix for each class and IoU threshold. + + Assumes the matches and prediction_class_ids are sorted by confidence + in descending order. + + Arguments: + sorted_matches: np.ndarray, bool, shape (P, Th), that is True + if the prediction is a true positive at the given IoU threshold. + sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing + the class id for each prediction. + unique_classes: np.ndarray, int, shape (C,), containing the unique + class ids. + class_counts: np.ndarray, int, shape (C,), containing the number + of true instances for each class. + + Returns: + np.ndarray, shape (C, Th, 3), containing the true positives, false + positives, and false negatives for each class and IoU threshold. + """ + + num_thresholds = sorted_matches.shape[1] + num_classes = unique_classes.shape[0] + + confusion_matrix = np.zeros((num_classes, num_thresholds, 3)) + for class_idx, class_id in enumerate(unique_classes): + is_class = sorted_prediction_class_ids == class_id + num_true = class_counts[class_idx] + num_predictions = is_class.sum() + + if num_predictions == 0: + true_positives = np.zeros(num_thresholds) + false_positives = np.zeros(num_thresholds) + false_negatives = np.full(num_thresholds, num_true) + elif num_true == 0: + true_positives = np.zeros(num_thresholds) + false_positives = np.full(num_thresholds, num_predictions) + false_negatives = np.zeros(num_thresholds) + else: + true_positives = sorted_matches[is_class].sum(0) + false_positives = (1 - sorted_matches[is_class]).sum(0) + false_negatives = num_true - true_positives + confusion_matrix[class_idx] = np.stack( + [true_positives, false_positives, false_negatives], axis=1 + ) + + return confusion_matrix + + @staticmethod + def _compute_f1(confusion_matrix: np.ndarray) -> np.ndarray: + """ + Broadcastable function, computing the F1 score from the confusion matrix. + + Arguments: + confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension + contains the true positives, false positives, and false negatives. + + Returns: + np.ndarray, shape (N, ...), containing the F1 score for each element. + """ + if not confusion_matrix.shape[-1] == 3: + raise ValueError( + f"Confusion matrix must have shape (..., 3), got " + f"{confusion_matrix.shape}" + ) + true_positives = confusion_matrix[..., 0] + false_positives = confusion_matrix[..., 1] + false_negatives = confusion_matrix[..., 2] + + # Alternate formula, avoids multiple zero division checks + denominator = 2 * true_positives + false_positives + false_negatives + f1_score = np.where(denominator == 0, 0, 2 * true_positives / denominator) + + return f1_score + def _detections_content(self, detections: Detections) -> np.ndarray: """Return boxes, masks or oriented bounding boxes from detections.""" if self._metric_target == MetricTarget.BOXES: @@ -245,20 +312,11 @@ class F1Score(Metric): return ( detections.mask if detections.mask is not None - else self._make_empty_content() + else np.empty((0, 0, 0), dtype=bool) ) if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: if obb := detections.data.get(ORIENTED_BOX_COORDINATES): return np.ndarray(obb, dtype=np.float32) - return self._make_empty_content() - raise ValueError(f"Invalid metric target: {self._metric_target}") - - def _make_empty_content(self) -> np.ndarray: - if self._metric_target == MetricTarget.BOXES: - return np.empty((0, 4), dtype=np.float32) - if self._metric_target == MetricTarget.MASKS: - return np.empty((0, 0, 0), dtype=bool) - if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: return np.empty((0, 8), dtype=np.float32) raise ValueError(f"Invalid metric target: {self._metric_target}") @@ -288,543 +346,192 @@ class F1Score(Metric): return new_detections + def _filter_predictions_and_targets_by_size( + self, + predictions_list: List[Detections], + targets_list: List[Detections], + size_category: ObjectSizeCategory, + ) -> Tuple[List[Detections], List[Detections]]: + """ + Filter predictions and targets by object size category. + """ + new_predictions_list = [] + new_targets_list = [] + for predictions, targets in zip(predictions_list, targets_list): + new_predictions_list.append( + self._filter_detections_by_size(predictions, size_category) + ) + new_targets_list.append( + self._filter_detections_by_size(targets, size_category) + ) + return new_predictions_list, new_targets_list + @dataclass class F1ScoreResult: + metric_target: MetricTarget + """ + Defines the type of data used for the metric - boxes, masks or + oriented bounding boxes. + """ + + averaging_method: AveragingMethod + """Which averaging method was used to compute the F1 scores.""" + + @property + def f1_50(self) -> float: + """ + The F1 score at IoU threshold of 0.5. + Considered to be the main result of this metric. + """ + return self.f1_scores[0] + + @property + def f1_75(self) -> float: + """The F1 score at IoU threshold of 0.75.""" + return self.f1_scores[5] + f1_scores: np.ndarray + """The F1 scores, one for each IoU threshold.""" + f1_per_class: np.ndarray + """ + The F1 scores per class and IoU threshold. + Shape: (num_target_classes, num_iou_thresholds) + """ + iou_thresholds: np.ndarray - # classes: np.ndarray + """ + Array of IoU thresholds used in the calculations. + + Note: IoU thresholds define what it means to be a True Positive, so + the F1 score remains high at lower threshold values. + """ + + matched_classes: np.ndarray + """ + The class IDs of classes that were matched during the calculation. + Corresponds to the rows of f1_per_class. + """ small_objects: Optional[F1ScoreResult] + """F1 metric results for small objects.""" + medium_objects: Optional[F1ScoreResult] + """F1 metric results for medium objects.""" + large_objects: Optional[F1ScoreResult] - -# class MeanAveragePrecision(Metric): -# def __init__( -# self, -# metric_target: MetricTarget = MetricTarget.BOXES, -# class_agnostic: bool = False, -# ): -# """ -# Initialize the Mean Average Precision metric. - -# Args: -# metric_target (MetricTarget): The type of detection data to use. -# class_agnostic (bool): Whether to treat all data as a single class. -# """ -# self._metric_target = metric_target -# if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: -# raise NotImplementedError( -# "Mean Average Precision is not implemented for oriented bounding boxes." -# ) - -# self._class_agnostic = class_agnostic - -# self._predictions_list: List[Detections] = [] -# self._targets_list: List[Detections] = [] - -# def reset(self) -> None: -# self._predictions_list = [] -# self._targets_list = [] - -# def update( -# self, -# predictions: Union[Detections, List[Detections]], -# targets: Union[Detections, List[Detections]], -# ) -> MeanAveragePrecision: -# """ -# Add new predictions and targets to the metric, but do not compute the result. - -# Args: -# predictions (Union[Detections, List[Detections]]): The predicted detections. -# targets (Union[Detections, List[Detections]]): The ground-truth detections. - -# Returns: -# (MeanAveragePrecision): The updated metric instance. -# """ -# if not isinstance(predictions, list): -# predictions = [predictions] -# if not isinstance(targets, list): -# targets = [targets] - -# if len(predictions) != len(targets): -# raise ValueError( -# f"The number of predictions ({len(predictions)}) and" -# f" targets ({len(targets)}) during the update must be the same." -# ) - -# self._predictions_list.extend(predictions) -# self._targets_list.extend(targets) - -# return self - -# def compute( -# self, -# ) -> MeanAveragePrecisionResult: -# """ -# Calculate Mean Average Precision based on predicted and ground-truth -# detections at different thresholds. - -# Returns: -# (MeanAveragePrecisionResult): New instance of MeanAveragePrecision. - -# Example: -# ```python -# import supervision as sv -# from supervision.metrics import MeanAveragePrecision - -# predictions = sv.Detections(...) -# targets = sv.Detections(...) - -# map_metric = MeanAveragePrecision() -# map_result = map_metric.update(predictions, targets).compute() - -# print(map_result) -# print(map_result.map50_95) -# map_result.plot() -# ``` -# """ -# result = self._compute(self._predictions_list, self._targets_list) - -# small_predictions = [] -# small_targets = [] -# for predictions, targets in zip(self._predictions_list, self._targets_list): -# small_predictions.append( -# self._filter_detections_by_size(predictions, ObjectSizeCategory.SMALL) -# ) -# small_targets.append( -# self._filter_detections_by_size(targets, ObjectSizeCategory.SMALL) -# ) -# result.small_objects = self._compute(small_predictions, small_targets) - -# medium_predictions = [] -# medium_targets = [] -# for predictions, targets in zip(self._predictions_list, self._targets_list): -# medium_predictions.append( -# self._filter_detections_by_size(predictions, ObjectSizeCategory.MEDIUM) -# ) -# medium_targets.append( -# self._filter_detections_by_size(targets, ObjectSizeCategory.MEDIUM) -# ) -# result.medium_objects = self._compute(medium_predictions, medium_targets) - -# large_predictions = [] -# large_targets = [] -# for predictions, targets in zip(self._predictions_list, self._targets_list): -# large_predictions.append( -# self._filter_detections_by_size(predictions, ObjectSizeCategory.LARGE) -# ) -# large_targets.append( -# self._filter_detections_by_size(targets, ObjectSizeCategory.LARGE) -# ) -# result.large_objects = self._compute(large_predictions, large_targets) - -# return result - -# def _compute( -# self, -# predictions_list: List[Detections], -# targets_list: List[Detections], -# ) -> MeanAveragePrecisionResult: -# iou_thresholds = np.linspace(0.5, 0.95, 10) -# stats = [] - -# for predictions, targets in zip(predictions_list, targets_list): -# prediction_contents = self._detections_content(predictions) -# target_contents = self._detections_content(targets) - -# if len(targets) > 0: -# if len(predictions) == 0: -# stats.append( -# ( -# np.zeros((0, iou_thresholds.size), dtype=bool), -# np.zeros((0,), dtype=np.float32), -# np.zeros((0,), dtype=int), -# targets.class_id, -# ) -# ) - -# else: -# if self._metric_target == MetricTarget.BOXES: -# iou = box_iou_batch(target_contents, prediction_contents) -# elif self._metric_target == MetricTarget.MASKS: -# iou = mask_iou_batch(target_contents, prediction_contents) -# else: -# raise NotImplementedError( -# "Unsupported metric target for IoU calculation" -# ) - -# matches = self._match_detection_batch( -# predictions.class_id, targets.class_id, iou, iou_thresholds -# ) -# stats.append( -# ( -# matches, -# predictions.confidence, -# predictions.class_id, -# targets.class_id, -# ) -# ) - -# # Compute average precisions if any matches exist -# if stats: -# concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] -# average_precisions = self._average_precisions_per_class(*concatenated_stats) -# map50 = average_precisions[:, 0].mean() -# map75 = average_precisions[:, 5].mean() -# map50_95 = average_precisions.mean() -# else: -# map50, map75, map50_95 = 0, 0, 0 -# average_precisions = np.empty((0, len(iou_thresholds)), dtype=np.float32) - -# return MeanAveragePrecisionResult( -# iou_thresholds=iou_thresholds, -# map50_95=map50_95, -# map50=map50, -# map75=map75, -# per_class_ap50_95=average_precisions, -# metric_target=self._metric_target, -# ) - -# @staticmethod -# def _compute_average_precision(recall: np.ndarray, precision: np.ndarray) -> float: -# """ -# Compute the average precision using 101-point interpolation (COCO), given -# the recall and precision curves. - -# Args: -# recall (np.ndarray): The recall curve. -# precision (np.ndarray): The precision curve. - -# Returns: -# (float): Average precision. -# """ -# if len(recall) == 0 and len(precision) == 0: -# return 0.0 - -# recall_levels = np.linspace(0, 1, 101) -# precision_levels = np.zeros_like(recall_levels) -# for r, p in zip(recall[::-1], precision[::-1]): -# precision_levels[recall_levels <= r] = p - -# average_precision = (1 / 100 * precision_levels).sum() -# return average_precision - -# @staticmethod -# def _match_detection_batch( -# predictions_classes: np.ndarray, -# target_classes: np.ndarray, -# iou: np.ndarray, -# iou_thresholds: np.ndarray, -# ) -> np.ndarray: -# num_predictions, num_iou_levels = ( -# predictions_classes.shape[0], -# iou_thresholds.shape[0], -# ) -# correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) -# correct_class = target_classes[:, None] == predictions_classes - -# for i, iou_level in enumerate(iou_thresholds): -# matched_indices = np.where((iou >= iou_level) & correct_class) - -# if matched_indices[0].shape[0]: -# combined_indices = np.stack(matched_indices, axis=1) -# iou_values = iou[matched_indices][:, None] -# matches = np.hstack([combined_indices, iou_values]) - -# if matched_indices[0].shape[0] > 1: -# matches = matches[matches[:, 2].argsort()[::-1]] -# matches = matches[np.unique(matches[:, 1], return_index=True)[1]] -# matches = matches[np.unique(matches[:, 0], return_index=True)[1]] - -# correct[matches[:, 1].astype(int), i] = True - -# return correct - -# @staticmethod -# def _average_precisions_per_class( -# matches: np.ndarray, -# prediction_confidence: np.ndarray, -# prediction_class_ids: np.ndarray, -# true_class_ids: np.ndarray, -# ) -> np.ndarray: -# """ -# Compute the average precision, given the recall and precision curves. -# Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. - -# Args: -# matches (np.ndarray): True positives. -# prediction_confidence (np.ndarray): Objectness value from 0-1. -# prediction_class_ids (np.ndarray): Predicted object classes. -# true_class_ids (np.ndarray): True object classes. -# eps (float, optional): Small value to prevent division by zero. - -# Returns: -# (np.ndarray): Average precision for different IoU levels. -# """ -# eps = 1e-16 - -# sorted_indices = np.argsort(-prediction_confidence) -# matches = matches[sorted_indices] -# prediction_class_ids = prediction_class_ids[sorted_indices] - -# unique_classes, class_counts = np.unique(true_class_ids, return_counts=True) -# num_classes = unique_classes.shape[0] - -# average_precisions = np.zeros((num_classes, matches.shape[1])) - -# for class_idx, class_id in enumerate(unique_classes): -# is_class = prediction_class_ids == class_id -# total_true = class_counts[class_idx] -# total_prediction = is_class.sum() - -# if total_prediction == 0 or total_true == 0: -# continue - -# false_positives = (1 - matches[is_class]).cumsum(0) -# true_positives = matches[is_class].cumsum(0) -# false_negatives = total_true - true_positives - -# recall = true_positives / (true_positives + false_negatives + eps) -# precision = true_positives / (true_positives + false_positives) - -# for iou_level_idx in range(matches.shape[1]): -# average_precisions[class_idx, iou_level_idx] = ( -# MeanAveragePrecision._compute_average_precision( -# recall[:, iou_level_idx], precision[:, iou_level_idx] -# ) -# ) - -# return average_precisions - -# def _detections_content(self, detections: Detections) -> np.ndarray: -# """Return boxes, masks or oriented bounding boxes from detections.""" -# if self._metric_target == MetricTarget.BOXES: -# return detections.xyxy -# if self._metric_target == MetricTarget.MASKS: -# return ( -# detections.mask -# if detections.mask is not None -# else self._make_empty_content() -# ) -# if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: -# if obb := detections.data.get(ORIENTED_BOX_COORDINATES): -# return np.ndarray(obb, dtype=np.float32) -# return self._make_empty_content() -# raise ValueError(f"Invalid metric target: {self._metric_target}") - -# def _make_empty_content(self) -> np.ndarray: -# if self._metric_target == MetricTarget.BOXES: -# return np.empty((0, 4), dtype=np.float32) -# if self._metric_target == MetricTarget.MASKS: -# return np.empty((0, 0, 0), dtype=bool) -# if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: -# return np.empty((0, 8), dtype=np.float32) -# raise ValueError(f"Invalid metric target: {self._metric_target}") - -# def _filter_detections_by_size( -# self, detections: Detections, size_category: ObjectSizeCategory -# ) -> Detections: -# """Return a copy of detections with contents filtered by object size.""" -# new_detections = deepcopy(detections) -# if detections.is_empty() or size_category == ObjectSizeCategory.ANY: -# return new_detections - -# sizes = get_detection_size_category(new_detections, self._metric_target) -# size_mask = sizes == size_category.value - -# new_detections.xyxy = new_detections.xyxy[size_mask] -# if new_detections.mask is not None: -# new_detections.mask = new_detections.mask[size_mask] -# if new_detections.class_id is not None: -# new_detections.class_id = new_detections.class_id[size_mask] -# if new_detections.confidence is not None: -# new_detections.confidence = new_detections.confidence[size_mask] -# if new_detections.tracker_id is not None: -# new_detections.tracker_id = new_detections.tracker_id[size_mask] -# if new_detections.data is not None: -# for key, value in new_detections.data.items(): -# new_detections.data[key] = np.array(value)[size_mask] - -# return new_detections - - -# @dataclass -# class MeanAveragePrecisionResult: -# iou_thresholds: np.ndarray -# """Array of IoU thresholds used in the calculations""" -# map50_95: float -# """Mean Average Precision over IoU thresholds from 0.5 to 0.95""" - -# map50: float -# """Mean Average Precision at IoU threshold of 0.5""" - -# map75: float -# """Mean Average Precision at IoU threshold of 0.75""" - -# per_class_ap50_95: np.ndarray -# """Average precision for each class at different IoU thresholds""" - -# metric_target: MetricTarget -# """ -# Defines the type of data used for the metric - boxes, masks or -# oriented bounding boxes. -# """ - -# small_objects: Optional[MeanAveragePrecisionResult] = None -# """Mean Average Precision results for small objects""" - -# medium_objects: Optional[MeanAveragePrecisionResult] = None -# """Mean Average Precision results for medium objects""" - -# large_objects: Optional[MeanAveragePrecisionResult] = None -# """Mean Average Precision results for large objects""" - -# def __str__(self) -> str: -# """ -# Format the mAP results as a pretty string. - -# Example: -# ```python -# print(map_result) -# ``` -# """ - -# out_str = ( -# f"{self.__class__.__name__}:\n" -# f"iou_thresholds: {self.iou_thresholds}\n" -# f"map50_95: {self.map50_95:.4f}\n" -# f"map50: {self.map50:.4f}\n" -# f"map75: {self.map75:.4f}\n" -# f"per_class_ap50_95:" -# ) - -# for class_id, ap in enumerate(self.per_class_ap50_95): -# out_str += f"\n {class_id}: {ap}" - -# indent = " " -# if self.small_objects is not None: -# indented_str = indent + str(self.small_objects).replace("\n", f"\n{indent}") -# out_str += f"\nSmall objects:\n{indented_str}" -# if self.medium_objects is not None: -# indented_str = indent + str(self.medium_objects).replace( -# "\n", f"\n{indent}" -# ) -# out_str += f"\nMedium objects:\n{indented_str}" -# if self.large_objects is not None: -# indented_str = indent + str(self.large_objects).replace("\n", f"\n{indent}") -# out_str += f"\nLarge objects:\n{indented_str}" - -# return out_str - -# def to_pandas(self) -> "pd.DataFrame": -# """ -# Convert the result to a pandas DataFrame. - -# Returns: -# (pd.DataFrame): The result as a DataFrame. -# """ -# ensure_pandas_installed() -# import pandas as pd - -# pandas_data = { -# "mAP_50_95": self.map50_95, -# "mAP_50": self.map50, -# "mAP_75": self.map75, -# } -# if self.small_objects is not None: -# small_objects_df = self.small_objects.to_pandas() -# for key, value in small_objects_df.items(): -# pandas_data[f"small_objects_{key}"] = value -# if self.medium_objects is not None: -# medium_objects_df = self.medium_objects.to_pandas() -# for key, value in medium_objects_df.items(): -# pandas_data[f"medium_objects_{key}"] = value -# if self.large_objects is not None: -# large_objects_df = self.large_objects.to_pandas() -# for key, value in large_objects_df.items(): -# pandas_data[f"large_objects_{key}"] = value - -# # Average precisions are currently not included in the DataFrame. - -# return pd.DataFrame( -# pandas_data, -# index=[0], -# ) - -# def plot(self): -# """ -# Plot the mAP results. -# """ - -# labels = ["mAP_50_95", "mAP_50", "mAP_75"] -# values = [self.map50_95, self.map50, self.map75] -# colors = [LEGACY_COLOR_PALETTE[0]] * 3 - -# if self.small_objects is not None: -# labels += [ -# "small_objects_mAP_50_95", -# "small_objects_mAP_50", -# "small_objects_mAP_75", -# ] -# values += [ -# self.small_objects.map50_95, -# self.small_objects.map50, -# self.small_objects.map75, -# ] -# colors += [LEGACY_COLOR_PALETTE[3]] * 3 - -# if self.medium_objects is not None: -# labels += [ -# "medium_objects_mAP_50_95", -# "medium_objects_mAP_50", -# "medium_objects_mAP_75", -# ] -# values += [ -# self.medium_objects.map50_95, -# self.medium_objects.map50, -# self.medium_objects.map75, -# ] -# colors += [LEGACY_COLOR_PALETTE[2]] * 3 - -# if self.large_objects is not None: -# labels += [ -# "large_objects_mAP_50_95", -# "large_objects_mAP_50", -# "large_objects_mAP_75", -# ] -# values += [ -# self.large_objects.map50_95, -# self.large_objects.map50, -# self.large_objects.map75, -# ] -# colors += [LEGACY_COLOR_PALETTE[4]] * 3 - -# plt.rcParams["font.family"] = "monospace" - -# _, ax = plt.subplots(figsize=(10, 6)) -# ax.set_ylim(0, 1) -# ax.set_ylabel("Value", fontweight="bold") -# ax.set_title("Mean Average Precision", fontweight="bold") - -# x_positions = range(len(labels)) -# bars = ax.bar(x_positions, values, color=colors, align="center") - -# ax.set_xticks(x_positions) -# ax.set_xticklabels(labels, rotation=45, ha="right") - -# for bar in bars: -# y_value = bar.get_height() -# ax.text( -# bar.get_x() + bar.get_width() / 2, -# y_value + 0.02, -# f"{y_value:.2f}", -# ha="center", -# va="bottom", -# ) - -# plt.rcParams["font.family"] = "sans-serif" - -# plt.tight_layout() -# plt.show() + """F1 metric results for large objects.""" + + def __str__(self) -> str: + out_str = ( + f"{self.__class__.__name__}:\n" + f"Metric target: {self.metric_target}\n" + f"Averaging method: {self.averaging_method}\n" + f"F1@50: {self.f1_50:.4f}\n" + f"F1@75: {self.f1_75:.4f}\n" + f"F1@50:95: {self.f1_scores}\n" + f"IoU thresh: {self.iou_thresholds}\n" + f"F1 per class:\n" + ) + if self.f1_per_class.size == 0: + out_str += " No results\n" + for class_id, f1_of_class in zip(self.matched_classes, self.f1_per_class): + out_str += f" {class_id}: {f1_of_class}\n" + + indent = " " + if self.small_objects is not None: + indented = indent + str(self.small_objects).replace("\n", f"\n{indent}") + out_str += f"\nSmall objects:\n{indented}" + if self.medium_objects is not None: + indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}") + out_str += f"\nMedium objects:\n{indented}" + if self.large_objects is not None: + indented = indent + str(self.large_objects).replace("\n", f"\n{indent}") + out_str += f"\nLarge objects:\n{indented}" + + return out_str + + def to_pandas(self) -> "pd.DataFrame": + ensure_pandas_installed() + import pandas as pd + + pandas_data = { + "F1@50": self.f1_50, + "F1@75": self.f1_75, + } + + if self.small_objects is not None: + small_objects_df = self.small_objects.to_pandas() + for key, value in small_objects_df.items(): + pandas_data[f"small_objects_{key}"] = value + if self.medium_objects is not None: + medium_objects_df = self.medium_objects.to_pandas() + for key, value in medium_objects_df.items(): + pandas_data[f"medium_objects_{key}"] = value + if self.large_objects is not None: + large_objects_df = self.large_objects.to_pandas() + for key, value in large_objects_df.items(): + pandas_data[f"large_objects_{key}"] = value + + return pd.DataFrame(pandas_data, index=[0]) + + def plot(self): + """ + Plot the F1 results. + """ + + labels = ["F1@50", "F1@75"] + values = [self.f1_50, self.f1_75] + colors = [LEGACY_COLOR_PALETTE[0]] * 2 + + if self.small_objects is not None: + small_objects = self.small_objects + labels += ["Small: F1@50", "Small: F1@75"] + values += [small_objects.f1_50, small_objects.f1_75] + colors += [LEGACY_COLOR_PALETTE[3]] * 2 + + if self.medium_objects is not None: + medium_objects = self.medium_objects + labels += ["Medium: F1@50", "Medium: F1@75"] + values += [medium_objects.f1_50, medium_objects.f1_75] + colors += [LEGACY_COLOR_PALETTE[2]] * 2 + + if self.large_objects is not None: + large_objects = self.large_objects + labels += ["Large: F1@50", "Large: F1@75"] + values += [large_objects.f1_50, large_objects.f1_75] + colors += [LEGACY_COLOR_PALETTE[4]] * 2 + + plt.rcParams["font.family"] = "monospace" + + _, ax = plt.subplots(figsize=(10, 6)) + ax.set_ylim(0, 1) + ax.set_ylabel("Value", fontweight="bold") + title = ( + f"F1 Score, by Object Size" + f"\n(target: {self.metric_target.value}," + f" averaging: {self.averaging_method.value})" + ) + ax.set_title(title, fontweight="bold") + + x_positions = range(len(labels)) + bars = ax.bar(x_positions, values, color=colors, align="center") + + ax.set_xticks(x_positions) + ax.set_xticklabels(labels, rotation=45, ha="right") + + for bar in bars: + y_value = bar.get_height() + ax.text( + bar.get_x() + bar.get_width() / 2, + y_value + 0.02, + f"{y_value:.2f}", + ha="center", + va="bottom", + ) + + plt.rcParams["font.family"] = "sans-serif" + + plt.tight_layout() + plt.show() diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 5e6b54bb..2ef4739a 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -421,22 +421,26 @@ class MeanAveragePrecisionResult: f"map75: {self.map75:.4f}\n" f"per_class_ap50_95:" ) + # TODO: newline here? + # TODO: rename to "mAP@50, etc" + # TODO: print thresholds later." + # TODO: print metric target + # TODO: align IoU thresh + # TODO: that's not the class_id. for class_id, ap in enumerate(self.per_class_ap50_95): out_str += f"\n {class_id}: {ap}" indent = " " if self.small_objects is not None: - indented_str = indent + str(self.small_objects).replace("\n", f"\n{indent}") - out_str += f"\nSmall objects:\n{indented_str}" + indented = indent + str(self.small_objects).replace("\n", f"\n{indent}") + out_str += f"\nSmall objects:\n{indented}" if self.medium_objects is not None: - indented_str = indent + str(self.medium_objects).replace( - "\n", f"\n{indent}" - ) - out_str += f"\nMedium objects:\n{indented_str}" + indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}") + out_str += f"\nMedium objects:\n{indented}" if self.large_objects is not None: - indented_str = indent + str(self.large_objects).replace("\n", f"\n{indent}") - out_str += f"\nLarge objects:\n{indented_str}" + indented = indent + str(self.large_objects).replace("\n", f"\n{indent}") + out_str += f"\nLarge objects:\n{indented}" return out_str @@ -455,6 +459,8 @@ class MeanAveragePrecisionResult: "mAP_50": self.map50, "mAP_75": self.map75, } + # TODO: change names to mAP@50, etc + if self.small_objects is not None: small_objects_df = self.small_objects.to_pandas() for key, value in small_objects_df.items(): @@ -481,6 +487,7 @@ class MeanAveragePrecisionResult: """ labels = ["mAP_50_95", "mAP_50", "mAP_75"] + # TODO: change labels to @ notation. values = [self.map50_95, self.map50, self.map75] colors = [LEGACY_COLOR_PALETTE[0]] * 3 diff --git a/supervision/metrics/utils/internal_data_store.py b/supervision/metrics/utils/internal_data_store.py deleted file mode 100644 index f069f506..00000000 --- a/supervision/metrics/utils/internal_data_store.py +++ /dev/null @@ -1,300 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, List, Optional, Set, Tuple - -import numpy as np -import numpy.typing as npt - -from supervision.config import ORIENTED_BOX_COORDINATES -from supervision.metrics.core import CLASS_ID_NONE, MetricTarget -from supervision.metrics.utils.object_size import ( - ObjectSizeCategory, - get_object_size_category, -) -from supervision.metrics.utils.utils import unify_pad_masks_shape - -if TYPE_CHECKING: - from supervision.detection.core import Detections - - -class MetricData: - """ - A container for detection contents, decouple from `Detections`. - """ - - def __init__(self, metric_target: MetricTarget, class_agnostic: bool = False): - self._metric_target = metric_target - self._class_agnostic = class_agnostic - self._content_list: List[npt.NDArray] = [] - self._confidence_list: List[npt.NDArray[np.float32]] = [] - self._class_id_list: List[npt.NDArray[np.int_]] = [] - - def update(self, detections: Detections): - """Add new detections to the store.""" - if detections.is_empty() or len(detections) == 0: - return - - # Relies on Detections to ensure that member vars are equal in length - # or None/empty. - self._validate_new_entry(detections) - - new_content = self._detections_content(detections) - self._validate_shape(new_content) - self._content_list.append(new_content) - - if detections.confidence is not None: - self._confidence_list.append(detections.confidence) - - if detections.class_id is not None: - if self._class_agnostic: - self._class_id_list.append( - np.full(len(detections.class_id), CLASS_ID_NONE, dtype=int) - ) - else: - self._class_id_list.append(detections.class_id) - - def get_classes(self) -> Set[int]: - """Return all class IDs.""" - if self._class_agnostic: - return {CLASS_ID_NONE} - return set.union(*[set(class_id) for class_id in self._class_id_list]) - - def get( - self, class_id: Optional[int] = None, size_category=ObjectSizeCategory.ANY - ) -> Tuple[npt.NDArray, npt.NDArray, npt.NDArray]: - """ - Get the contents, class_ids and confidences, optionally filtered by - class_id and/or size. - - Args: - class_id (Optional[int]): The class ID for the data to retrieve - size_category (ObjectSizeCategory): The size of the objects to retrieve. - - Returns: - (np.ndarray): Boxes, masks or obb, all stacked in a single array. - """ - if self._metric_target in [ - MetricTarget.BOXES, - MetricTarget.ORIENTED_BOUNDING_BOXES, - ]: - self._merge_boxes() - if len(self._content_list) == 0: - content = self._make_empty_content() - else: - content = self._content_list[0] - - elif self._metric_target == MetricTarget.MASKS: - self._merge_masks() - if len(self._content_list) == 0: - content = self._make_empty_content() - else: - content = self._content_list[0] - - size_mask = np.full(len(content), True) - if size_category != ObjectSizeCategory.ANY: - sizes = get_object_size_category(content, self._metric_target) - size_mask = sizes == size_category.value - - class_mask = np.full(len(content), True) - if class_id is not None: - self._merge_class_id() - class_ids = self._class_id_list[0] - class_mask = class_ids == class_id - - content = content[(size_mask & class_mask)] - - self._merge_class_id() - if len(self._class_id_list) == 0: - class_ids = np.array([], dtype=int) - else: - class_ids = self._class_id_list[0] - if len(class_ids) > 0: - class_ids = class_ids[(size_mask & class_mask)] - - self._merge_confidence() - if len(self._confidence_list) == 0: - confidences = np.array([], dtype=float) - else: - confidences = self._confidence_list[0] - if len(confidences) > 0: - confidences = confidences[(size_mask & class_mask)] - - return content, class_ids, confidences - - def get_class_id(self) -> npt.NDArray[np.int_]: - self._merge_class_id() - if len(self._class_id_list) == 0: - return np.array([], dtype=int) - return self._class_id_list[0] - - def get_confidence(self) -> npt.NDArray[np.float32]: - self._merge_confidence() - if len(self._confidence_list) == 0: - return np.array([], dtype=float) - return self._confidence_list[0] - - def _detections_content(self, detections: Detections) -> npt.NDArray: - """Return boxes, masks or oriented bounding boxes from detections.""" - if self._metric_target == MetricTarget.BOXES: - return detections.xyxy - if self._metric_target == MetricTarget.MASKS: - return ( - detections.mask - if detections.mask is not None - else self._make_empty_content() - ) - if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - if obb := detections.data.get(ORIENTED_BOX_COORDINATES): - return np.ndarray(obb, dtype=np.float32) - return self._make_empty_content() - raise ValueError(f"Invalid metric target: {self._metric_target}") - - def _make_empty_content(self) -> npt.NDArray: - if self._metric_target == MetricTarget.BOXES: - return np.empty((0, 4), dtype=np.float32) - if self._metric_target == MetricTarget.MASKS: - return np.empty((0, 0, 0), dtype=bool) - if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - return np.empty((0, 8), dtype=np.float32) - raise ValueError(f"Invalid metric target: {self._metric_target}") - - def _validate_new_entry(self, detections: Detections) -> None: - if ( - len(self._content_list) == 0 - or detections.is_empty() - or len(detections) == 0 - ): - return - - is_self_class_empty = len(self._class_id_list) == 0 - is_self_confidence_empty = len(self._confidence_list) == 0 - is_detection_class_empty = ( - detections.class_id is None or len(detections.class_id) == 0 - ) - is_detection_confidence_empty = ( - detections.confidence is None or len(detections.confidence) == 0 - ) - - if is_self_confidence_empty and not is_detection_confidence_empty: - raise ValueError( - "Previously stored detections without confidence," - " but new detections have it." - ) - if is_self_class_empty and not is_detection_class_empty: - raise ValueError( - "Previously stored detections without class ID," - " but new detections have it." - ) - if not is_self_class_empty and is_detection_class_empty: - raise ValueError( - "Started storing detections with class ID, but" - " new detections do not have it." - ) - if not is_self_confidence_empty and is_detection_confidence_empty: - raise ValueError( - "Started storing detections with confidence," - " but new detections do not have it." - ) - - def _validate_shape(self, data: npt.NDArray) -> None: - if self._metric_target == MetricTarget.BOXES: - if len(data.shape) != 2 or data.shape[1] != 4: - raise ValueError(f"Invalid xyxy shape: {data.shape}. Expected: (N, 4)") - elif self._metric_target == MetricTarget.MASKS: - if len(data.shape) != 3: - raise ValueError( - f"Invalid mask shape: {data.shape}. Expected: (N, H, W)" - ) - elif self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - if len(data.shape) != 2 or data.shape[1] != 8: - raise ValueError(f"Invalid obb shape: {data.shape}. Expected: (N, 8)") - else: - raise ValueError(f"Invalid metric target: {self._metric_target}") - - def _merge_boxes(self): - """Merge all boxes into a single array.""" - if self._metric_target not in ( - MetricTarget.BOXES, - MetricTarget.ORIENTED_BOUNDING_BOXES, - ): - raise ValueError("Invalid metric target for merging boxes") - if len(self._content_list) < 2: - return - self._content_list = [np.vstack(self._content_list)] - - def _merge_masks(self): - """Merge all masks into a single array.""" - if self._metric_target != MetricTarget.MASKS: - raise ValueError("Invalid metric target for merging masks") - if len(self._content_list) < 2: - return - - mew_mask_list = unify_pad_masks_shape(*self._content_list) - self._content_list = [np.vstack(mew_mask_list)] - - def _merge_class_id(self): - """Merge all class IDs into a single array.""" - if len(self._class_id_list) < 2: - return - self._class_id_list = [np.hstack(self._class_id_list)] - - def _merge_confidence(self): - """Merge all confidences into a single array.""" - if len(self._confidence_list) < 2: - return - self._confidence_list = [np.hstack(self._confidence_list)] - - -class MetricDataStore: - """ - Stores internal data for metrics. - - Provides a class-agnostic way to access it. - """ - - def __init__(self, metric_target: MetricTarget, class_agnostic: bool = False): - self._metric_target = metric_target - self._class_agnostic = class_agnostic - self._data_1: MetricData - self._data_2: MetricData - self.reset() - - def reset(self) -> None: - self._data_1 = MetricData(self._metric_target, self._class_agnostic) - self._data_2 = MetricData(self._metric_target, self._class_agnostic) - - def update(self, data_1: Detections, data_2: Detections) -> None: - """ - Add new data to the store. - - Use sv.Detections.empty() if only one set of data is available. - """ - self._data_1.update(data_1) - self._data_2.update(data_2) - - def get( - self, - class_id: Optional[int] = None, - size_category: ObjectSizeCategory = ObjectSizeCategory.ANY, - ) -> Tuple[ - Tuple[npt.NDArray, npt.NDArray, npt.NDArray], - Tuple[npt.NDArray, npt.NDArray, npt.NDArray], - ]: - """ - Get the data for both sets, optionally filtered by class_id and/or size. - """ - - data_1, class_id_1, confidence_1 = self._data_1.get(class_id, size_category) - data_2, class_id_2, confidence_2 = self._data_2.get(class_id, size_category) - - if self._metric_target == MetricTarget.MASKS: - data_1, data_2 = unify_pad_masks_shape(data_1, data_2) - - return ( - (data_1, class_id_1, confidence_1), - (data_2, class_id_2, confidence_2), - ) - - def get_classes(self) -> Set[int]: - """Return all class IDs.""" - return self._data_1.get_classes() | self._data_2.get_classes() diff --git a/supervision/metrics/utils/utils.py b/supervision/metrics/utils/utils.py index 72065fe5..6ae41a5f 100644 --- a/supervision/metrics/utils/utils.py +++ b/supervision/metrics/utils/utils.py @@ -1,9 +1,3 @@ -from typing import List, Tuple - -import numpy as np -import numpy.typing as npt - - def ensure_pandas_installed(): try: import pandas # noqa @@ -13,58 +7,3 @@ def ensure_pandas_installed(): " Run `pip install 'supervision[metrics]'` or" " `poetry add supervision -E metrics`" ) - - -def pad_mask(mask: npt.NDArray, new_shape: Tuple[int, int]) -> npt.NDArray: - """Pad a mask to a new shape, inserting zeros on the right and bottom.""" - if len(mask.shape) != 3: - raise ValueError(f"Invalid mask shape: {mask.shape}. Expected: (N, H, W)") - - new_mask = np.pad( - mask, - ( - (0, 0), - (0, new_shape[0] - mask.shape[1]), - (0, new_shape[1] - mask.shape[2]), - ), - mode="constant", - constant_values=0, - ) - - return new_mask - - -def unify_pad_masks_shape( - *masks: npt.NDArray[np.bool_], -) -> List[npt.NDArray[np.bool_]]: - """ - Given any number of (N, H, W) mask objects, return copies of the - same (H, W), padded to the largest dimensions. - - Args: - *masks (np.ndarray): The mask arrays to unify. Each shaped (_, H, W), - - Returns: - List[np.ndarray]: The masks, padded to the largest dimensions. - Each list element shaped (_, H_max, W_max) - """ - new_h = 0 - new_w = 0 - for mask in masks: - if len(mask.shape) != 3: - raise ValueError(f"Invalid mask shape: {mask.shape}. Expected: (N, H, W)") - - _, h, w = mask.shape - new_h = max(new_h, h) - new_w = max(new_w, w) - - results = [] - for mask in masks: - results.append(pad_mask(mask, (new_h, new_w))) - - return results - - -def len0_like(data: npt.NDArray) -> npt.NDArray: - """Create an empty array with the same shape as input, but with 0 rows.""" - return np.empty((0, *data.shape[1:]), dtype=data.dtype) From d25016e42848104a0d5fc8126f71f7e6df01bbf8 Mon Sep 17 00:00:00 2001 From: LinasKo Date: Wed, 18 Sep 2024 12:02:19 +0300 Subject: [PATCH 3/5] Typo: mAP in F1 score class --- supervision/metrics/f1_score.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py index e5ecba44..e0ae9168 100644 --- a/supervision/metrics/f1_score.py +++ b/supervision/metrics/f1_score.py @@ -31,7 +31,7 @@ class F1Score(Metric): self._metric_target = metric_target if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: raise NotImplementedError( - "Mean Average Precision is not implemented for oriented bounding boxes." + "F1 score is not implemented for oriented bounding boxes." ) self._metric_target = metric_target From 9d2dc08e3438daf02f9718b0e1d064205d39b20c Mon Sep 17 00:00:00 2001 From: LinasKo Date: Wed, 18 Sep 2024 16:40:49 +0300 Subject: [PATCH 4/5] Change mAP result to match F1 result --- supervision/metrics/f1_score.py | 8 +- supervision/metrics/mean_average_precision.py | 130 +++++++++--------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py index e0ae9168..e0ddbc7b 100644 --- a/supervision/metrics/f1_score.py +++ b/supervision/metrics/f1_score.py @@ -428,10 +428,10 @@ class F1ScoreResult: f"{self.__class__.__name__}:\n" f"Metric target: {self.metric_target}\n" f"Averaging method: {self.averaging_method}\n" - f"F1@50: {self.f1_50:.4f}\n" - f"F1@75: {self.f1_75:.4f}\n" - f"F1@50:95: {self.f1_scores}\n" - f"IoU thresh: {self.iou_thresholds}\n" + f"F1 @ 50: {self.f1_50:.4f}\n" + f"F1 @ 75: {self.f1_75:.4f}\n" + f"F1 @ thresh: {self.f1_scores}\n" + f"IoU thresh: {self.iou_thresholds}\n" f"F1 per class:\n" ) if self.f1_per_class.size == 0: diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 2ef4739a..6d797730 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -2,7 +2,7 @@ from __future__ import annotations from copy import deepcopy from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Tuple, Union import numpy as np from matplotlib import pyplot as plt @@ -192,21 +192,21 @@ class MeanAveragePrecision(Metric): # Compute average precisions if any matches exist if stats: concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] - average_precisions = self._average_precisions_per_class(*concatenated_stats) - map50 = average_precisions[:, 0].mean() - map75 = average_precisions[:, 5].mean() - map50_95 = average_precisions.mean() + average_precisions, unique_classes = self._average_precisions_per_class( + *concatenated_stats + ) + mAP_scores = np.mean(average_precisions, axis=0) else: - map50, map75, map50_95 = 0, 0, 0 + mAP_scores = np.zeros((10,), dtype=np.float32) + unique_classes = np.empty((0,), dtype=int) average_precisions = np.empty((0, len(iou_thresholds)), dtype=np.float32) return MeanAveragePrecisionResult( - iou_thresholds=iou_thresholds, - map50_95=map50_95, - map50=map50, - map75=map75, - per_class_ap50_95=average_precisions, metric_target=self._metric_target, + mAP_scores=mAP_scores, + iou_thresholds=iou_thresholds, + matched_classes=unique_classes, + ap_per_class=average_precisions, ) @staticmethod @@ -270,7 +270,7 @@ class MeanAveragePrecision(Metric): prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_class_ids: np.ndarray, - ) -> np.ndarray: + ) -> Tuple[np.ndarray, np.ndarray]: """ Compute the average precision, given the recall and precision curves. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. @@ -283,7 +283,8 @@ class MeanAveragePrecision(Metric): eps (float, optional): Small value to prevent division by zero. Returns: - (np.ndarray): Average precision for different IoU levels. + (Tuple[np.ndarray, np.ndarray]): Average precision for different + IoU levels, and an array of class IDs that were matched. """ eps = 1e-16 @@ -318,7 +319,7 @@ class MeanAveragePrecision(Metric): ) ) - return average_precisions + return average_precisions, unique_classes def _detections_content(self, detections: Detections) -> np.ndarray: """Return boxes, masks or oriented bounding boxes from detections.""" @@ -374,24 +375,40 @@ class MeanAveragePrecision(Metric): @dataclass class MeanAveragePrecisionResult: - iou_thresholds: np.ndarray - """Array of IoU thresholds used in the calculations""" - map50_95: float - """Mean Average Precision over IoU thresholds from 0.5 to 0.95""" - - map50: float - """Mean Average Precision at IoU threshold of 0.5""" - - map75: float - """Mean Average Precision at IoU threshold of 0.75""" - - per_class_ap50_95: np.ndarray - """Average precision for each class at different IoU thresholds""" - metric_target: MetricTarget """ - Defines the type of data used for the metric - boxes, masks or - oriented bounding boxes. + The class IDs of classes that were matched during the calculation. + Corresponds to the rows of per_class_ap50_95. + """ + + @property + def map50_95(self) -> float: + """Mean Average Precision over IoU thresholds from 0.5 to 0.95""" + return self.mAP_scores.mean() + + @property + def map50(self) -> float: + """Mean Average Precision at IoU threshold of 0.5""" + return self.mAP_scores[0] + + @property + def map75(self) -> float: + """Mean Average Precision at IoU threshold of 0.75""" + return self.mAP_scores[5] + + mAP_scores: np.ndarray + """Mean Average Precision scores at different IoU thresholds""" + + ap_per_class: np.ndarray + """Average precision for each class at different IoU thresholds""" + + iou_thresholds: np.ndarray + """Array of IoU thresholds used in the calculations""" + + matched_classes: np.ndarray + """ + The class IDs of classes that were matched during the calculation. + Corresponds to the rows of ap_per_class. """ small_objects: Optional[MeanAveragePrecisionResult] = None @@ -415,21 +432,18 @@ class MeanAveragePrecisionResult: out_str = ( f"{self.__class__.__name__}:\n" - f"iou_thresholds: {self.iou_thresholds}\n" - f"map50_95: {self.map50_95:.4f}\n" - f"map50: {self.map50:.4f}\n" - f"map75: {self.map75:.4f}\n" - f"per_class_ap50_95:" + f"Metric target: {self.metric_target}\n" + f"mAP @ 50:95: {self.map50_95:.4f}\n" + f"mAP @ 50: {self.map50:.4f}\n" + f"mAP @ 75: {self.map75:.4f}\n" + f"mAP scores: {self.mAP_scores}\n" + f"IoU thresh: {self.iou_thresholds}\n" + f"AP per class:\n" ) - # TODO: newline here? - # TODO: rename to "mAP@50, etc" - # TODO: print thresholds later." - # TODO: print metric target - # TODO: align IoU thresh - - # TODO: that's not the class_id. - for class_id, ap in enumerate(self.per_class_ap50_95): - out_str += f"\n {class_id}: {ap}" + if self.ap_per_class.size == 0: + out_str += " No results\n" + for class_id, ap_of_class in zip(self.matched_classes, self.ap_per_class): + out_str += f" {class_id}: {ap_of_class}\n" indent = " " if self.small_objects is not None: @@ -455,11 +469,10 @@ class MeanAveragePrecisionResult: import pandas as pd pandas_data = { - "mAP_50_95": self.map50_95, - "mAP_50": self.map50, - "mAP_75": self.map75, + "mAP@50:95": self.map50_95, + "mAP@50": self.map50, + "mAP@75": self.map75, } - # TODO: change names to mAP@50, etc if self.small_objects is not None: small_objects_df = self.small_objects.to_pandas() @@ -486,17 +499,12 @@ class MeanAveragePrecisionResult: Plot the mAP results. """ - labels = ["mAP_50_95", "mAP_50", "mAP_75"] - # TODO: change labels to @ notation. + labels = ["mAP@50:95", "mAP@50", "mAP@75"] values = [self.map50_95, self.map50, self.map75] colors = [LEGACY_COLOR_PALETTE[0]] * 3 if self.small_objects is not None: - labels += [ - "small_objects_mAP_50_95", - "small_objects_mAP_50", - "small_objects_mAP_75", - ] + labels += ["Small: mAP@50:95", "Small: mAP@50", "Small: mAP@75"] values += [ self.small_objects.map50_95, self.small_objects.map50, @@ -505,11 +513,7 @@ class MeanAveragePrecisionResult: colors += [LEGACY_COLOR_PALETTE[3]] * 3 if self.medium_objects is not None: - labels += [ - "medium_objects_mAP_50_95", - "medium_objects_mAP_50", - "medium_objects_mAP_75", - ] + labels += ["Medium: mAP@50:95", "Medium: mAP@50", "Medium: mAP@75"] values += [ self.medium_objects.map50_95, self.medium_objects.map50, @@ -518,11 +522,7 @@ class MeanAveragePrecisionResult: colors += [LEGACY_COLOR_PALETTE[2]] * 3 if self.large_objects is not None: - labels += [ - "large_objects_mAP_50_95", - "large_objects_mAP_50", - "large_objects_mAP_75", - ] + labels += ["Large: mAP@50:95", "Large: mAP@50", "Large: mAP@75"] values += [ self.large_objects.map50_95, self.large_objects.map50, From 7f85963314314d5a256dc099f92c9672e057faa3 Mon Sep 17 00:00:00 2001 From: LinasKo Date: Fri, 20 Sep 2024 14:50:46 +0300 Subject: [PATCH 5/5] Add F1 to docs, rewrite the docstrings --- docs/index.md | 45 ++++++------ docs/metrics/f1_score.md | 18 +++++ mkdocs.yml | 1 + supervision/metrics/f1_score.py | 72 ++++++++++--------- supervision/metrics/mean_average_precision.py | 51 ++++++------- 5 files changed, 109 insertions(+), 78 deletions(-) create mode 100644 docs/metrics/f1_score.md diff --git a/docs/index.md b/docs/index.md index e681f605..60de4214 100644 --- a/docs/index.md +++ b/docs/index.md @@ -110,49 +110,50 @@ You can install `supervision` in a
-- __Detect and Annotate__ +- **Detect and Annotate** - --- + *** - Annotate predictions from a range of object detection and segmentation models + Annotate predictions from a range of object detection and segmentation models - [:octicons-arrow-right-24: Tutorial](how_to/detect_and_annotate.md) + [:octicons-arrow-right-24: Tutorial](how_to/detect_and_annotate.md) -- __Track Objects__ +- **Track Objects** - --- + *** - Discover how to enhance video analysis by implementing seamless object tracking + Discover how to enhance video analysis by implementing seamless object tracking - [:octicons-arrow-right-24: Tutorial](how_to/track_objects.md) + [:octicons-arrow-right-24: Tutorial](how_to/track_objects.md) -- __Detect Small Objects__ +- **Detect Small Objects** - --- + *** - Learn how to detect small objects in images + Learn how to detect small objects in images - [:octicons-arrow-right-24: Tutorial](how_to/detect_small_objects.md) + [:octicons-arrow-right-24: Tutorial](how_to/detect_small_objects.md) -- > __Count Objects Crossing Line__ +- **Count Objects Crossing Line** - --- + [:octicons-arrow-right-24: Notebook](https://supervision.roboflow.com/latest/notebooks/count-objects-crossing-the-line/) - Explore methods to accurately count and analyze objects crossing a predefined line + *** -- > __Filter Objects in Zone__ + Explore methods to accurately count and analyze objects crossing a predefined line - --- +- > **Filter Objects in Zone** - Master the techniques to selectively filter and focus on objects within a specific zone + *** -- **Cheatsheet** + Master the techniques to selectively filter and focus on objects within a specific zone - *** +- **Cheatsheet** - Access a quick reference guide to the most common `supervision` functions + *** - [:octicons-arrow-right-24: Cheatsheet](https://roboflow.github.io/cheatsheet-supervision/) + Access a quick reference guide to the most common `supervision` functions + [:octicons-arrow-right-24: Cheatsheet](https://roboflow.github.io/cheatsheet-supervision/)
diff --git a/docs/metrics/f1_score.md b/docs/metrics/f1_score.md new file mode 100644 index 00000000..e31a9557 --- /dev/null +++ b/docs/metrics/f1_score.md @@ -0,0 +1,18 @@ +--- +comments: true +status: new +--- + +# F1 Score + +
+

F1Score

+
+ +:::supervision.metrics.f1_score.F1Score + +
+

F1ScoreResult

+
+ +:::supervision.metrics.f1_score.F1ScoreResult diff --git a/mkdocs.yml b/mkdocs.yml index 097889a2..3cd86759 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -66,6 +66,7 @@ nav: - Utils: datasets/utils.md - Metrics: - mAP: metrics/mean_average_precision.md + - F1 Score: metrics/f1_score.md - Legacy Metrics: detection/metrics.md - Utils: - Video: utils/video.md diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py index e0ddbc7b..2ca5bca5 100644 --- a/supervision/metrics/f1_score.py +++ b/supervision/metrics/f1_score.py @@ -369,61 +369,63 @@ class F1Score(Metric): @dataclass class F1ScoreResult: - metric_target: MetricTarget """ - Defines the type of data used for the metric - boxes, masks or - oriented bounding boxes. + The results of the F1 score metric calculation. + + Defaults to `0` if no detections or targets were provided. + Provides a custom `__str__` method for pretty printing. + + Attributes: + metric_target (MetricTarget): the type of data used for the metric - + boxes, masks or oriented bounding boxes. + averaging_method (AveragingMethod): the averaging method used to compute the + F1 scores. Determines how the F1 scores are aggregated across classes. + f1_50 (float): the F1 score at IoU threshold of `0.5`. + f1_75 (float): the F1 score at IoU threshold of `0.75`. + f1_scores (np.ndarray): the F1 scores at each IoU threshold. + Shape: `(num_iou_thresholds,)` + f1_per_class (np.ndarray): the F1 scores per class and IoU threshold. + Shape: `(num_target_classes, num_iou_thresholds)` + iou_thresholds (np.ndarray): the IoU thresholds used in the calculations. + matched_classes (np.ndarray): the class IDs of all matched classes. + Corresponds to the rows of `f1_per_class`. + small_objects (Optional[F1ScoreResult]): the F1 metric results + for small objects. + medium_objects (Optional[F1ScoreResult]): the F1 metric results + for medium objects. + large_objects (Optional[F1ScoreResult]): the F1 metric results + for large objects. """ + metric_target: MetricTarget averaging_method: AveragingMethod - """Which averaging method was used to compute the F1 scores.""" @property def f1_50(self) -> float: - """ - The F1 score at IoU threshold of 0.5. - Considered to be the main result of this metric. - """ return self.f1_scores[0] @property def f1_75(self) -> float: - """The F1 score at IoU threshold of 0.75.""" return self.f1_scores[5] f1_scores: np.ndarray - """The F1 scores, one for each IoU threshold.""" - f1_per_class: np.ndarray - """ - The F1 scores per class and IoU threshold. - Shape: (num_target_classes, num_iou_thresholds) - """ - iou_thresholds: np.ndarray - """ - Array of IoU thresholds used in the calculations. - - Note: IoU thresholds define what it means to be a True Positive, so - the F1 score remains high at lower threshold values. - """ - matched_classes: np.ndarray - """ - The class IDs of classes that were matched during the calculation. - Corresponds to the rows of f1_per_class. - """ small_objects: Optional[F1ScoreResult] - """F1 metric results for small objects.""" - medium_objects: Optional[F1ScoreResult] - """F1 metric results for medium objects.""" - large_objects: Optional[F1ScoreResult] - """F1 metric results for large objects.""" def __str__(self) -> str: + """ + Format as a pretty string. + + Example: + ```python + print(f1_result) + ``` + """ out_str = ( f"{self.__class__.__name__}:\n" f"Metric target: {self.metric_target}\n" @@ -453,6 +455,12 @@ class F1ScoreResult: return out_str def to_pandas(self) -> "pd.DataFrame": + """ + Convert the result to a pandas DataFrame. + + Returns: + (pd.DataFrame): The result as a DataFrame. + """ ensure_pandas_installed() import pandas as pd diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 6d797730..04a5fe9d 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -375,54 +375,57 @@ class MeanAveragePrecision(Metric): @dataclass class MeanAveragePrecisionResult: + """ + The result of the Mean Average Precision calculation. + + Defaults to `0` when no detections or targets are present. + + Attributes: + metric_target (MetricTarget): the type of data used for the metric - + boxes, masks or oriented bounding boxes. + mAP_map50_95 (float): the mAP score at IoU thresholds from `0.5` to `0.95`. + mAP_map50 (float): the mAP score at IoU threshold of `0.5`. + mAP_map75 (float): the mAP score at IoU threshold of `0.75`. + mAP_scores (np.ndarray): the mAP scores at each IoU threshold. + Shape: `(num_iou_thresholds,)` + ap_per_class (np.ndarray): the average precision scores per + class and IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)` + iou_thresholds (np.ndarray): the IoU thresholds used in the calculations. + matched_classes (np.ndarray): the class IDs of all matched classes. + Corresponds to the rows of `ap_per_class`. + small_objects (Optional[MeanAveragePrecisionResult]): the mAP results + for small objects. + medium_objects (Optional[MeanAveragePrecisionResult]): the mAP results + for medium objects. + large_objects (Optional[MeanAveragePrecisionResult]): the mAP results + for large objects. + """ + metric_target: MetricTarget - """ - The class IDs of classes that were matched during the calculation. - Corresponds to the rows of per_class_ap50_95. - """ @property def map50_95(self) -> float: - """Mean Average Precision over IoU thresholds from 0.5 to 0.95""" return self.mAP_scores.mean() @property def map50(self) -> float: - """Mean Average Precision at IoU threshold of 0.5""" return self.mAP_scores[0] @property def map75(self) -> float: - """Mean Average Precision at IoU threshold of 0.75""" return self.mAP_scores[5] mAP_scores: np.ndarray - """Mean Average Precision scores at different IoU thresholds""" - ap_per_class: np.ndarray - """Average precision for each class at different IoU thresholds""" - iou_thresholds: np.ndarray - """Array of IoU thresholds used in the calculations""" - matched_classes: np.ndarray - """ - The class IDs of classes that were matched during the calculation. - Corresponds to the rows of ap_per_class. - """ - small_objects: Optional[MeanAveragePrecisionResult] = None - """Mean Average Precision results for small objects""" - medium_objects: Optional[MeanAveragePrecisionResult] = None - """Mean Average Precision results for medium objects""" - large_objects: Optional[MeanAveragePrecisionResult] = None - """Mean Average Precision results for large objects""" def __str__(self) -> str: """ - Format the mAP results as a pretty string. + Format as a pretty string. Example: ```python