okooooooo/supervision
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1521 from roboflow/feat/f1-metric

Browse Source 
F1 metric
This commit is contained in:
LinasKo authored and GitHub committed 2024-09-20 19:12:14 +03:00
2 parents c94cfe2031 7de1f5855d
commit fa8378e50c
9 files changed
+692 -450
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/index.md
+18 -18
View File
@@ -103,48 +103,48 @@ You can install `supervision` in a
- **Detect and Annotate**
---
***
Annotate predictions from a range of object detection and segmentation models
Annotate predictions from a range of object detection and segmentation models
[:octicons-arrow-right-24: Tutorial](how_to/detect_and_annotate.md)
[:octicons-arrow-right-24: Tutorial](how_to/detect_and_annotate.md)
- **Track Objects**
---
***
Discover how to enhance video analysis by implementing seamless object tracking
Discover how to enhance video analysis by implementing seamless object tracking
[:octicons-arrow-right-24: Tutorial](how_to/track_objects.md)
[:octicons-arrow-right-24: Tutorial](how_to/track_objects.md)
- **Detect Small Objects**
---
***
Learn how to detect small objects in images
Learn how to detect small objects in images
[:octicons-arrow-right-24: Tutorial](how_to/detect_small_objects.md)
[:octicons-arrow-right-24: Tutorial](how_to/detect_small_objects.md)
- **Count Objects Crossing Line**
---
***
Explore methods to accurately count and analyze objects crossing a predefined line
Explore methods to accurately count and analyze objects crossing a predefined line
[:octicons-arrow-right-24: Notebook](https://supervision.roboflow.com/latest/notebooks/count-objects-crossing-the-line/)
[:octicons-arrow-right-24: Notebook](https://supervision.roboflow.com/latest/notebooks/count-objects-crossing-the-line/)
- **Filter Objects in Zone**
- > **Filter Objects in Zone**
---
***
Master the techniques to selectively filter and focus on objects within a specific zone
Master the techniques to selectively filter and focus on objects within a specific zone
- **Cheatsheet**
---
***
Access a quick reference guide to the most common `supervision` functions
Access a quick reference guide to the most common `supervision` functions
[:octicons-arrow-right-24: Cheatsheet](https://roboflow.github.io/cheatsheet-supervision/)
[:octicons-arrow-right-24: Cheatsheet](https://roboflow.github.io/cheatsheet-supervision/)
</div>
docs/metrics/f1_score.md
+18
View File
@@ -0,0 +1,18 @@
---
comments: true
status: new
---
# F1 Score
<div class="md-typeset">
<h2><a href="#supervision.metrics.f1_score.F1Score">F1Score</a></h2>
</div>
:::supervision.metrics.f1_score.F1Score
<div class="md-typeset">
<h2><a href="#supervision.metrics.f1_score.F1ScoreResult">F1ScoreResult</a></h2>
</div>
:::supervision.metrics.f1_score.F1ScoreResult
mkdocs.yml
+1
View File
@@ -66,6 +66,7 @@ nav:
- Utils: datasets/utils.md
- Metrics:
- mAP: metrics/mean_average_precision.md
- F1 Score: metrics/f1_score.md
- Legacy Metrics: detection/metrics.md
- Utils:
- Video: utils/video.md
supervision/metrics/__init__.py
+7 -1
View File
@@ -1,4 +1,10 @@
from supervision.metrics.core import CLASS_ID_NONE, Metric, MetricTarget
from supervision.metrics.core import (
CLASS_ID_NONE,
AveragingMethod,
Metric,
MetricTarget,
)
from supervision.metrics.f1_score import F1Score, F1ScoreResult
from supervision.metrics.mean_average_precision import (
MeanAveragePrecision,
MeanAveragePrecisionResult,
supervision/metrics/core.py
+23
View File
@@ -48,3 +48,26 @@ class MetricTarget(Enum):
BOXES = "boxes"
MASKS = "masks"
ORIENTED_BOUNDING_BOXES = "obb"
class AveragingMethod(Enum):
"""
Defines different ways of averaging the metric results.
Suppose, before returning the final result, a metric is computed for each class.
How do you combine those to get the final number?
* MACRO: Calculate the metric for each class and average the results. The simplest
averaging method, but it does not take class imbalance into account.
* MICRO: Calculate the metric globally by counting the total true positives, false
positives, and false negatives. Micro averaging is useful when you want to give
more importance to classes with more samples. It's also more appropriate if you
have an imbalance in the number of instances per class.
* WEIGHTED: Calculate the metric for each class and average the results, weighted by
the number of true instances of each class. Use weighted averaging if you want
to take class imbalance into account.
"""
MACRO = "macro"
MICRO = "micro"
WEIGHTED = "weighted"
supervision/metrics/f1_score.py
+545
View File
@@ -0,0 +1,545 @@
from __future__ import annotations
from copy import deepcopy
from dataclasses import dataclass
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
import numpy as np
from matplotlib import pyplot as plt
from supervision.config import ORIENTED_BOX_COORDINATES
from supervision.detection.core import Detections
from supervision.detection.utils import box_iou_batch, mask_iou_batch
from supervision.draw.color import LEGACY_COLOR_PALETTE
from supervision.metrics.core import AveragingMethod, Metric, MetricTarget
from supervision.metrics.utils.object_size import (
ObjectSizeCategory,
get_detection_size_category,
)
from supervision.metrics.utils.utils import ensure_pandas_installed
if TYPE_CHECKING:
import pandas as pd
class F1Score(Metric):
def __init__(
self,
metric_target: MetricTarget = MetricTarget.BOXES,
averaging_method: AveragingMethod = AveragingMethod.WEIGHTED,
):
self._metric_target = metric_target
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
raise NotImplementedError(
"F1 score is not implemented for oriented bounding boxes."
)
self._metric_target = metric_target
self.averaging_method = averaging_method
self._predictions_list: List[Detections] = []
self._targets_list: List[Detections] = []
def reset(self) -> None:
self._predictions_list = []
self._targets_list = []
def update(
self,
predictions: Union[Detections, List[Detections]],
targets: Union[Detections, List[Detections]],
) -> F1Score:
if not isinstance(predictions, list):
predictions = [predictions]
if not isinstance(targets, list):
targets = [targets]
if len(predictions) != len(targets):
raise ValueError(
f"The number of predictions ({len(predictions)}) and"
f" targets ({len(targets)}) during the update must be the same."
)
self._predictions_list.extend(predictions)
self._targets_list.extend(targets)
return self
def compute(self) -> F1ScoreResult:
result = self._compute(self._predictions_list, self._targets_list)
small_predictions, small_targets = self._filter_predictions_and_targets_by_size(
self._predictions_list, self._targets_list, ObjectSizeCategory.SMALL
)
result.small_objects = self._compute(small_predictions, small_targets)
medium_predictions, medium_targets = (
self._filter_predictions_and_targets_by_size(
self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM
)
)
result.medium_objects = self._compute(medium_predictions, medium_targets)
large_predictions, large_targets = self._filter_predictions_and_targets_by_size(
self._predictions_list, self._targets_list, ObjectSizeCategory.LARGE
)
result.large_objects = self._compute(large_predictions, large_targets)
return result
def _compute(
self, predictions_list: List[Detections], targets_list: List[Detections]
) -> F1ScoreResult:
iou_thresholds = np.linspace(0.5, 0.95, 10)
stats = []
for predictions, targets in zip(predictions_list, targets_list):
prediction_contents = self._detections_content(predictions)
target_contents = self._detections_content(targets)
if len(targets) > 0:
if len(predictions) == 0:
stats.append(
(
np.zeros((0, iou_thresholds.size), dtype=bool),
np.zeros((0,), dtype=np.float32),
np.zeros((0,), dtype=int),
targets.class_id,
)
)
else:
if self._metric_target == MetricTarget.BOXES:
iou = box_iou_batch(target_contents, prediction_contents)
elif self._metric_target == MetricTarget.MASKS:
iou = mask_iou_batch(target_contents, prediction_contents)
else:
raise NotImplementedError(
"Unsupported metric target for IoU calculation"
)
matches = self._match_detection_batch(
predictions.class_id, targets.class_id, iou, iou_thresholds
)
stats.append(
(
matches,
predictions.confidence,
predictions.class_id,
targets.class_id,
)
)
if not stats:
return F1ScoreResult(
metric_target=self._metric_target,
averaging_method=self.averaging_method,
f1_scores=np.zeros(iou_thresholds.shape[0]),
f1_per_class=np.zeros((0, iou_thresholds.shape[0])),
iou_thresholds=iou_thresholds,
matched_classes=np.array([], dtype=int),
small_objects=None,
medium_objects=None,
large_objects=None,
)
concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)]
f1_scores, f1_per_class, unique_classes = self._compute_f1_for_classes(
*concatenated_stats
)
return F1ScoreResult(
metric_target=self._metric_target,
averaging_method=self.averaging_method,
f1_scores=f1_scores,
f1_per_class=f1_per_class,
iou_thresholds=iou_thresholds,
matched_classes=unique_classes,
small_objects=None,
medium_objects=None,
large_objects=None,
)
def _compute_f1_for_classes(
self,
matches: np.ndarray,
prediction_confidence: np.ndarray,
prediction_class_ids: np.ndarray,
true_class_ids: np.ndarray,
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
sorted_indices = np.argsort(-prediction_confidence)
matches = matches[sorted_indices]
prediction_class_ids = prediction_class_ids[sorted_indices]
unique_classes, class_counts = np.unique(true_class_ids, return_counts=True)
# Shape: PxTh,P,C,C -> CxThx3
confusion_matrix = self._compute_confusion_matrix(
matches, prediction_class_ids, unique_classes, class_counts
)
# Shape: CxThx3 -> CxTh
f1_per_class = self._compute_f1(confusion_matrix)
# Shape: CxTh -> Th
if self.averaging_method == AveragingMethod.MACRO:
f1_scores = np.mean(f1_per_class, axis=0)
elif self.averaging_method == AveragingMethod.MICRO:
confusion_matrix_merged = confusion_matrix.sum(0)
f1_scores = self._compute_f1(confusion_matrix_merged)
elif self.averaging_method == AveragingMethod.WEIGHTED:
class_counts = class_counts.astype(np.float32)
f1_scores = np.average(f1_per_class, axis=0, weights=class_counts)
return f1_scores, f1_per_class, unique_classes
@staticmethod
def _match_detection_batch(
predictions_classes: np.ndarray,
target_classes: np.ndarray,
iou: np.ndarray,
iou_thresholds: np.ndarray,
) -> np.ndarray:
num_predictions, num_iou_levels = (
predictions_classes.shape[0],
iou_thresholds.shape[0],
)
correct = np.zeros((num_predictions, num_iou_levels), dtype=bool)
correct_class = target_classes[:, None] == predictions_classes
for i, iou_level in enumerate(iou_thresholds):
matched_indices = np.where((iou >= iou_level) & correct_class)
if matched_indices[0].shape[0]:
combined_indices = np.stack(matched_indices, axis=1)
iou_values = iou[matched_indices][:, None]
matches = np.hstack([combined_indices, iou_values])
if matched_indices[0].shape[0] > 1:
matches = matches[matches[:, 2].argsort()[::-1]]
matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
correct[matches[:, 1].astype(int), i] = True
return correct
@staticmethod
def _compute_confusion_matrix(
sorted_matches: np.ndarray,
sorted_prediction_class_ids: np.ndarray,
unique_classes: np.ndarray,
class_counts: np.ndarray,
) -> np.ndarray:
"""
Compute the confusion matrix for each class and IoU threshold.
Assumes the matches and prediction_class_ids are sorted by confidence
in descending order.
Arguments:
sorted_matches: np.ndarray, bool, shape (P, Th), that is True
if the prediction is a true positive at the given IoU threshold.
sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing
the class id for each prediction.
unique_classes: np.ndarray, int, shape (C,), containing the unique
class ids.
class_counts: np.ndarray, int, shape (C,), containing the number
of true instances for each class.
Returns:
np.ndarray, shape (C, Th, 3), containing the true positives, false
positives, and false negatives for each class and IoU threshold.
"""
num_thresholds = sorted_matches.shape[1]
num_classes = unique_classes.shape[0]
confusion_matrix = np.zeros((num_classes, num_thresholds, 3))
for class_idx, class_id in enumerate(unique_classes):
is_class = sorted_prediction_class_ids == class_id
num_true = class_counts[class_idx]
num_predictions = is_class.sum()
if num_predictions == 0:
true_positives = np.zeros(num_thresholds)
false_positives = np.zeros(num_thresholds)
false_negatives = np.full(num_thresholds, num_true)
elif num_true == 0:
true_positives = np.zeros(num_thresholds)
false_positives = np.full(num_thresholds, num_predictions)
false_negatives = np.zeros(num_thresholds)
else:
true_positives = sorted_matches[is_class].sum(0)
false_positives = (1 - sorted_matches[is_class]).sum(0)
false_negatives = num_true - true_positives
confusion_matrix[class_idx] = np.stack(
[true_positives, false_positives, false_negatives], axis=1
)
return confusion_matrix
@staticmethod
def _compute_f1(confusion_matrix: np.ndarray) -> np.ndarray:
"""
Broadcastable function, computing the F1 score from the confusion matrix.
Arguments:
confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension
contains the true positives, false positives, and false negatives.
Returns:
np.ndarray, shape (N, ...), containing the F1 score for each element.
"""
if not confusion_matrix.shape[-1] == 3:
raise ValueError(
f"Confusion matrix must have shape (..., 3), got "
f"{confusion_matrix.shape}"
)
true_positives = confusion_matrix[..., 0]
false_positives = confusion_matrix[..., 1]
false_negatives = confusion_matrix[..., 2]
# Alternate formula, avoids multiple zero division checks
denominator = 2 * true_positives + false_positives + false_negatives
f1_score = np.where(denominator == 0, 0, 2 * true_positives / denominator)
return f1_score
def _detections_content(self, detections: Detections) -> np.ndarray:
"""Return boxes, masks or oriented bounding boxes from detections."""
if self._metric_target == MetricTarget.BOXES:
return detections.xyxy
if self._metric_target == MetricTarget.MASKS:
return (
detections.mask
if detections.mask is not None
else np.empty((0, 0, 0), dtype=bool)
)
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
if obb := detections.data.get(ORIENTED_BOX_COORDINATES):
return np.ndarray(obb, dtype=np.float32)
return np.empty((0, 8), dtype=np.float32)
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _filter_detections_by_size(
self, detections: Detections, size_category: ObjectSizeCategory
) -> Detections:
"""Return a copy of detections with contents filtered by object size."""
new_detections = deepcopy(detections)
if detections.is_empty() or size_category == ObjectSizeCategory.ANY:
return new_detections
sizes = get_detection_size_category(new_detections, self._metric_target)
size_mask = sizes == size_category.value
new_detections.xyxy = new_detections.xyxy[size_mask]
if new_detections.mask is not None:
new_detections.mask = new_detections.mask[size_mask]
if new_detections.class_id is not None:
new_detections.class_id = new_detections.class_id[size_mask]
if new_detections.confidence is not None:
new_detections.confidence = new_detections.confidence[size_mask]
if new_detections.tracker_id is not None:
new_detections.tracker_id = new_detections.tracker_id[size_mask]
if new_detections.data is not None:
for key, value in new_detections.data.items():
new_detections.data[key] = np.array(value)[size_mask]
return new_detections
def _filter_predictions_and_targets_by_size(
self,
predictions_list: List[Detections],
targets_list: List[Detections],
size_category: ObjectSizeCategory,
) -> Tuple[List[Detections], List[Detections]]:
"""
Filter predictions and targets by object size category.
"""
new_predictions_list = []
new_targets_list = []
for predictions, targets in zip(predictions_list, targets_list):
new_predictions_list.append(
self._filter_detections_by_size(predictions, size_category)
)
new_targets_list.append(
self._filter_detections_by_size(targets, size_category)
)
return new_predictions_list, new_targets_list
@dataclass
class F1ScoreResult:
"""
The results of the F1 score metric calculation.
Defaults to `0` if no detections or targets were provided.
Provides a custom `__str__` method for pretty printing.
Attributes:
metric_target (MetricTarget): the type of data used for the metric -
boxes, masks or oriented bounding boxes.
averaging_method (AveragingMethod): the averaging method used to compute the
F1 scores. Determines how the F1 scores are aggregated across classes.
f1_50 (float): the F1 score at IoU threshold of `0.5`.
f1_75 (float): the F1 score at IoU threshold of `0.75`.
f1_scores (np.ndarray): the F1 scores at each IoU threshold.
Shape: `(num_iou_thresholds,)`
f1_per_class (np.ndarray): the F1 scores per class and IoU threshold.
Shape: `(num_target_classes, num_iou_thresholds)`
iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
matched_classes (np.ndarray): the class IDs of all matched classes.
Corresponds to the rows of `f1_per_class`.
small_objects (Optional[F1ScoreResult]): the F1 metric results
for small objects.
medium_objects (Optional[F1ScoreResult]): the F1 metric results
for medium objects.
large_objects (Optional[F1ScoreResult]): the F1 metric results
for large objects.
"""
metric_target: MetricTarget
averaging_method: AveragingMethod
@property
def f1_50(self) -> float:
return self.f1_scores[0]
@property
def f1_75(self) -> float:
return self.f1_scores[5]
f1_scores: np.ndarray
f1_per_class: np.ndarray
iou_thresholds: np.ndarray
matched_classes: np.ndarray
small_objects: Optional[F1ScoreResult]
medium_objects: Optional[F1ScoreResult]
large_objects: Optional[F1ScoreResult]
def __str__(self) -> str:
"""
Format as a pretty string.
Example:
```python
print(f1_result)
```
"""
out_str = (
f"{self.__class__.__name__}:\n"
f"Metric target: {self.metric_target}\n"
f"Averaging method: {self.averaging_method}\n"
f"F1 @ 50: {self.f1_50:.4f}\n"
f"F1 @ 75: {self.f1_75:.4f}\n"
f"F1 @ thresh: {self.f1_scores}\n"
f"IoU thresh: {self.iou_thresholds}\n"
f"F1 per class:\n"
)
if self.f1_per_class.size == 0:
out_str += " No results\n"
for class_id, f1_of_class in zip(self.matched_classes, self.f1_per_class):
out_str += f" {class_id}: {f1_of_class}\n"
indent = " "
if self.small_objects is not None:
indented = indent + str(self.small_objects).replace("\n", f"\n{indent}")
out_str += f"\nSmall objects:\n{indented}"
if self.medium_objects is not None:
indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}")
out_str += f"\nMedium objects:\n{indented}"
if self.large_objects is not None:
indented = indent + str(self.large_objects).replace("\n", f"\n{indent}")
out_str += f"\nLarge objects:\n{indented}"
return out_str
def to_pandas(self) -> "pd.DataFrame":
"""
Convert the result to a pandas DataFrame.
Returns:
(pd.DataFrame): The result as a DataFrame.
"""
ensure_pandas_installed()
import pandas as pd
pandas_data = {
"F1@50": self.f1_50,
"F1@75": self.f1_75,
}
if self.small_objects is not None:
small_objects_df = self.small_objects.to_pandas()
for key, value in small_objects_df.items():
pandas_data[f"small_objects_{key}"] = value
if self.medium_objects is not None:
medium_objects_df = self.medium_objects.to_pandas()
for key, value in medium_objects_df.items():
pandas_data[f"medium_objects_{key}"] = value
if self.large_objects is not None:
large_objects_df = self.large_objects.to_pandas()
for key, value in large_objects_df.items():
pandas_data[f"large_objects_{key}"] = value
return pd.DataFrame(pandas_data, index=[0])
def plot(self):
"""
Plot the F1 results.
"""
labels = ["F1@50", "F1@75"]
values = [self.f1_50, self.f1_75]
colors = [LEGACY_COLOR_PALETTE[0]] * 2
if self.small_objects is not None:
small_objects = self.small_objects
labels += ["Small: F1@50", "Small: F1@75"]
values += [small_objects.f1_50, small_objects.f1_75]
colors += [LEGACY_COLOR_PALETTE[3]] * 2
if self.medium_objects is not None:
medium_objects = self.medium_objects
labels += ["Medium: F1@50", "Medium: F1@75"]
values += [medium_objects.f1_50, medium_objects.f1_75]
colors += [LEGACY_COLOR_PALETTE[2]] * 2
if self.large_objects is not None:
large_objects = self.large_objects
labels += ["Large: F1@50", "Large: F1@75"]
values += [large_objects.f1_50, large_objects.f1_75]
colors += [LEGACY_COLOR_PALETTE[4]] * 2
plt.rcParams["font.family"] = "monospace"
_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
title = (
f"F1 Score, by Object Size"
f"\n(target: {self.metric_target.value},"
f" averaging: {self.averaging_method.value})"
)
ax.set_title(title, fontweight="bold")
x_positions = range(len(labels))
bars = ax.bar(x_positions, values, color=colors, align="center")
ax.set_xticks(x_positions)
ax.set_xticklabels(labels, rotation=45, ha="right")
for bar in bars:
y_value = bar.get_height()
ax.text(
bar.get_x() + bar.get_width() / 2,
y_value + 0.02,
f"{y_value:.2f}",
ha="center",
va="bottom",
)
plt.rcParams["font.family"] = "sans-serif"
plt.tight_layout()
plt.show()
supervision/metrics/mean_average_precision.py
+80 -70
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from copy import deepcopy
from dataclasses import dataclass
from typing import TYPE_CHECKING, List, Optional, Union
from typing import TYPE_CHECKING, List, Optional, Tuple, Union
import numpy as np
from matplotlib import pyplot as plt
@@ -192,21 +192,21 @@ class MeanAveragePrecision(Metric):
# Compute average precisions if any matches exist
if stats:
concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)]
average_precisions = self._average_precisions_per_class(*concatenated_stats)
map50 = average_precisions[:, 0].mean()
map75 = average_precisions[:, 5].mean()
map50_95 = average_precisions.mean()
average_precisions, unique_classes = self._average_precisions_per_class(
*concatenated_stats
)
mAP_scores = np.mean(average_precisions, axis=0)
else:
map50, map75, map50_95 = 0, 0, 0
mAP_scores = np.zeros((10,), dtype=np.float32)
unique_classes = np.empty((0,), dtype=int)
average_precisions = np.empty((0, len(iou_thresholds)), dtype=np.float32)
return MeanAveragePrecisionResult(
iou_thresholds=iou_thresholds,
map50_95=map50_95,
map50=map50,
map75=map75,
per_class_ap50_95=average_precisions,
metric_target=self._metric_target,
mAP_scores=mAP_scores,
iou_thresholds=iou_thresholds,
matched_classes=unique_classes,
ap_per_class=average_precisions,
)
@staticmethod
@@ -270,7 +270,7 @@ class MeanAveragePrecision(Metric):
prediction_confidence: np.ndarray,
prediction_class_ids: np.ndarray,
true_class_ids: np.ndarray,
) -> np.ndarray:
) -> Tuple[np.ndarray, np.ndarray]:
"""
Compute the average precision, given the recall and precision curves.
Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
@@ -283,7 +283,8 @@ class MeanAveragePrecision(Metric):
eps (float, optional): Small value to prevent division by zero.
Returns:
(np.ndarray): Average precision for different IoU levels.
(Tuple[np.ndarray, np.ndarray]): Average precision for different
IoU levels, and an array of class IDs that were matched.
"""
eps = 1e-16
@@ -318,7 +319,7 @@ class MeanAveragePrecision(Metric):
)
)
return average_precisions
return average_precisions, unique_classes
def _detections_content(self, detections: Detections) -> np.ndarray:
"""Return boxes, masks or oriented bounding boxes from detections."""
@@ -374,38 +375,57 @@ class MeanAveragePrecision(Metric):
@dataclass
class MeanAveragePrecisionResult:
iou_thresholds: np.ndarray
"""Array of IoU thresholds used in the calculations"""
map50_95: float
"""Mean Average Precision over IoU thresholds from 0.5 to 0.95"""
"""
The result of the Mean Average Precision calculation.
map50: float
"""Mean Average Precision at IoU threshold of 0.5"""
Defaults to `0` when no detections or targets are present.
map75: float
"""Mean Average Precision at IoU threshold of 0.75"""
per_class_ap50_95: np.ndarray
"""Average precision for each class at different IoU thresholds"""
Attributes:
metric_target (MetricTarget): the type of data used for the metric -
boxes, masks or oriented bounding boxes.
mAP_map50_95 (float): the mAP score at IoU thresholds from `0.5` to `0.95`.
mAP_map50 (float): the mAP score at IoU threshold of `0.5`.
mAP_map75 (float): the mAP score at IoU threshold of `0.75`.
mAP_scores (np.ndarray): the mAP scores at each IoU threshold.
Shape: `(num_iou_thresholds,)`
ap_per_class (np.ndarray): the average precision scores per
class and IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)`
iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
matched_classes (np.ndarray): the class IDs of all matched classes.
Corresponds to the rows of `ap_per_class`.
small_objects (Optional[MeanAveragePrecisionResult]): the mAP results
for small objects.
medium_objects (Optional[MeanAveragePrecisionResult]): the mAP results
for medium objects.
large_objects (Optional[MeanAveragePrecisionResult]): the mAP results
for large objects.
"""
metric_target: MetricTarget
"""
Defines the type of data used for the metric - boxes, masks or
oriented bounding boxes.
"""
@property
def map50_95(self) -> float:
return self.mAP_scores.mean()
@property
def map50(self) -> float:
return self.mAP_scores[0]
@property
def map75(self) -> float:
return self.mAP_scores[5]
mAP_scores: np.ndarray
ap_per_class: np.ndarray
iou_thresholds: np.ndarray
matched_classes: np.ndarray
small_objects: Optional[MeanAveragePrecisionResult] = None
"""Mean Average Precision results for small objects"""
medium_objects: Optional[MeanAveragePrecisionResult] = None
"""Mean Average Precision results for medium objects"""
large_objects: Optional[MeanAveragePrecisionResult] = None
"""Mean Average Precision results for large objects"""
def __str__(self) -> str:
"""
Format the mAP results as a pretty string.
Format as a pretty string.
Example:
```python
@@ -415,28 +435,29 @@ class MeanAveragePrecisionResult:
out_str = (
f"{self.__class__.__name__}:\n"
f"iou_thresholds: {self.iou_thresholds}\n"
f"map50_95: {self.map50_95:.4f}\n"
f"map50: {self.map50:.4f}\n"
f"map75: {self.map75:.4f}\n"
f"per_class_ap50_95:"
f"Metric target: {self.metric_target}\n"
f"mAP @ 50:95: {self.map50_95:.4f}\n"
f"mAP @ 50: {self.map50:.4f}\n"
f"mAP @ 75: {self.map75:.4f}\n"
f"mAP scores: {self.mAP_scores}\n"
f"IoU thresh: {self.iou_thresholds}\n"
f"AP per class:\n"
)
for class_id, ap in enumerate(self.per_class_ap50_95):
out_str += f"\n {class_id}: {ap}"
if self.ap_per_class.size == 0:
out_str += " No results\n"
for class_id, ap_of_class in zip(self.matched_classes, self.ap_per_class):
out_str += f" {class_id}: {ap_of_class}\n"
indent = " "
if self.small_objects is not None:
indented_str = indent + str(self.small_objects).replace("\n", f"\n{indent}")
out_str += f"\nSmall objects:\n{indented_str}"
indented = indent + str(self.small_objects).replace("\n", f"\n{indent}")
out_str += f"\nSmall objects:\n{indented}"
if self.medium_objects is not None:
indented_str = indent + str(self.medium_objects).replace(
"\n", f"\n{indent}"
)
out_str += f"\nMedium objects:\n{indented_str}"
indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}")
out_str += f"\nMedium objects:\n{indented}"
if self.large_objects is not None:
indented_str = indent + str(self.large_objects).replace("\n", f"\n{indent}")
out_str += f"\nLarge objects:\n{indented_str}"
indented = indent + str(self.large_objects).replace("\n", f"\n{indent}")
out_str += f"\nLarge objects:\n{indented}"
return out_str
@@ -451,10 +472,11 @@ class MeanAveragePrecisionResult:
import pandas as pd
pandas_data = {
"mAP_50_95": self.map50_95,
"mAP_50": self.map50,
"mAP_75": self.map75,
"mAP@50:95": self.map50_95,
"mAP@50": self.map50,
"mAP@75": self.map75,
}
if self.small_objects is not None:
small_objects_df = self.small_objects.to_pandas()
for key, value in small_objects_df.items():
@@ -480,16 +502,12 @@ class MeanAveragePrecisionResult:
Plot the mAP results.
"""
labels = ["mAP_50_95", "mAP_50", "mAP_75"]
labels = ["mAP@50:95", "mAP@50", "mAP@75"]
values = [self.map50_95, self.map50, self.map75]
colors = [LEGACY_COLOR_PALETTE[0]] * 3
if self.small_objects is not None:
labels += [
"small_objects_mAP_50_95",
"small_objects_mAP_50",
"small_objects_mAP_75",
]
labels += ["Small: mAP@50:95", "Small: mAP@50", "Small: mAP@75"]
values += [
self.small_objects.map50_95,
self.small_objects.map50,
@@ -498,11 +516,7 @@ class MeanAveragePrecisionResult:
colors += [LEGACY_COLOR_PALETTE[3]] * 3
if self.medium_objects is not None:
labels += [
"medium_objects_mAP_50_95",
"medium_objects_mAP_50",
"medium_objects_mAP_75",
]
labels += ["Medium: mAP@50:95", "Medium: mAP@50", "Medium: mAP@75"]
values += [
self.medium_objects.map50_95,
self.medium_objects.map50,
@@ -511,11 +525,7 @@ class MeanAveragePrecisionResult:
colors += [LEGACY_COLOR_PALETTE[2]] * 3
if self.large_objects is not None:
labels += [
"large_objects_mAP_50_95",
"large_objects_mAP_50",
"large_objects_mAP_75",
]
labels += ["Large: mAP@50:95", "Large: mAP@50", "Large: mAP@75"]
values += [
self.large_objects.map50_95,
self.large_objects.map50,
supervision/metrics/utils/internal_data_store.py
-300
View File
@@ -1,300 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING, List, Optional, Set, Tuple
import numpy as np
import numpy.typing as npt
from supervision.config import ORIENTED_BOX_COORDINATES
from supervision.metrics.core import CLASS_ID_NONE, MetricTarget
from supervision.metrics.utils.object_size import (
ObjectSizeCategory,
get_object_size_category,
)
from supervision.metrics.utils.utils import unify_pad_masks_shape
if TYPE_CHECKING:
from supervision.detection.core import Detections
class MetricData:
"""
A container for detection contents, decouple from `Detections`.
"""
def __init__(self, metric_target: MetricTarget, class_agnostic: bool = False):
self._metric_target = metric_target
self._class_agnostic = class_agnostic
self._content_list: List[npt.NDArray] = []
self._confidence_list: List[npt.NDArray[np.float32]] = []
self._class_id_list: List[npt.NDArray[np.int_]] = []
def update(self, detections: Detections):
"""Add new detections to the store."""
if detections.is_empty() or len(detections) == 0:
return
# Relies on Detections to ensure that member vars are equal in length
# or None/empty.
self._validate_new_entry(detections)
new_content = self._detections_content(detections)
self._validate_shape(new_content)
self._content_list.append(new_content)
if detections.confidence is not None:
self._confidence_list.append(detections.confidence)
if detections.class_id is not None:
if self._class_agnostic:
self._class_id_list.append(
np.full(len(detections.class_id), CLASS_ID_NONE, dtype=int)
)
else:
self._class_id_list.append(detections.class_id)
def get_classes(self) -> Set[int]:
"""Return all class IDs."""
if self._class_agnostic:
return {CLASS_ID_NONE}
return set.union(*[set(class_id) for class_id in self._class_id_list])
def get(
self, class_id: Optional[int] = None, size_category=ObjectSizeCategory.ANY
) -> Tuple[npt.NDArray, npt.NDArray, npt.NDArray]:
"""
Get the contents, class_ids and confidences, optionally filtered by
class_id and/or size.
Args:
class_id (Optional[int]): The class ID for the data to retrieve
size_category (ObjectSizeCategory): The size of the objects to retrieve.
Returns:
(np.ndarray): Boxes, masks or obb, all stacked in a single array.
"""
if self._metric_target in [
MetricTarget.BOXES,
MetricTarget.ORIENTED_BOUNDING_BOXES,
]:
self._merge_boxes()
if len(self._content_list) == 0:
content = self._make_empty_content()
else:
content = self._content_list[0]
elif self._metric_target == MetricTarget.MASKS:
self._merge_masks()
if len(self._content_list) == 0:
content = self._make_empty_content()
else:
content = self._content_list[0]
size_mask = np.full(len(content), True)
if size_category != ObjectSizeCategory.ANY:
sizes = get_object_size_category(content, self._metric_target)
size_mask = sizes == size_category.value
class_mask = np.full(len(content), True)
if class_id is not None:
self._merge_class_id()
class_ids = self._class_id_list[0]
class_mask = class_ids == class_id
content = content[(size_mask & class_mask)]
self._merge_class_id()
if len(self._class_id_list) == 0:
class_ids = np.array([], dtype=int)
else:
class_ids = self._class_id_list[0]
if len(class_ids) > 0:
class_ids = class_ids[(size_mask & class_mask)]
self._merge_confidence()
if len(self._confidence_list) == 0:
confidences = np.array([], dtype=float)
else:
confidences = self._confidence_list[0]
if len(confidences) > 0:
confidences = confidences[(size_mask & class_mask)]
return content, class_ids, confidences
def get_class_id(self) -> npt.NDArray[np.int_]:
self._merge_class_id()
if len(self._class_id_list) == 0:
return np.array([], dtype=int)
return self._class_id_list[0]
def get_confidence(self) -> npt.NDArray[np.float32]:
self._merge_confidence()
if len(self._confidence_list) == 0:
return np.array([], dtype=float)
return self._confidence_list[0]
def _detections_content(self, detections: Detections) -> npt.NDArray:
"""Return boxes, masks or oriented bounding boxes from detections."""
if self._metric_target == MetricTarget.BOXES:
return detections.xyxy
if self._metric_target == MetricTarget.MASKS:
return (
detections.mask
if detections.mask is not None
else self._make_empty_content()
)
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
if obb := detections.data.get(ORIENTED_BOX_COORDINATES):
return np.ndarray(obb, dtype=np.float32)
return self._make_empty_content()
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _make_empty_content(self) -> npt.NDArray:
if self._metric_target == MetricTarget.BOXES:
return np.empty((0, 4), dtype=np.float32)
if self._metric_target == MetricTarget.MASKS:
return np.empty((0, 0, 0), dtype=bool)
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
return np.empty((0, 8), dtype=np.float32)
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _validate_new_entry(self, detections: Detections) -> None:
if (
len(self._content_list) == 0
or detections.is_empty()
or len(detections) == 0
):
return
is_self_class_empty = len(self._class_id_list) == 0
is_self_confidence_empty = len(self._confidence_list) == 0
is_detection_class_empty = (
detections.class_id is None or len(detections.class_id) == 0
)
is_detection_confidence_empty = (
detections.confidence is None or len(detections.confidence) == 0
)
if is_self_confidence_empty and not is_detection_confidence_empty:
raise ValueError(
"Previously stored detections without confidence,"
" but new detections have it."
)
if is_self_class_empty and not is_detection_class_empty:
raise ValueError(
"Previously stored detections without class ID,"
" but new detections have it."
)
if not is_self_class_empty and is_detection_class_empty:
raise ValueError(
"Started storing detections with class ID, but"
" new detections do not have it."
)
if not is_self_confidence_empty and is_detection_confidence_empty:
raise ValueError(
"Started storing detections with confidence,"
" but new detections do not have it."
)
def _validate_shape(self, data: npt.NDArray) -> None:
if self._metric_target == MetricTarget.BOXES:
if len(data.shape) != 2 or data.shape[1] != 4:
raise ValueError(f"Invalid xyxy shape: {data.shape}. Expected: (N, 4)")
elif self._metric_target == MetricTarget.MASKS:
if len(data.shape) != 3:
raise ValueError(
f"Invalid mask shape: {data.shape}. Expected: (N, H, W)"
)
elif self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
if len(data.shape) != 2 or data.shape[1] != 8:
raise ValueError(f"Invalid obb shape: {data.shape}. Expected: (N, 8)")
else:
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _merge_boxes(self):
"""Merge all boxes into a single array."""
if self._metric_target not in (
MetricTarget.BOXES,
MetricTarget.ORIENTED_BOUNDING_BOXES,
):
raise ValueError("Invalid metric target for merging boxes")
if len(self._content_list) < 2:
return
self._content_list = [np.vstack(self._content_list)]
def _merge_masks(self):
"""Merge all masks into a single array."""
if self._metric_target != MetricTarget.MASKS:
raise ValueError("Invalid metric target for merging masks")
if len(self._content_list) < 2:
return
mew_mask_list = unify_pad_masks_shape(*self._content_list)
self._content_list = [np.vstack(mew_mask_list)]
def _merge_class_id(self):
"""Merge all class IDs into a single array."""
if len(self._class_id_list) < 2:
return
self._class_id_list = [np.hstack(self._class_id_list)]
def _merge_confidence(self):
"""Merge all confidences into a single array."""
if len(self._confidence_list) < 2:
return
self._confidence_list = [np.hstack(self._confidence_list)]
class MetricDataStore:
"""
Stores internal data for metrics.
Provides a class-agnostic way to access it.
"""
def __init__(self, metric_target: MetricTarget, class_agnostic: bool = False):
self._metric_target = metric_target
self._class_agnostic = class_agnostic
self._data_1: MetricData
self._data_2: MetricData
self.reset()
def reset(self) -> None:
self._data_1 = MetricData(self._metric_target, self._class_agnostic)
self._data_2 = MetricData(self._metric_target, self._class_agnostic)
def update(self, data_1: Detections, data_2: Detections) -> None:
"""
Add new data to the store.
Use sv.Detections.empty() if only one set of data is available.
"""
self._data_1.update(data_1)
self._data_2.update(data_2)
def get(
self,
class_id: Optional[int] = None,
size_category: ObjectSizeCategory = ObjectSizeCategory.ANY,
) -> Tuple[
Tuple[npt.NDArray, npt.NDArray, npt.NDArray],
Tuple[npt.NDArray, npt.NDArray, npt.NDArray],
]:
"""
Get the data for both sets, optionally filtered by class_id and/or size.
"""
data_1, class_id_1, confidence_1 = self._data_1.get(class_id, size_category)
data_2, class_id_2, confidence_2 = self._data_2.get(class_id, size_category)
if self._metric_target == MetricTarget.MASKS:
data_1, data_2 = unify_pad_masks_shape(data_1, data_2)
return (
(data_1, class_id_1, confidence_1),
(data_2, class_id_2, confidence_2),
)
def get_classes(self) -> Set[int]:
"""Return all class IDs."""
return self._data_1.get_classes() | self._data_2.get_classes()
supervision/metrics/utils/utils.py
-61
View File
@@ -1,9 +1,3 @@
from typing import List, Tuple
import numpy as np
import numpy.typing as npt
def ensure_pandas_installed():
try:
import pandas # noqa
@@ -13,58 +7,3 @@ def ensure_pandas_installed():
" Run `pip install 'supervision[metrics]'` or"
" `poetry add supervision -E metrics`"
)
def pad_mask(mask: npt.NDArray, new_shape: Tuple[int, int]) -> npt.NDArray:
"""Pad a mask to a new shape, inserting zeros on the right and bottom."""
if len(mask.shape) != 3:
raise ValueError(f"Invalid mask shape: {mask.shape}. Expected: (N, H, W)")
new_mask = np.pad(
mask,
(
(0, 0),
(0, new_shape[0] - mask.shape[1]),
(0, new_shape[1] - mask.shape[2]),
),
mode="constant",
constant_values=0,
)
return new_mask
def unify_pad_masks_shape(
*masks: npt.NDArray[np.bool_],
) -> List[npt.NDArray[np.bool_]]:
"""
Given any number of (N, H, W) mask objects, return copies of the
same (H, W), padded to the largest dimensions.
Args:
*masks (np.ndarray): The mask arrays to unify. Each shaped (_, H, W),
Returns:
List[np.ndarray]: The masks, padded to the largest dimensions.
Each list element shaped (_, H_max, W_max)
"""
new_h = 0
new_w = 0
for mask in masks:
if len(mask.shape) != 3:
raise ValueError(f"Invalid mask shape: {mask.shape}. Expected: (N, H, W)")
_, h, w = mask.shape
new_h = max(new_h, h)
new_w = max(new_w, w)
results = []
for mask in masks:
results.append(pad_mask(mask, (new_h, new_w)))
return results
def len0_like(data: npt.NDArray) -> npt.NDArray:
"""Create an empty array with the same shape as input, but with 0 rows."""
return np.empty((0, *data.shape[1:]), dtype=data.dtype)