Correct confusion matrix calculation-function evaluate_detection_batch (#1853)
* Correct confusion matrix calculation-function evaluate_detection_batch * Correct confusion matrix computation * Add 3 more tests for empty Detections-GTs * Minor changes in too-long lines corrected * Replace deprecated mock_detections with _create_detections * Fix indentation in test assertions and add missing `self` parameter in `test_confusion_matrix`. * Update metric computations to improve numerical stability and replace deprecated NumPy functions * Add IoU+class matching tests and synthetic dataset fixtures for detection metrics * Use `Optional` for type hinting `classes` parameter in `_yolo_dataset_factory`. Import `Optional` from `typing`. * Refactor detection metric tests to simplify confusion matrix assertions and enhance IoU+class matching validation. * Apply suggestions from code review * fix(pre_commit): 🎨 auto format pre-commit hooks --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
7 files changed
+820
-60
No files matched your search
@@ -148,7 +148,6 @@ class ConfusionMatrix:
|
||||
|
||||
```
|
||||
"""
|
||||
|
||||
prediction_tensors = []
|
||||
target_tensors = []
|
||||
for prediction, target in zip(predictions, targets):
|
||||
@@ -274,9 +273,28 @@ class ConfusionMatrix:
|
||||
"""
|
||||
result_matrix = np.zeros((num_classes + 1, num_classes + 1))
|
||||
|
||||
# Filter predictions by confidence threshold
|
||||
conf_idx = 5
|
||||
confidence = predictions[:, conf_idx]
|
||||
detection_batch_filtered = predictions[confidence > conf_threshold]
|
||||
detection_batch_filtered = predictions[confidence >= conf_threshold]
|
||||
|
||||
if len(detection_batch_filtered) == 0:
|
||||
# No detections pass confidence threshold - all GT are FN
|
||||
class_id_idx = 4
|
||||
true_classes = np.array(targets[:, class_id_idx], dtype=np.int16)
|
||||
for gt_class in true_classes:
|
||||
result_matrix[gt_class, num_classes] += 1
|
||||
return result_matrix
|
||||
|
||||
if len(targets) == 0:
|
||||
# No ground truth - all detections are FP
|
||||
class_id_idx = 4
|
||||
detection_classes = np.array(
|
||||
detection_batch_filtered[:, class_id_idx], dtype=np.int16
|
||||
)
|
||||
for det_class in detection_classes:
|
||||
result_matrix[num_classes, det_class] += 1
|
||||
return result_matrix
|
||||
|
||||
class_id_idx = 4
|
||||
true_classes = np.array(targets[:, class_id_idx], dtype=np.int16)
|
||||
@@ -286,37 +304,71 @@ class ConfusionMatrix:
|
||||
true_boxes = targets[:, :class_id_idx]
|
||||
detection_boxes = detection_batch_filtered[:, :class_id_idx]
|
||||
|
||||
# Calculate IoU matrix
|
||||
iou_batch = box_iou_batch(
|
||||
boxes_true=true_boxes, boxes_detection=detection_boxes
|
||||
)
|
||||
matched_idx = np.asarray(iou_batch > iou_threshold).nonzero()
|
||||
|
||||
if matched_idx[0].shape[0]:
|
||||
matches = np.stack(
|
||||
(matched_idx[0], matched_idx[1], iou_batch[matched_idx]), axis=1
|
||||
)
|
||||
matches = ConfusionMatrix._drop_extra_matches(matches=matches)
|
||||
# Find all valid matches (IoU > threshold, regardless of class)
|
||||
# Use vectorized operations to avoid nested Python loops
|
||||
iou_mask = iou_batch > iou_threshold
|
||||
gt_indices, det_indices = np.nonzero(iou_mask)
|
||||
|
||||
# If no pairs exceed the IoU threshold, skip matching
|
||||
if gt_indices.size == 0:
|
||||
valid_matches = []
|
||||
else:
|
||||
matches = np.zeros((0, 3))
|
||||
ious = iou_batch[gt_indices, det_indices]
|
||||
gt_match_classes = true_classes[gt_indices]
|
||||
det_match_classes = detection_classes[det_indices]
|
||||
class_matches = gt_match_classes == det_match_classes
|
||||
|
||||
matched_true_idx, matched_detection_idx, _ = matches.transpose().astype(
|
||||
np.int16
|
||||
)
|
||||
# Sort matches by class match first (True before False),
|
||||
# then by IoU descending.
|
||||
# np.lexsort sorts by the last key first, in ascending order.
|
||||
# We use ~class_matches so that True becomes 0
|
||||
# and False becomes 1 (True first),
|
||||
# and -ious so that larger IoUs come first.
|
||||
sort_indices = np.lexsort((-ious, ~class_matches))
|
||||
|
||||
for i, true_class_value in enumerate(true_classes):
|
||||
j = matched_true_idx == i
|
||||
if matches.shape[0] > 0 and sum(j) == 1:
|
||||
result_matrix[
|
||||
true_class_value, detection_classes[matched_detection_idx[j]]
|
||||
] += 1 # TP
|
||||
else:
|
||||
result_matrix[true_class_value, num_classes] += 1 # FN
|
||||
# Build list of matches in the same format as before:
|
||||
# (gt_idx, det_idx, iou, class_match)
|
||||
valid_matches = [
|
||||
(
|
||||
int(gt_indices[idx]),
|
||||
int(det_indices[idx]),
|
||||
float(ious[idx]),
|
||||
bool(class_matches[idx]),
|
||||
)
|
||||
for idx in sort_indices
|
||||
]
|
||||
# Greedily assign matches, ensuring each GT
|
||||
# and detection is matched at most once
|
||||
matched_gt_idx = set()
|
||||
matched_det_idx = set()
|
||||
|
||||
for i, detection_class_value in enumerate(detection_classes):
|
||||
if not any(matched_detection_idx == i):
|
||||
result_matrix[num_classes, detection_class_value] += 1 # FP
|
||||
final_result_matrix: npt.NDArray[np.int32] = result_matrix
|
||||
return final_result_matrix
|
||||
for gt_idx, det_idx, iou, class_match in valid_matches:
|
||||
if gt_idx not in matched_gt_idx and det_idx not in matched_det_idx:
|
||||
# Valid spatial match - record the class prediction
|
||||
gt_class = true_classes[gt_idx]
|
||||
det_class = detection_classes[det_idx]
|
||||
|
||||
# This handles both correct classification (TP) and misclassification
|
||||
result_matrix[gt_class, det_class] += 1
|
||||
matched_gt_idx.add(gt_idx)
|
||||
matched_det_idx.add(det_idx)
|
||||
|
||||
# Count unmatched ground truth as FN
|
||||
for gt_idx, gt_class in enumerate(true_classes):
|
||||
if gt_idx not in matched_gt_idx:
|
||||
result_matrix[gt_class, num_classes] += 1
|
||||
|
||||
# Count unmatched detections as FP
|
||||
for det_idx, det_class in enumerate(detection_classes):
|
||||
if det_idx not in matched_det_idx:
|
||||
result_matrix[num_classes, det_class] += 1
|
||||
|
||||
return result_matrix
|
||||
|
||||
@staticmethod
|
||||
def _drop_extra_matches(
|
||||
|
||||
@@ -385,7 +385,12 @@ class Precision(Metric):
|
||||
false_positives = confusion_matrix[..., 1]
|
||||
|
||||
denominator = true_positives + false_positives
|
||||
precision = np.where(denominator == 0, 0, true_positives / denominator)
|
||||
precision = np.divide(
|
||||
true_positives,
|
||||
denominator,
|
||||
out=np.zeros_like(true_positives),
|
||||
where=denominator != 0,
|
||||
)
|
||||
|
||||
result_precision: npt.NDArray[np.float64] = precision
|
||||
return result_precision
|
||||
|
||||
@@ -383,7 +383,12 @@ class Recall(Metric):
|
||||
false_negatives = confusion_matrix[..., 2]
|
||||
|
||||
denominator = true_positives + false_negatives
|
||||
recall = np.where(denominator == 0, 0, true_positives / denominator)
|
||||
recall = np.divide(
|
||||
true_positives,
|
||||
denominator,
|
||||
out=np.zeros_like(true_positives),
|
||||
where=denominator != 0,
|
||||
)
|
||||
|
||||
result_recall: npt.NDArray[np.float64] = recall
|
||||
return result_recall
|
||||
|
||||
@@ -414,3 +414,67 @@ def create_yolo_dataset(
|
||||
"image_size": image_size,
|
||||
"image_annotations": image_annotations,
|
||||
}
|
||||
|
||||
|
||||
def create_predictions_with_class_iou_tests(
|
||||
gt_detections: Detections, num_classes: int
|
||||
) -> Detections:
|
||||
"""
|
||||
Create predictions that test IoU+class matching behavior.
|
||||
|
||||
For each ground truth detection, creates predictions with different patterns:
|
||||
- Pattern 0 (i%3==0): Correct match (same bbox, correct class)
|
||||
- Pattern 1 (i%3==1): Wrong class with perfect IoU + correct class with offset
|
||||
- Pattern 2 (i%3==2): Correct class with slight offset
|
||||
|
||||
This tests that predictions with wrong class don't match even with high IoU,
|
||||
which is the key fix in the confusion matrix calculation.
|
||||
|
||||
Args:
|
||||
gt_detections: Ground truth detections to create predictions for
|
||||
num_classes: Total number of classes in the dataset
|
||||
|
||||
Returns:
|
||||
Detections object with predictions designed to test IoU+class matching
|
||||
"""
|
||||
if len(gt_detections) == 0:
|
||||
# No ground truth, return a single false positive
|
||||
return _create_detections(
|
||||
xyxy=[[10, 10, 50, 50]], class_id=[0], confidence=[0.9]
|
||||
)
|
||||
|
||||
pred_boxes = []
|
||||
pred_classes = []
|
||||
pred_confs = []
|
||||
|
||||
for i, (box, cls) in enumerate(zip(gt_detections.xyxy, gt_detections.class_id)):
|
||||
if i % 3 == 0:
|
||||
# Pattern 1: Correct match
|
||||
pred_boxes.append(box)
|
||||
pred_classes.append(cls)
|
||||
pred_confs.append(0.95)
|
||||
|
||||
elif i % 3 == 1:
|
||||
# Pattern 2: Test the fix - add wrong class prediction with perfect IoU,
|
||||
# then correct class with slightly offset bbox
|
||||
wrong_cls = (cls + 1) % num_classes
|
||||
pred_boxes.append(box) # Perfect IoU
|
||||
pred_classes.append(wrong_cls) # Wrong class
|
||||
pred_confs.append(0.90)
|
||||
|
||||
# Add correct class with slight offset
|
||||
offset_box = box + np.array([2, 2, 2, 2], dtype=np.float32)
|
||||
pred_boxes.append(offset_box)
|
||||
pred_classes.append(cls) # Correct class
|
||||
pred_confs.append(0.85)
|
||||
|
||||
else:
|
||||
# Pattern 3: Correct match with slight offset
|
||||
offset_box = box + np.array([1, 1, 1, 1], dtype=np.float32)
|
||||
pred_boxes.append(offset_box)
|
||||
pred_classes.append(cls)
|
||||
pred_confs.append(0.92)
|
||||
|
||||
return _create_detections(
|
||||
xyxy=pred_boxes, class_id=pred_classes, confidence=pred_confs
|
||||
)
|
||||
@@ -1,3 +1,5 @@
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
@@ -138,3 +140,85 @@ def target_class_1():
|
||||
xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32),
|
||||
class_id=np.array([1]),
|
||||
)
|
||||
|
||||
|
||||
def _yolo_dataset_factory(
|
||||
tmp_path,
|
||||
num_images: int = 20,
|
||||
classes: Optional[list[str]] = None,
|
||||
objects_per_image_range: tuple[int, int] = (1, 3),
|
||||
):
|
||||
"""
|
||||
Factory function to create synthetic YOLO-format datasets with custom parameters.
|
||||
|
||||
Args:
|
||||
tmp_path: Pytest tmp_path fixture
|
||||
num_images: Number of images to generate
|
||||
classes: List of class names
|
||||
objects_per_image_range: Range of objects per image as (min, max)
|
||||
|
||||
Returns:
|
||||
dict with dataset paths and metadata
|
||||
"""
|
||||
from tests.helpers import create_yolo_dataset
|
||||
|
||||
if classes is None:
|
||||
classes = ["dog", "cat", "person"]
|
||||
|
||||
return create_yolo_dataset(
|
||||
dataset_dir=str(tmp_path / "yolo_dataset"),
|
||||
num_images=num_images,
|
||||
image_size=(640, 640, 3),
|
||||
classes=classes,
|
||||
objects_per_image_range=objects_per_image_range,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def yolo_dataset_structure(tmp_path):
|
||||
"""
|
||||
Synthetic YOLO-format dataset for testing confusion matrix and detection metrics.
|
||||
|
||||
Configuration:
|
||||
- 20 images
|
||||
- 640x640 resolution
|
||||
- 3 classes: ["dog", "cat", "person"]
|
||||
- 1-3 objects per image
|
||||
|
||||
Use this for tests that need multi-class scenarios (3+ classes).
|
||||
|
||||
Returns:
|
||||
dict with dataset paths and metadata
|
||||
"""
|
||||
return _yolo_dataset_factory(
|
||||
tmp_path,
|
||||
num_images=20,
|
||||
classes=["dog", "cat", "person"],
|
||||
objects_per_image_range=(1, 3),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def yolo_dataset_two_classes(tmp_path):
|
||||
"""
|
||||
Synthetic YOLO-format dataset for testing mAR and binary classification metrics.
|
||||
|
||||
Configuration:
|
||||
- 15 images
|
||||
- 640x640 resolution
|
||||
- 2 classes: ["class_0", "class_1"]
|
||||
- 2-4 objects per image
|
||||
|
||||
Use this for tests that specifically need 2-class scenarios or depend on
|
||||
specific class distributions (e.g., mAR @ K per-image limiting tests).
|
||||
|
||||
Returns:
|
||||
dict with dataset paths and metadata
|
||||
"""
|
||||
return _yolo_dataset_factory(
|
||||
tmp_path,
|
||||
num_images=15,
|
||||
classes=["class_0", "class_1"],
|
||||
objects_per_image_range=(2, 4),
|
||||
)
|
||||
@@ -6,13 +6,18 @@ from typing import ClassVar
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from supervision.dataset.core import DetectionDataset
|
||||
from supervision.detection.core import Detections
|
||||
from supervision.metrics.detection import (
|
||||
ConfusionMatrix,
|
||||
MeanAveragePrecision,
|
||||
detections_to_tensor,
|
||||
)
|
||||
from tests.helpers import _create_detections, assert_almost_equal
|
||||
from tests.helpers import (
|
||||
_create_detections,
|
||||
assert_almost_equal,
|
||||
create_predictions_with_class_iou_tests,
|
||||
)
|
||||
|
||||
|
||||
class TestDetectionMetrics:
|
||||
@@ -347,8 +352,8 @@ class TestDetectionMetrics:
|
||||
iou_threshold=iou_threshold,
|
||||
)
|
||||
|
||||
assert result.matrix.diagonal().sum() == expected_result.diagonal().sum()
|
||||
assert np.array_equal(result.matrix, expected_result)
|
||||
assert result.matrix.diagonal().sum() == expected_result.diagonal().sum()
|
||||
assert np.array_equal(result.matrix, expected_result)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
@@ -391,8 +396,8 @@ class TestDetectionMetrics:
|
||||
iou_threshold=iou_threshold,
|
||||
)
|
||||
|
||||
assert result.diagonal().sum() == result.sum()
|
||||
assert np.array_equal(result, expected_result)
|
||||
assert result.diagonal().sum() == result.sum()
|
||||
assert np.array_equal(result, expected_result)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("matches", "expected_result", "exception"),
|
||||
@@ -463,3 +468,574 @@ class TestDetectionMetrics:
|
||||
recall=recall, precision=precision
|
||||
)
|
||||
assert_almost_equal(result, expected_result, tolerance=0.01)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"predictions",
|
||||
"targets",
|
||||
"classes",
|
||||
"conf_threshold",
|
||||
"iou_threshold",
|
||||
"expected_result",
|
||||
"exception",
|
||||
),
|
||||
[
|
||||
# Test 1: Class priority over IoU - correct class with lower IoU should win
|
||||
(
|
||||
[
|
||||
_create_detections( # Predicted bboxes
|
||||
xyxy=[[0.1, 0.1, 2.1, 2.1], [0.0, 0.0, 2.0, 2.0]],
|
||||
class_id=[0, 1],
|
||||
confidence=[0.9, 0.95],
|
||||
)
|
||||
],
|
||||
[_create_detections(xyxy=[[0, 0, 2, 2]], class_id=[0])], # GT bboxes
|
||||
[0, 1, 2], # Class ids
|
||||
0.5, # Confidence Threshold
|
||||
0.5, # IOU Threshold
|
||||
np.array( # Expected confusion matrix
|
||||
[
|
||||
[1.0, 0.0, 0.0, 0.0], # 1 TP
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
[0.0, 1.0, 0.0, 0.0], # 1 FP:
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 2: Multiple overlapping predictions with different classes
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0.1, 0.1, 2.1, 2.1],
|
||||
[0.2, 0.2, 2.2, 2.2],
|
||||
[0.3, 0.3, 2.3, 2.3],
|
||||
[4.1, 4.1, 6.1, 6.1],
|
||||
],
|
||||
class_id=[0, 1, 2, 1],
|
||||
confidence=[0.9, 0.8, 0.7, 0.85],
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [4, 4, 6, 6]], class_id=[0, 1]
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[1.0, 0.0, 0.0, 0.0], # 1 TP
|
||||
[0.0, 1.0, 0.0, 0.0], # 1 TP
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
[0.0, 1.0, 1.0, 0.0], # 2 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 3: Confidence threshold filtering with edge cases
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [4, 4, 6, 6], [8, 8, 10, 10]],
|
||||
class_id=[0, 1, 2],
|
||||
confidence=[0.6, 0.4, 0.8], # middle one below threshold
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [4, 4, 6, 6]], class_id=[0, 1]
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[1.0, 0.0, 0.0, 0.0], # 1 TP
|
||||
[0.0, 0.0, 0.0, 1.0], # 1 FN (filtered by conf)
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
[0.0, 0.0, 1.0, 0.0], # 1 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 4: IoU threshold boundary (IoU = 0.5625, slightly above threshold)
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 1.5, 1.5],
|
||||
[4, 4, 5.5, 5.5],
|
||||
], # IoU = 0.5625 for both
|
||||
class_id=[0, 1],
|
||||
confidence=[0.9, 0.8],
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [4, 4, 6, 6]], class_id=[0, 1]
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[1.0, 0.0, 0.0, 0.0], # 1 TP (IoU exceeds threshold)
|
||||
[0.0, 1.0, 0.0, 0.0], # 1 TP (IoU exceeds threshold)
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 5: Chain of overlapping detections
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0.1, 0.1, 2.1, 2.1], [1.9, 1.9, 3.9, 3.9]],
|
||||
class_id=[0, 2],
|
||||
confidence=[0.9, 0.8],
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [1, 1, 3, 3], [2, 2, 4, 4]],
|
||||
class_id=[0, 1, 2],
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[1.0, 0.0, 0.0, 0.0], # 1 TP
|
||||
[0.0, 0.0, 0.0, 1.0], # 1 FN (no matching label)
|
||||
[0.0, 0.0, 1.0, 0.0], # 1 TP
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 6: All false positives (no ground truth)
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [4, 4, 6, 6], [8, 8, 10, 10]],
|
||||
class_id=[0, 1, 2],
|
||||
confidence=[0.9, 0.8, 0.7],
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=np.empty((0, 4)), class_id=np.array([], dtype=int)
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
[0.0, 0.0, 0.0, 0.0], # none
|
||||
[1.0, 1.0, 1.0, 0.0], # 3 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 7: Empty predictions and empty ground truth
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=np.empty((0, 4)),
|
||||
class_id=np.array([], dtype=int),
|
||||
confidence=np.array([], dtype=float),
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=np.empty((0, 4)), class_id=np.array([], dtype=int)
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.zeros((4, 4)),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 8: Multi-class misclassifications
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [4, 4, 6, 6], [10, 10, 12, 12]],
|
||||
class_id=[0, 2, 1],
|
||||
confidence=[0.9, 0.8, 0.7],
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [4, 4, 6, 6], [8, 8, 10, 10]],
|
||||
class_id=[0, 1, 2],
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[1.0, 0.0, 0.0, 0.0], # 1 TP
|
||||
[0.0, 0.0, 1.0, 0.0], # 1 misclassified
|
||||
[0.0, 0.0, 0.0, 1.0], # 1 FN
|
||||
[0.0, 1.0, 0.0, 0.0], # 1 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 9: Complex multiple predictions with mixed results
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 2, 2],
|
||||
[4, 4, 6, 6],
|
||||
[8, 8, 10, 10],
|
||||
[12, 12, 14, 14],
|
||||
[16, 16, 18, 18],
|
||||
],
|
||||
class_id=[0, 1, 1, 2, 2],
|
||||
confidence=[0.9, 0.8, 0.7, 0.6, 0.5],
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 2, 2],
|
||||
[4, 4, 6, 6],
|
||||
[8, 8, 10, 10],
|
||||
[12, 12, 14, 14],
|
||||
],
|
||||
class_id=[0, 1, 2, 0],
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[1.0, 0.0, 1.0, 0.0], # 1 TP and 1 misclassified
|
||||
[0.0, 1.0, 0.0, 0.0], # 1 TP
|
||||
[0.0, 1.0, 0.0, 0.0], # 1 misclassified
|
||||
[0.0, 0.0, 1.0, 0.0], # 1 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 10: Large complex example with confidence filtering
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 2, 2],
|
||||
[4, 4, 6, 6],
|
||||
[8, 8, 10, 10],
|
||||
[12, 12, 14, 14],
|
||||
[16, 16, 18, 18],
|
||||
[18, 18, 20, 20],
|
||||
],
|
||||
class_id=[0, 0, 1, 2, 1, 2],
|
||||
confidence=[0.9, 0.8, 0.7, 0.6, 0.5, 0.4], # last one filtered
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 2, 2],
|
||||
[4, 4, 6, 6],
|
||||
[8, 8, 10, 10],
|
||||
[12, 12, 14, 14],
|
||||
],
|
||||
class_id=[0, 1, 2, 0],
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5, # conf_threshold filters out last prediction
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[1.0, 0.0, 1.0, 0.0], # 1 TP and 1 misclassified
|
||||
[1.0, 0.0, 0.0, 0.0], # 1 misclassified
|
||||
[0.0, 1.0, 0.0, 0.0], # 1 misclassified
|
||||
[0.0, 1.0, 0.0, 0.0], # 1 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 11: High counts with multiple TPs and misclassifications
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 2, 2],
|
||||
[0, 3, 2, 5],
|
||||
[0, 6, 2, 8],
|
||||
[4, 0, 6, 2],
|
||||
[4, 3, 6, 5],
|
||||
[8, 0, 10, 2],
|
||||
[12, 0, 14, 2],
|
||||
],
|
||||
class_id=[0, 0, 0, 2, 2, 2, 0],
|
||||
confidence=[0.95, 0.95, 0.95, 0.9, 0.9, 0.9, 0.8],
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 2, 2],
|
||||
[0, 3, 2, 5],
|
||||
[0, 6, 2, 8],
|
||||
[4, 0, 6, 2],
|
||||
[4, 3, 6, 5],
|
||||
[8, 0, 10, 2],
|
||||
[8, 3, 10, 5],
|
||||
],
|
||||
class_id=[0, 0, 0, 1, 1, 2, 2],
|
||||
)
|
||||
],
|
||||
[0, 1, 2],
|
||||
0.5,
|
||||
0.5,
|
||||
np.array(
|
||||
[
|
||||
[3.0, 0.0, 0.0, 0.0], # 3 TP
|
||||
[0.0, 0.0, 2.0, 0.0], # 2 misclassified
|
||||
[0.0, 0.0, 1.0, 1.0], # 1 TP, 1 FN
|
||||
[1.0, 0.0, 0.0, 0.0], # 1 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 12: Symmetric multi-class confusions with higher counts
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 2, 2],
|
||||
[0, 4, 2, 6],
|
||||
[4, 0, 6, 2],
|
||||
[4, 4, 6, 6],
|
||||
[8, 0, 10, 2],
|
||||
[8, 4, 10, 6],
|
||||
[12, 0, 14, 2],
|
||||
[12, 4, 14, 6],
|
||||
],
|
||||
class_id=[0, 0, 1, 1, 0, 0, 1, 1],
|
||||
confidence=[0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.8, 0.8],
|
||||
)
|
||||
],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[
|
||||
[0, 0, 2, 2],
|
||||
[0, 4, 2, 6],
|
||||
[4, 0, 6, 2],
|
||||
[4, 4, 6, 6],
|
||||
[8, 0, 10, 2],
|
||||
[8, 4, 10, 6],
|
||||
],
|
||||
class_id=[0, 0, 1, 1, 2, 2],
|
||||
)
|
||||
],
|
||||
[0, 1, 2], # Class ids
|
||||
0.5, # Confidence threshold
|
||||
0.5, # IOU threshold
|
||||
np.array(
|
||||
[
|
||||
[2.0, 0.0, 0.0, 0.0], # 2 TP
|
||||
[0.0, 2.0, 0.0, 0.0], # TP
|
||||
[2.0, 0.0, 0.0, 0.0], # 2 misclassified
|
||||
[0.0, 2.0, 0.0, 0.0], # 2 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 13: Empty Ground Truths
|
||||
(
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [0, 4, 2, 6]],
|
||||
class_id=[0, 0],
|
||||
confidence=[0.9, 0.9],
|
||||
)
|
||||
],
|
||||
[Detections.empty()],
|
||||
[0, 1, 2], # Class ids
|
||||
0.5, # Confidence threshold
|
||||
0.5, # IOU threshold
|
||||
np.array(
|
||||
[
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
[2.0, 0.0, 0.0, 0.0], # 2 FP
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 14: Empty Detections
|
||||
(
|
||||
[Detections.empty()],
|
||||
[
|
||||
_create_detections(
|
||||
xyxy=[[0, 0, 2, 2], [0, 4, 2, 6]], class_id=[0, 0]
|
||||
)
|
||||
],
|
||||
[0, 1, 2], # Class ids
|
||||
0.5, # Confidence threshold
|
||||
0.5, # IOU threshold
|
||||
np.array(
|
||||
[
|
||||
[0.0, 0.0, 0.0, 2.0], # 2 TP
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
# Test 15: Symmetric multi-class confusions with higher counts
|
||||
(
|
||||
[Detections.empty()],
|
||||
[Detections.empty()],
|
||||
[0, 1, 2], # Class ids
|
||||
0.5, # Confidence threshold
|
||||
0.5, # IOU threshold
|
||||
np.array(
|
||||
[
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
[0.0, 0.0, 0.0, 0.0],
|
||||
]
|
||||
),
|
||||
DoesNotRaise(),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_confusion_matrix(
|
||||
self,
|
||||
predictions,
|
||||
targets,
|
||||
classes,
|
||||
conf_threshold,
|
||||
iou_threshold,
|
||||
expected_result,
|
||||
exception: Exception,
|
||||
):
|
||||
with exception:
|
||||
confusion_matrix = ConfusionMatrix.from_detections(
|
||||
predictions=predictions,
|
||||
targets=targets,
|
||||
classes=classes,
|
||||
conf_threshold=conf_threshold,
|
||||
iou_threshold=iou_threshold,
|
||||
)
|
||||
|
||||
# Verify the confusion matrix matches expected
|
||||
# AssertionError if the two arrays are not equal
|
||||
np.testing.assert_array_equal(confusion_matrix.matrix, expected_result)
|
||||
|
||||
def test_confusion_matrix_on_yolo_dataset(self, yolo_dataset_structure):
|
||||
"""
|
||||
Test confusion matrix calculation on a YOLO-format dataset.
|
||||
|
||||
This test verifies that the confusion matrix fix (considering both IoU AND
|
||||
class agreement) works correctly when applied to a dataset loaded from
|
||||
roboflow-format YOLO data. It creates a synthetic dataset with specific
|
||||
scenarios where predictions have high IoU but wrong class, ensuring only
|
||||
predictions with correct class are matched.
|
||||
"""
|
||||
dataset_info = yolo_dataset_structure
|
||||
classes = ["dog", "cat", "person"]
|
||||
|
||||
# Load dataset using supervision's YOLO loader
|
||||
dataset = DetectionDataset.from_yolo(
|
||||
images_directory_path=dataset_info["images_dir"],
|
||||
annotations_directory_path=dataset_info["labels_dir"],
|
||||
data_yaml_path=dataset_info["data_yaml_path"],
|
||||
)
|
||||
|
||||
# Verify dataset loaded correctly
|
||||
assert len(dataset) == dataset_info["num_images"], (
|
||||
f"Dataset should have {dataset_info['num_images']} images, "
|
||||
f"but got {len(dataset)}. Dataset loading may have failed."
|
||||
)
|
||||
assert dataset.classes == classes, (
|
||||
f"Dataset classes should be {classes}, but got {dataset.classes}. "
|
||||
f"Check data.yaml parsing."
|
||||
)
|
||||
|
||||
# Test confusion matrix with the dataset
|
||||
# Split the dataset to test split functionality
|
||||
train_dataset, test_dataset = dataset.split(
|
||||
split_ratio=0.5, random_state=42, shuffle=True
|
||||
)
|
||||
|
||||
assert len(train_dataset) + len(test_dataset) == len(dataset), (
|
||||
f"Split datasets should sum to original dataset size ({len(dataset)}), "
|
||||
f"but got {len(train_dataset)} + {len(test_dataset)} = "
|
||||
f"{len(train_dataset) + len(test_dataset)}. Dataset split may be broken."
|
||||
)
|
||||
assert train_dataset.classes == classes, (
|
||||
"Train dataset should preserve class list after split"
|
||||
)
|
||||
assert test_dataset.classes == classes, (
|
||||
"Test dataset should preserve class list after split"
|
||||
)
|
||||
|
||||
# Create predictions that test the IoU+class matching fix
|
||||
predictions = []
|
||||
targets = []
|
||||
|
||||
for img_path, img, gt_detections in test_dataset:
|
||||
targets.append(gt_detections)
|
||||
predictions.append(
|
||||
create_predictions_with_class_iou_tests(gt_detections, len(classes))
|
||||
)
|
||||
|
||||
# Calculate confusion matrix
|
||||
confusion_matrix = ConfusionMatrix.from_detections(
|
||||
predictions=predictions,
|
||||
targets=targets,
|
||||
classes=list(range(len(classes))),
|
||||
conf_threshold=0.5,
|
||||
iou_threshold=0.5,
|
||||
)
|
||||
|
||||
# Verify confusion matrix structure and basic properties
|
||||
n = len(classes) + 1
|
||||
assert confusion_matrix.matrix.shape == (n, n), (
|
||||
f"Expected shape ({n}, {n}), got {confusion_matrix.matrix.shape}"
|
||||
)
|
||||
|
||||
# Count TPs (diagonal) and total ground truths
|
||||
total_gt = sum(len(t) for t in targets if len(t) > 0)
|
||||
total_tp = sum(confusion_matrix.matrix[i, i] for i in range(len(classes)))
|
||||
|
||||
assert total_tp > 0, (
|
||||
f"No TPs found (TP={total_tp}, GT={total_gt}), matching is broken"
|
||||
)
|
||||
|
||||
# Count FPs (last column) - should include wrong-class predictions
|
||||
total_fp = confusion_matrix.matrix[: len(classes), -1].sum()
|
||||
assert total_fp >= 0, f"FP count negative ({total_fp}), computation bug"
|
||||
|
||||
# Verify IoU+class fix: wrong-class preds should become FPs, not match GTs
|
||||
assert total_fp > 0 or total_tp == total_gt, (
|
||||
f"Expected FPs from wrong-class preds (got {total_fp}) or all GTs "
|
||||
f"matched (TP={total_tp}, GT={total_gt}). IoU+class fix may be broken: "
|
||||
f"wrong-class preds with high IoU might incorrectly match GTs."
|
||||
)
|
||||
@@ -359,32 +359,6 @@ def three_class_single_image_detections():
|
||||
return predictions, targets
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dataset_split_yolo_structure(tmp_path):
|
||||
"""
|
||||
Synthetic YOLO-format dataset for testing dataset loading.
|
||||
|
||||
Uses `create_yolo_dataset()` helper with default parameters:
|
||||
- 15 images
|
||||
- 640x640 resolution
|
||||
- 2 classes
|
||||
- 2-4 objects per image
|
||||
|
||||
Returns:
|
||||
dict with dataset paths and metadata
|
||||
"""
|
||||
from tests.helpers import create_yolo_dataset
|
||||
|
||||
return create_yolo_dataset(
|
||||
dataset_dir=str(tmp_path),
|
||||
num_images=15,
|
||||
image_size=(640, 640, 3),
|
||||
classes=["class_0", "class_1"],
|
||||
objects_per_image_range=(2, 4),
|
||||
seed=42,
|
||||
)
|
||||
|
||||
|
||||
def test_single_perfect_detection():
|
||||
"""Test that a single perfect detection yields 1.0 recall."""
|
||||
detections = Detections(
|
||||
@@ -531,7 +505,7 @@ def test_three_class_single_image_scenario(three_class_single_image_detections):
|
||||
)
|
||||
|
||||
|
||||
def test_dataset_split_integration(dataset_split_yolo_structure):
|
||||
def test_dataset_split_integration(yolo_dataset_two_classes):
|
||||
"""
|
||||
Test mAR with a roboflow-format dataset loaded from disk.
|
||||
|
||||
@@ -547,7 +521,7 @@ def test_dataset_split_integration(dataset_split_yolo_structure):
|
||||
"""
|
||||
from supervision import DetectionDataset
|
||||
|
||||
dataset_info = dataset_split_yolo_structure
|
||||
dataset_info = yolo_dataset_two_classes
|
||||
np.random.seed(42) # Match fixture seed for offset generation
|
||||
|
||||
# Load dataset from YOLO format
|
||||
|
||||
Reference in new issue
Block a user