From cfaea751f2114934858135486d7ec937bd8ea10e Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Mon, 2 Feb 2026 20:18:10 +0900 Subject: [PATCH] Refactor metric tests with fixtures and improved structure and clarity (#2123) * Refactor metric test files by encapsulating fixtures and improved structure and clarity * Update `test/metrics/conftest.py` fixture values to refine test case coverage --- test/metrics/conftest.py | 140 ++++ test/metrics/test_detection.py | 831 ++++++++++---------- test/metrics/test_f1_score.py | 164 +--- test/metrics/test_mean_average_precision.py | 568 +++++++------ test/metrics/test_precision.py | 156 +--- test/metrics/test_recall.py | 168 +--- 6 files changed, 896 insertions(+), 1131 deletions(-) create mode 100644 test/metrics/conftest.py diff --git a/test/metrics/conftest.py b/test/metrics/conftest.py new file mode 100644 index 00000000..30790160 --- /dev/null +++ b/test/metrics/conftest.py @@ -0,0 +1,140 @@ +import numpy as np +import pytest + +from supervision.detection.core import Detections + + +@pytest.fixture +def detections_50_50(): + return Detections( + xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32), + confidence=np.array([0.9]), + class_id=np.array([0]), + ) + + +@pytest.fixture +def targets_50_50(): + return Detections( + xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32), + class_id=np.array([0]), + ) + + +@pytest.fixture +def dummy_prediction(): + return Detections( + xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32), + confidence=np.array([0.8]), + class_id=np.array([0]), + ) + + +@pytest.fixture +def predictions_no_overlap(): + return Detections( + xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32), + confidence=np.array([0.9]), + class_id=np.array([0]), + ) + + +@pytest.fixture +def targets_no_overlap(): + return Detections( + xyxy=np.array([[100, 100, 110, 110]], dtype=np.float32), + class_id=np.array([0]), + ) + + +@pytest.fixture +def targets_two_objects_class_0(): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], + [100, 100, 110, 110], + ], + dtype=np.float32, + ), + class_id=np.array([0, 0]), + ) + + +@pytest.fixture +def predictions_multiple_classes(): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], # class 0, matches target + [60, 60, 100, 100], # class 1, matches target + [120, 120, 130, 130], # class 1, false positive + ], + dtype=np.float32, + ), + confidence=np.array([0.9, 0.8, 0.7]), + class_id=np.array([0, 1, 1]), + ) + + +@pytest.fixture +def targets_multiple_classes(): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], # class 0 + [60, 60, 100, 100], # class 1 + ], + dtype=np.float32, + ), + class_id=np.array([0, 1]), + ) + + +@pytest.fixture +def predictions_iou_064(): + return Detections( + xyxy=np.array([[15, 15, 55, 55]], dtype=np.float32), + confidence=np.array([0.9]), + class_id=np.array([0]), + ) + + +@pytest.fixture +def targets_iou_064(): + return Detections( + xyxy=np.array([[10, 10, 60, 60]], dtype=np.float32), + class_id=np.array([0]), + ) + + +@pytest.fixture +def predictions_confidence_ranking(): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], + [11, 11, 49, 49], + ], + dtype=np.float32, + ), + confidence=np.array([0.6, 0.9]), + class_id=np.array([0, 0]), + ) + + +@pytest.fixture +def prediction_class_1(): + return Detections( + xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32), + confidence=np.array([0.8]), + class_id=np.array([1]), + ) + + +@pytest.fixture +def target_class_1(): + return Detections( + xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32), + class_id=np.array([1]), + ) diff --git a/test/metrics/test_detection.py b/test/metrics/test_detection.py index 46dc0892..094831d2 100644 --- a/test/metrics/test_detection.py +++ b/test/metrics/test_detection.py @@ -1,6 +1,7 @@ from __future__ import annotations from contextlib import ExitStack as DoesNotRaise +from typing import ClassVar import numpy as np import pytest @@ -13,460 +14,430 @@ from supervision.metrics.detection import ( ) from test.helpers import _create_detections, assert_almost_equal -CLASSES = np.arange(80) -NUM_CLASSES = len(CLASSES) -PREDICTIONS = np.array( - [ - [2254, 906, 2447, 1353, 0.90538, 0], - [2049, 1133, 2226, 1371, 0.59002, 56], - [727, 1224, 838, 1601, 0.51119, 39], - [808, 1214, 910, 1564, 0.45287, 39], - [6, 52, 1131, 2133, 0.45057, 72], - [299, 1225, 512, 1663, 0.45029, 39], - [529, 874, 645, 945, 0.31101, 39], - [8, 47, 1935, 2135, 0.28192, 72], - [2265, 813, 2328, 901, 0.2714, 62], - ], - dtype=np.float32, -) +class TestDetectionMetrics: + CLASSES = np.arange(80) + NUM_CLASSES = len(CLASSES) -TARGET_TENSORS = [ - np.array( + PREDICTIONS = np.array( [ - [2254, 906, 2447, 1353, 0], - [2049, 1133, 2226, 1371, 56], - [727, 1224, 838, 1601, 39], - [808, 1214, 910, 1564, 39], - [6, 52, 1131, 2133, 72], - [299, 1225, 512, 1663, 39], - [529, 874, 645, 945, 39], - [8, 47, 1935, 2135, 72], - [2265, 813, 2328, 901, 62], - ] + [2254, 906, 2447, 1353, 0.90538, 0], + [2049, 1133, 2226, 1371, 0.59002, 56], + [727, 1224, 838, 1601, 0.51119, 39], + [808, 1214, 910, 1564, 0.45287, 39], + [6, 52, 1131, 2133, 0.45057, 72], + [299, 1225, 512, 1663, 0.45029, 39], + [529, 874, 645, 945, 0.31101, 39], + [8, 47, 1935, 2135, 0.28192, 72], + [2265, 813, 2328, 901, 0.2714, 62], + ], + dtype=np.float32, ) -] -DETECTIONS = Detections( - xyxy=PREDICTIONS[:, :4], - confidence=PREDICTIONS[:, 4], - class_id=PREDICTIONS[:, 5].astype(int), -) -CERTAIN_DETECTIONS = Detections( - xyxy=PREDICTIONS[:, :4], - confidence=np.ones(len(PREDICTIONS)), - class_id=PREDICTIONS[:, 5].astype(int), -) + TARGET_TENSORS: ClassVar[list[np.ndarray]] = [ + np.array( + [ + [2254, 906, 2447, 1353, 0], + [2049, 1133, 2226, 1371, 56], + [727, 1224, 838, 1601, 39], + [808, 1214, 910, 1564, 39], + [6, 52, 1131, 2133, 72], + [299, 1225, 512, 1663, 39], + [529, 874, 645, 945, 39], + [8, 47, 1935, 2135, 72], + [2265, 813, 2328, 901, 62], + ] + ) + ] -DETECTION_TENSORS = [ - np.concatenate( + DETECTIONS = Detections( + xyxy=PREDICTIONS[:, :4], + confidence=PREDICTIONS[:, 4], + class_id=PREDICTIONS[:, 5].astype(int), + ) + CERTAIN_DETECTIONS = Detections( + xyxy=PREDICTIONS[:, :4], + confidence=np.ones(len(PREDICTIONS)), + class_id=PREDICTIONS[:, 5].astype(int), + ) + + DETECTION_TENSORS: ClassVar[list[np.ndarray]] = [ + np.concatenate( + [ + det.xyxy, + np.expand_dims(det.class_id, 1), + np.expand_dims(det.confidence, 1), + ], + axis=1, + ) + for det in [DETECTIONS] + ] + CERTAIN_DETECTION_TENSORS: ClassVar[list[np.ndarray]] = [ + np.concatenate( + [ + det.xyxy, + np.expand_dims(det.class_id, 1), + np.ones((len(det), 1)), + ], + axis=1, + ) + for det in [DETECTIONS] + ] + + IDEAL_MATCHES = np.stack( [ - det.xyxy, - np.expand_dims(det.class_id, 1), - np.expand_dims(det.confidence, 1), + np.arange(len(PREDICTIONS)), + np.arange(len(PREDICTIONS)), + np.ones(len(PREDICTIONS)), ], axis=1, ) - for det in [DETECTIONS] -] -CERTAIN_DETECTION_TENSORS = [ - np.concatenate( - [ - det.xyxy, - np.expand_dims(det.class_id, 1), - np.ones((len(det), 1)), - ], - axis=1, + + @staticmethod + def create_empty_conf_matrix(num_classes: int, do_add_dummy_class: bool = True): + if do_add_dummy_class: + num_classes += 1 + return np.zeros((num_classes, num_classes)) + + @staticmethod + def update_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray): + for class_id, count in zip(*np.unique(class_ids, return_counts=True)): + class_id = int(class_id) + conf_matrix[class_id, class_id] += count + return conf_matrix + + @staticmethod + def worsen_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray | list): + for class_id in class_ids: + class_id = int(class_id) + conf_matrix[class_id, class_id] -= 1 + conf_matrix[class_id, 80] += 1 + return conf_matrix + + IDEAL_CONF_MATRIX = create_empty_conf_matrix.__func__(NUM_CLASSES) + IDEAL_CONF_MATRIX = update_ideal_conf_matrix.__func__( + IDEAL_CONF_MATRIX, PREDICTIONS[:, 5] ) - for det in [DETECTIONS] -] -IDEAL_MATCHES = np.stack( - [ - np.arange(len(PREDICTIONS)), - np.arange(len(PREDICTIONS)), - np.ones(len(PREDICTIONS)), - ], - axis=1, -) + GOOD_CONF_MATRIX = worsen_ideal_conf_matrix.__func__( + IDEAL_CONF_MATRIX.copy(), [62, 72] + ) + BAD_CONF_MATRIX = worsen_ideal_conf_matrix.__func__( + IDEAL_CONF_MATRIX.copy(), [62, 72, 72, 39, 39, 39, 39, 56] + ) -def create_empty_conf_matrix(num_classes: int, do_add_dummy_class: bool = True): - if do_add_dummy_class: - num_classes += 1 - return np.zeros((num_classes, num_classes)) + @pytest.mark.parametrize( + ("detections", "with_confidence", "expected_result", "exception"), + [ + ( + Detections.empty(), + False, + np.empty((0, 5), dtype=np.float32), + DoesNotRaise(), + ), # empty detections; no confidence + ( + Detections.empty(), + True, + np.empty((0, 6), dtype=np.float32), + DoesNotRaise(), + ), # empty detections; with confidence + ( + _create_detections( + xyxy=[[0, 0, 10, 10]], class_id=[0], confidence=[0.5] + ), + False, + np.array([[0, 0, 10, 10, 0]], dtype=np.float32), + DoesNotRaise(), + ), # single detection; no confidence + ( + _create_detections( + xyxy=[[0, 0, 10, 10]], class_id=[0], confidence=[0.5] + ), + True, + np.array([[0, 0, 10, 10, 0, 0.5]], dtype=np.float32), + DoesNotRaise(), + ), # single detection; with confidence + ( + _create_detections( + xyxy=[[0, 0, 10, 10], [0, 0, 20, 20]], + class_id=[0, 1], + confidence=[0.5, 0.2], + ), + False, + np.array([[0, 0, 10, 10, 0], [0, 0, 20, 20, 1]], dtype=np.float32), + DoesNotRaise(), + ), # multiple detections; no confidence + ( + _create_detections( + xyxy=[[0, 0, 10, 10], [0, 0, 20, 20]], + class_id=[0, 1], + confidence=[0.5, 0.2], + ), + True, + np.array( + [[0, 0, 10, 10, 0, 0.5], [0, 0, 20, 20, 1, 0.2]], dtype=np.float32 + ), + DoesNotRaise(), + ), # multiple detections; with confidence + ], + ) + def test_detections_to_tensor( + self, + detections: Detections, + with_confidence: bool, + expected_result: np.ndarray | None, + exception: Exception, + ): + with exception: + result = detections_to_tensor( + detections=detections, with_confidence=with_confidence + ) + assert np.array_equal(result, expected_result) - -def update_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray): - for class_id, count in zip(*np.unique(class_ids, return_counts=True)): - class_id = int(class_id) - conf_matrix[class_id, class_id] += count - return conf_matrix - - -def worsen_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray | list): - for class_id in class_ids: - class_id = int(class_id) - conf_matrix[class_id, class_id] -= 1 - conf_matrix[class_id, 80] += 1 - return conf_matrix - - -IDEAL_CONF_MATRIX = create_empty_conf_matrix(NUM_CLASSES) -IDEAL_CONF_MATRIX = update_ideal_conf_matrix(IDEAL_CONF_MATRIX, PREDICTIONS[:, 5]) - -GOOD_CONF_MATRIX = worsen_ideal_conf_matrix(IDEAL_CONF_MATRIX.copy(), [62, 72]) - -BAD_CONF_MATRIX = worsen_ideal_conf_matrix( - IDEAL_CONF_MATRIX.copy(), [62, 72, 72, 39, 39, 39, 39, 56] -) - - -@pytest.mark.parametrize( - ("detections", "with_confidence", "expected_result", "exception"), - [ + @pytest.mark.parametrize( ( - Detections.empty(), - False, - np.empty((0, 5), dtype=np.float32), - DoesNotRaise(), - ), # empty detections; no confidence - ( - Detections.empty(), - True, - np.empty((0, 6), dtype=np.float32), - DoesNotRaise(), - ), # empty detections; with confidence - ( - _create_detections(xyxy=[[0, 0, 10, 10]], class_id=[0], confidence=[0.5]), - False, - np.array([[0, 0, 10, 10, 0]], dtype=np.float32), - DoesNotRaise(), - ), # single detection; no confidence - ( - _create_detections(xyxy=[[0, 0, 10, 10]], class_id=[0], confidence=[0.5]), - True, - np.array([[0, 0, 10, 10, 0, 0.5]], dtype=np.float32), - DoesNotRaise(), - ), # single detection; with confidence - ( - _create_detections( - xyxy=[[0, 0, 10, 10], [0, 0, 20, 20]], - class_id=[0, 1], - confidence=[0.5, 0.2], + "predictions", + "targets", + "classes", + "conf_threshold", + "iou_threshold", + "expected_result", + "exception", + ), + [ + ( + DETECTION_TENSORS, + TARGET_TENSORS, + CLASSES, + 0.2, + 0.5, + IDEAL_CONF_MATRIX, + DoesNotRaise(), ), - False, - np.array([[0, 0, 10, 10, 0], [0, 0, 20, 20, 1]], dtype=np.float32), - DoesNotRaise(), - ), # multiple detections; no confidence - ( - _create_detections( - xyxy=[[0, 0, 10, 10], [0, 0, 20, 20]], - class_id=[0, 1], - confidence=[0.5, 0.2], + ( + [], + [], + CLASSES, + 0.2, + 0.5, + create_empty_conf_matrix.__func__(NUM_CLASSES), + DoesNotRaise(), ), - True, - np.array( - [[0, 0, 10, 10, 0, 0.5], [0, 0, 20, 20, 1, 0.2]], dtype=np.float32 + ( + DETECTION_TENSORS, + TARGET_TENSORS, + CLASSES, + 0.3, + 0.5, + GOOD_CONF_MATRIX, + DoesNotRaise(), ), - DoesNotRaise(), - ), # multiple detections; with confidence - ], -) -def test_detections_to_tensor( - detections: Detections, - with_confidence: bool, - expected_result: np.ndarray | None, - exception: Exception, -): - with exception: - result = detections_to_tensor( - detections=detections, with_confidence=with_confidence - ) - assert np.array_equal(result, expected_result) - - -@pytest.mark.parametrize( - ( - "predictions", - "targets", - "classes", - "conf_threshold", - "iou_threshold", - "expected_result", - "exception", - ), - [ - ( - DETECTION_TENSORS, - TARGET_TENSORS, - CLASSES, - 0.2, - 0.5, - IDEAL_CONF_MATRIX, - DoesNotRaise(), - ), - ( - [], - [], - CLASSES, - 0.2, - 0.5, - create_empty_conf_matrix(NUM_CLASSES), - DoesNotRaise(), - ), - ( - DETECTION_TENSORS, - TARGET_TENSORS, - CLASSES, - 0.3, - 0.5, - GOOD_CONF_MATRIX, - DoesNotRaise(), - ), - ( - DETECTION_TENSORS, - TARGET_TENSORS, - CLASSES, - 0.6, - 0.5, - BAD_CONF_MATRIX, - DoesNotRaise(), - ), - ( - [ - np.array( - [ - [0.0, 0.0, 3.0, 3.0, 0, 0.9], # correct detection of [0] + ( + DETECTION_TENSORS, + TARGET_TENSORS, + CLASSES, + 0.6, + 0.5, + BAD_CONF_MATRIX, + DoesNotRaise(), + ), + ( + [ + np.array( [ - 0.1, - 0.1, - 3.0, - 3.0, - 0, - 0.9, - ], # additional detection of [0] - FP + # correct detection of [0] + [0.0, 0.0, 3.0, 3.0, 0, 0.9], + # additional detection of [0] - FP + [0.1, 0.1, 3.0, 3.0, 0, 0.9], + # correct detection with incorrect class + [6.0, 1.0, 8.0, 3.0, 1, 0.8], + # incorrect detection - FP + [1.0, 6.0, 2.0, 7.0, 1, 0.8], + # incorrect detection with low IoU - FP + [1.0, 2.0, 2.0, 4.0, 1, 0.8], + ] + ) + ], + [ + np.array( + [ # [0] detected + [0.0, 0.0, 3.0, 3.0, 0], + # [1] undetected - FN + [2.0, 2.0, 5.0, 5.0, 1], + # [2] correct detection with incorrect class + [6.0, 1.0, 8.0, 3.0, 2], + ] + ) + ], + CLASSES[:3], + 0.6, + 0.5, + np.array([[1, 0, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [1, 2, 0, 0]]), + DoesNotRaise(), + ), + ( + [ + np.array( [ - 6.0, - 1.0, - 8.0, - 3.0, - 1, - 0.8, - ], # correct detection with incorrect class - [1.0, 6.0, 2.0, 7.0, 1, 0.8], # incorrect detection - FP + # correct detection of [0] + [0.0, 0.0, 3.0, 3.0, 0, 0.9], + # additional detection of [0] - FP + [0.1, 0.1, 3.0, 3.0, 0, 0.9], + # correct detection with incorrect class + [6.0, 1.0, 8.0, 3.0, 1, 0.8], + # incorrect detection - FP + [1.0, 6.0, 2.0, 7.0, 1, 0.8], + # incorrect detection with low IoU - FP + [1.0, 2.0, 2.0, 4.0, 1, 0.8], + ] + ) + ], + [ + np.array( [ - 1.0, - 2.0, - 2.0, - 4.0, - 1, - 0.8, - ], # incorrect detection with low IoU - FP - ] - ) - ], - [ - np.array( - [ - [0.0, 0.0, 3.0, 3.0, 0], # [0] detected - [2.0, 2.0, 5.0, 5.0, 1], # [1] undetected - FN - [ - 6.0, - 1.0, - 8.0, - 3.0, - 2, - ], # [2] correct detection with incorrect class - ] - ) - ], - CLASSES[:3], - 0.6, - 0.5, - np.array([[1, 0, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [1, 2, 0, 0]]), - DoesNotRaise(), + # [0] detected + [0.0, 0.0, 3.0, 3.0, 0], + # [1] undetected - FN + [2.0, 2.0, 5.0, 5.0, 1], + # [2] correct detection with incorrect class + [6.0, 1.0, 8.0, 3.0, 2], + ] + ) + ], + CLASSES[:3], + 0.6, + 1.0, + np.array([[0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1], [2, 3, 0, 0]]), + DoesNotRaise(), + ), + ], + ) + def test_from_tensors( + self, + predictions, + targets, + classes, + conf_threshold, + iou_threshold, + expected_result: np.ndarray | None, + exception: Exception, + ): + with exception: + result = ConfusionMatrix.from_tensors( + predictions=predictions, + targets=targets, + classes=classes, + conf_threshold=conf_threshold, + iou_threshold=iou_threshold, + ) + + assert result.matrix.diagonal().sum() == expected_result.diagonal().sum() + assert np.array_equal(result.matrix, expected_result) + + @pytest.mark.parametrize( + ( + "predictions", + "targets", + "num_classes", + "conf_threshold", + "iou_threshold", + "expected_result", + "exception", ), - ( - [ - np.array( - [ - [0.0, 0.0, 3.0, 3.0, 0, 0.9], # correct detection of [0] - [ - 0.1, - 0.1, - 3.0, - 3.0, - 0, - 0.9, - ], # additional detection of [0] - FP - [ - 6.0, - 1.0, - 8.0, - 3.0, - 1, - 0.8, - ], # correct detection with incorrect class - [1.0, 6.0, 2.0, 7.0, 1, 0.8], # incorrect detection - FP - [ - 1.0, - 2.0, - 2.0, - 4.0, - 1, - 0.8, - ], # incorrect detection with low IoU - FP - ] - ) - ], - [ - np.array( - [ - [0.0, 0.0, 3.0, 3.0, 0], # [0] detected - [2.0, 2.0, 5.0, 5.0, 1], # [1] undetected - FN - [ - 6.0, - 1.0, - 8.0, - 3.0, - 2, - ], # [2] correct detection with incorrect class - ] - ) - ], - CLASSES[:3], - 0.6, - 1.0, - np.array([[0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1], [2, 3, 0, 0]]), - DoesNotRaise(), - ), - ], -) -def test_from_tensors( - predictions, - targets, - classes, - conf_threshold, - iou_threshold, - expected_result: np.ndarray | None, - exception: Exception, -): - with exception: - result = ConfusionMatrix.from_tensors( - predictions=predictions, - targets=targets, - classes=classes, - conf_threshold=conf_threshold, - iou_threshold=iou_threshold, - ) + [ + ( + DETECTION_TENSORS[0], + CERTAIN_DETECTION_TENSORS[0], + NUM_CLASSES, + 0.2, + 0.5, + IDEAL_CONF_MATRIX, + DoesNotRaise(), + ) + ], + ) + def test_evaluate_detection_batch( + self, + predictions, + targets, + num_classes, + conf_threshold, + iou_threshold, + expected_result: np.ndarray | None, + exception: Exception, + ): + with exception: + result = ConfusionMatrix.evaluate_detection_batch( + predictions=predictions, + targets=targets, + num_classes=num_classes, + conf_threshold=conf_threshold, + iou_threshold=iou_threshold, + ) - assert result.matrix.diagonal().sum() == expected_result.diagonal().sum() - assert np.array_equal(result.matrix, expected_result) + assert result.diagonal().sum() == result.sum() + assert np.array_equal(result, expected_result) + @pytest.mark.parametrize( + ("matches", "expected_result", "exception"), + [ + ( + IDEAL_MATCHES, + IDEAL_MATCHES, + DoesNotRaise(), + ) + ], + ) + def test_drop_extra_matches( + self, + matches, + expected_result: np.ndarray | None, + exception: Exception, + ): + with exception: + result = ConfusionMatrix._drop_extra_matches(matches) -@pytest.mark.parametrize( - ( - "predictions", - "targets", - "num_classes", - "conf_threshold", - "iou_threshold", - "expected_result", - "exception", - ), - [ - ( - DETECTION_TENSORS[0], - CERTAIN_DETECTION_TENSORS[0], - NUM_CLASSES, - 0.2, - 0.5, - IDEAL_CONF_MATRIX, - DoesNotRaise(), - ) - ], -) -def test_evaluate_detection_batch( - predictions, - targets, - num_classes, - conf_threshold, - iou_threshold, - expected_result: np.ndarray | None, - exception: Exception, -): - with exception: - result = ConfusionMatrix.evaluate_detection_batch( - predictions=predictions, - targets=targets, - num_classes=num_classes, - conf_threshold=conf_threshold, - iou_threshold=iou_threshold, - ) + assert np.array_equal(result, expected_result) - assert result.diagonal().sum() == result.sum() - assert np.array_equal(result, expected_result) - - -@pytest.mark.parametrize( - ("matches", "expected_result", "exception"), - [ - ( - IDEAL_MATCHES, - IDEAL_MATCHES, - DoesNotRaise(), - ) - ], -) -def test_drop_extra_matches( - matches, - expected_result: np.ndarray | None, - exception: Exception, -): - with exception: - result = ConfusionMatrix._drop_extra_matches(matches) - - assert np.array_equal(result, expected_result) - - -@pytest.mark.parametrize( - ("recall", "precision", "expected_result", "exception"), - [ - ( - np.array([1.0]), - np.array([1.0]), - 1.0, - DoesNotRaise(), - ), # perfect recall and precision - ( - np.array([0.0]), - np.array([0.0]), - 0.0, - DoesNotRaise(), - ), # no recall and precision - ( - np.array([0.0, 0.2, 0.2, 0.8, 0.8, 1.0]), - np.array([0.7, 0.8, 0.4, 0.5, 0.1, 0.2]), - 0.5, - DoesNotRaise(), - ), - ( - np.array([0.0, 0.5, 0.5, 1.0]), - np.array([0.75, 0.75, 0.75, 0.75]), - 0.75, - DoesNotRaise(), - ), - ], -) -def test_compute_average_precision( - recall: np.ndarray, - precision: np.ndarray, - expected_result: float, - exception: Exception, -) -> None: - with exception: - result = MeanAveragePrecision.compute_average_precision( - recall=recall, precision=precision - ) - assert_almost_equal(result, expected_result, tolerance=0.01) + @pytest.mark.parametrize( + ("recall", "precision", "expected_result", "exception"), + [ + ( + np.array([1.0]), + np.array([1.0]), + 1.0, + DoesNotRaise(), + ), # perfect recall and precision + ( + np.array([0.0]), + np.array([0.0]), + 0.0, + DoesNotRaise(), + ), # no recall and precision + ( + np.array([0.0, 0.2, 0.2, 0.8, 0.8, 1.0]), + np.array([0.7, 0.8, 0.4, 0.5, 0.1, 0.2]), + 0.5, + DoesNotRaise(), + ), + ( + np.array([0.0, 0.5, 0.5, 1.0]), + np.array([0.75, 0.75, 0.75, 0.75]), + 0.75, + DoesNotRaise(), + ), + ], + ) + def test_compute_average_precision( + self, + recall: np.ndarray, + precision: np.ndarray, + expected_result: float, + exception: Exception, + ) -> None: + with exception: + result = MeanAveragePrecision.compute_average_precision( + recall=recall, precision=precision + ) + assert_almost_equal(result, expected_result, tolerance=0.01) diff --git a/test/metrics/test_f1_score.py b/test/metrics/test_f1_score.py index a7eb8ed2..7e6ceae7 100644 --- a/test/metrics/test_f1_score.py +++ b/test/metrics/test_f1_score.py @@ -7,143 +7,35 @@ from supervision.metrics.f1_score import F1Score from test.helpers import assert_almost_equal -@pytest.fixture -def detections_50_50(): - return Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_50_50(): - return Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def dummy_prediction(): - return Detections( - xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32), - confidence=np.array([0.8]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def predictions_no_overlap(): - return Detections( - xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_no_overlap(): - return Detections( - xyxy=np.array([[100, 100, 110, 110]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_two_objects_class_0(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], - [100, 100, 110, 110], - ], - dtype=np.float32, - ), - class_id=np.array([0, 0]), - ) - - -@pytest.fixture -def predictions_multiple_classes(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # class 0, matches target - [60, 60, 100, 100], # class 1, matches target - [120, 120, 130, 130], # class 1, false positive - ], - dtype=np.float32, - ), - confidence=np.array([0.9, 0.8, 0.7]), - class_id=np.array([0, 1, 1]), - ) - - -@pytest.fixture -def targets_multiple_classes(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # class 0 - [60, 60, 100, 100], # class 1 - ], - dtype=np.float32, - ), - class_id=np.array([0, 1]), - ) - - -@pytest.fixture -def predictions_iou_064(): - return Detections( - xyxy=np.array([[15, 15, 55, 55]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_iou_064(): - return Detections( - xyxy=np.array([[10, 10, 60, 60]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def predictions_confidence_ranking(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # low confidence, perfect match - [11, 11, 49, 49], # high confidence, good match - ], - dtype=np.float32, - ), - confidence=np.array([0.6, 0.9]), # second has higher confidence - class_id=np.array([0, 0]), - ) - - -@pytest.fixture -def prediction_class_1(): - return Detections( - xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32), - confidence=np.array([0.8]), - class_id=np.array([1]), - ) - - -@pytest.fixture -def target_class_1(): - return Detections( - xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32), - class_id=np.array([1]), - ) - - class TestF1Score: + @pytest.fixture + def predictions_multiple_classes(self): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], # class 0, matches target + [60, 60, 100, 100], # class 1, matches target + [120, 120, 130, 130], # class 1, false positive + ], + dtype=np.float32, + ), + confidence=np.array([0.9, 0.8, 0.7]), + class_id=np.array([0, 1, 1]), + ) + + @pytest.fixture + def targets_multiple_classes(self): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], # class 0 + [60, 60, 100, 100], # class 1 + ], + dtype=np.float32, + ), + class_id=np.array([0, 1]), + ) + def test_initialization_default(self): """Test that F1Score can be initialized with default parameters""" metric = F1Score() diff --git a/test/metrics/test_mean_average_precision.py b/test/metrics/test_mean_average_precision.py index f17cb1ce..9d19fd79 100644 --- a/test/metrics/test_mean_average_precision.py +++ b/test/metrics/test_mean_average_precision.py @@ -1,328 +1,300 @@ -""" -Tests for Mean Average Precision ID=0 bug fix -""" - import numpy as np from supervision.detection.core import Detections from supervision.metrics.mean_average_precision import MeanAveragePrecision -def test_single_perfect_detection(): - """Test that single perfect detection gets 1.0 mAP (not 0.0 due to ID=0 bug)""" - # Perfect detection (identical prediction and target) - detection = Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float64), - class_id=np.array([0]), - confidence=np.array([0.9]), - ) +class TestMeanAveragePrecision: + def test_single_perfect_detection(self, detections_50_50, targets_50_50): + """Test that single perfect detection gets 1.0 mAP (not 0.0 due to ID=0 bug)""" + metric = MeanAveragePrecision() + metric.update([detections_50_50], [targets_50_50]) + result = metric.compute() - metric = MeanAveragePrecision() - metric.update([detection], [detection]) - result = metric.compute() + # Should be perfect 1.0 mAP, not 0.0 due to ID=0 bug + assert abs(result.map50_95 - 1.0) < 1e-6 - # Should be perfect 1.0 mAP, not 0.0 due to ID=0 bug - assert abs(result.map50_95 - 1.0) < 1e-6 + def test_multiple_perfect_detections(self): + """Test that multiple perfect detections get 1.0 mAP""" + # Multiple perfect detections in one image + detections = Detections( + xyxy=np.array( + [[10, 10, 50, 50], [100, 100, 140, 140], [200, 200, 240, 240]], + dtype=np.float64, + ), + class_id=np.array([0, 0, 0]), + confidence=np.array([0.9, 0.9, 0.9]), + ) + metric = MeanAveragePrecision() + metric.update([detections], [detections]) + result = metric.compute() -def test_multiple_perfect_detections(): - """Test that multiple perfect detections get 1.0 mAP""" - # Multiple perfect detections in one image - detections = Detections( - xyxy=np.array( - [[10, 10, 50, 50], [100, 100, 140, 140], [200, 200, 240, 240]], - dtype=np.float64, - ), - class_id=np.array([0, 0, 0]), - confidence=np.array([0.9, 0.9, 0.9]), - ) + # Should be perfect 1.0 mAP + assert abs(result.map50_95 - 1.0) < 1e-6 - metric = MeanAveragePrecision() - metric.update([detections], [detections]) - result = metric.compute() + def test_batch_updates_perfect_detections(self, detections_50_50, targets_50_50): + """Test that batch updates with perfect detections get 1.0 mAP""" + metric = MeanAveragePrecision() + # Add 3 batch updates + metric.update([detections_50_50], [targets_50_50]) + metric.update([detections_50_50], [targets_50_50]) + metric.update([detections_50_50], [targets_50_50]) + result = metric.compute() - # Should be perfect 1.0 mAP - assert abs(result.map50_95 - 1.0) < 1e-6 + # Should be perfect 1.0 mAP across all batches + assert abs(result.map50_95 - 1.0) < 1e-6 - -def test_batch_updates_perfect_detections(): - """Test that batch updates with perfect detections get 1.0 mAP""" - # Single perfect detection for multiple batch updates - detection = Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float64), - class_id=np.array([0]), - confidence=np.array([0.9]), - ) - - metric = MeanAveragePrecision() - # Add 3 batch updates - metric.update([detection], [detection]) - metric.update([detection], [detection]) - metric.update([detection], [detection]) - result = metric.compute() - - # Should be perfect 1.0 mAP across all batches - assert abs(result.map50_95 - 1.0) < 1e-6 - - -def test_scenario_1_success_case_imperfect_match(): - """Scenario 1: Success Case with imperfect match""" - # Small object (class 0) - area = 30*30 = 900 < 1024 - small_perfect = Detections( - xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), - class_id=np.array([0]), - confidence=np.array([0.95]), - data={"area": np.array([900])}, - ) - - # Medium object (class 1) - area = 50*50 = 2500 (between 1024 and 9216) - medium_target = Detections( - xyxy=np.array([[10, 10, 60, 60]], dtype=np.float64), - class_id=np.array([1]), - data={"area": np.array([2500])}, - ) - medium_pred = Detections( - xyxy=np.array([[12, 12, 60, 60]], dtype=np.float64), # Slightly off - class_id=np.array([1]), - confidence=np.array([0.9]), - data={"area": np.array([2304])}, # 48*48 - ) - - # Large objects (classes 0, 1, 2) - area = 100*100 = 10000 > 9216 - large_targets = Detections( - xyxy=np.array( - [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], - dtype=np.float64, - ), - class_id=np.array([2, 0, 1]), - data={"area": np.array([10000, 10000, 10000])}, - ) - large_preds = Detections( - xyxy=np.array( - [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], - dtype=np.float64, - ), - class_id=np.array([2, 0, 1]), - confidence=np.array([0.9, 0.9, 0.9]), - data={"area": np.array([10000, 10000, 10000])}, - ) - - metric = MeanAveragePrecision() - metric.update([small_perfect], [small_perfect]) - metric.update([medium_pred], [medium_target]) - metric.update([large_preds], [large_targets]) - result = metric.compute() - - # Should be close to 0.9 (slightly less than perfect due to medium object) - assert 0.85 < result.map50_95 < 0.98 # Adjusted upper bound - assert result.medium_objects.map50_95 < 1.0 # Medium should be less than perfect - - -def test_scenario_2_missed_detection(): - """Scenario 2: GT Present, No Prediction (Missed Detection)""" - # Small object - area = 30*30 = 900 < 1024 - small_detection = Detections( - xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), - class_id=np.array([0]), - confidence=np.array([0.95]), - data={"area": np.array([900])}, - ) - - # Medium object - area = 50*50 = 2500 (between 1024 and 9216) - missed - medium_target = Detections( - xyxy=np.array([[10, 10, 60, 60]], dtype=np.float64), - class_id=np.array([1]), - data={"area": np.array([2500])}, - ) - no_medium_pred = Detections.empty() - - # Large objects - area = 100*100 = 10000 > 9216 - large_detections = Detections( - xyxy=np.array( - [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], - dtype=np.float64, - ), - class_id=np.array([2, 0, 1]), - confidence=np.array([0.9, 0.9, 0.9]), - data={"area": np.array([10000, 10000, 10000])}, - ) - - metric = MeanAveragePrecision() - metric.update([small_detection], [small_detection]) - metric.update([no_medium_pred], [medium_target]) - metric.update([large_detections], [large_detections]) - result = metric.compute() - - # Medium objects should have 0.0 mAP (missed detection) - assert abs(result.medium_objects.map50_95 - 0.0) < 1e-6 - - -def test_scenario_3_false_positive(): - """Scenario 3: No GT, Prediction Present (False Positive)""" - # Small object - area = 30*30 = 900 < 1024 - small_detection = Detections( - xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), - class_id=np.array([0]), - confidence=np.array([0.95]), - data={"area": np.array([900])}, - ) - - # Medium object - area = 50*50 = 2500 - false positive (no GT) - medium_pred = Detections( - xyxy=np.array([[12, 12, 62, 62]], dtype=np.float64), - class_id=np.array([1]), - confidence=np.array([0.9]), - data={"area": np.array([2500])}, - ) - no_medium_target = Detections.empty() - - # Large objects - area = 100*100 = 10000 > 9216 - large_detections = Detections( - xyxy=np.array( - [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], - dtype=np.float64, - ), - class_id=np.array([2, 0, 1]), - confidence=np.array([0.9, 0.9, 0.9]), - data={"area": np.array([10000, 10000, 10000])}, - ) - - metric = MeanAveragePrecision() - metric.update([small_detection], [small_detection]) - metric.update([medium_pred], [no_medium_target]) - metric.update([large_detections], [large_detections]) - result = metric.compute() - - # Medium objects should have -1 mAP (false positive, matching pycocotools) - assert result.medium_objects.map50_95 == -1 - - -def test_scenario_4_no_data(): - """Scenario 4: No GT, No Prediction (Category has no data)""" - # Small object - area = 30*30 = 900 < 1024 - small_detection = Detections( - xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), - class_id=np.array([0]), - confidence=np.array([0.95]), - data={"area": np.array([900])}, - ) - - # Medium object - no data at all - no_medium = Detections.empty() - - # Large objects - area = 100*100 = 10000 > 9216 - only classes 0 and 2 (no class 1) - large_targets = Detections( - xyxy=np.array( - [ - [10, 10, 110, 110], - [120, 120, 220, 220], - ], - dtype=np.float64, - ), - class_id=np.array([2, 0]), - data={"area": np.array([10000, 10000])}, - ) - large_preds = Detections( - xyxy=np.array( - [ - [10, 10, 110, 110], - [120, 120, 220, 220], - ], - dtype=np.float64, - ), - class_id=np.array([2, 0]), - confidence=np.array([0.9, 0.9]), - data={"area": np.array([10000, 10000])}, - ) - - metric = MeanAveragePrecision() - metric.update([small_detection], [small_detection]) - metric.update([no_medium], [no_medium]) - metric.update([large_preds], [large_targets]) - result = metric.compute() - - # Should NOT have negative mAP values for overall - assert result.map50_95 >= 0.0 - # Medium objects should have -1 mAP (no data, matching pycocotools) - assert result.medium_objects.map50_95 == -1 - - -def test_scenario_5_only_one_class_present(): - """Scenario 5: Only 1 of 3 Classes Present (Perfect Match)""" - # Only class 0 objects with perfect matches - detections_class_0 = [ - Detections( + def test_scenario_1_success_case_imperfect_match(self): + """Scenario 1: Success Case with imperfect match""" + # Small object (class 0) - area = 30*30 = 900 < 1024 + small_perfect = Detections( xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), class_id=np.array([0]), confidence=np.array([0.95]), - ), - Detections( - xyxy=np.array([[20, 20, 230, 130]], dtype=np.float64), - class_id=np.array([0]), + data={"area": np.array([900])}, + ) + + # Medium object (class 1) - area = 50*50 = 2500 (between 1024 and 9216) + medium_target = Detections( + xyxy=np.array([[10, 10, 60, 60]], dtype=np.float64), + class_id=np.array([1]), + data={"area": np.array([2500])}, + ) + medium_pred = Detections( + xyxy=np.array([[12, 12, 60, 60]], dtype=np.float64), # Slightly off + class_id=np.array([1]), confidence=np.array([0.9]), - ), - ] + data={"area": np.array([2304])}, # 48*48 + ) - metric = MeanAveragePrecision() - for det in detections_class_0: - metric.update([det], [det]) + # Large objects (classes 0, 1, 2) - area = 100*100 = 10000 > 9216 + large_targets = Detections( + xyxy=np.array( + [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], + dtype=np.float64, + ), + class_id=np.array([2, 0, 1]), + data={"area": np.array([10000, 10000, 10000])}, + ) + large_preds = Detections( + xyxy=np.array( + [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], + dtype=np.float64, + ), + class_id=np.array([2, 0, 1]), + confidence=np.array([0.9, 0.9, 0.9]), + data={"area": np.array([10000, 10000, 10000])}, + ) - result = metric.compute() + metric = MeanAveragePrecision() + metric.update([small_perfect], [small_perfect]) + metric.update([medium_pred], [medium_target]) + metric.update([large_preds], [large_targets]) + result = metric.compute() - # Should be 1.0 mAP (perfect match for the only class present) - assert abs(result.map50_95 - 1.0) < 1e-6 - assert abs(result.map50 - 1.0) < 1e-6 - assert abs(result.map75 - 1.0) < 1e-6 + # Should be close to 0.9 (slightly less than perfect due to medium object) + assert 0.85 < result.map50_95 < 0.98 # Adjusted upper bound + assert ( + result.medium_objects.map50_95 < 1.0 + ) # Medium should be less than perfect + def test_scenario_2_missed_detection(self): + """Scenario 2: GT Present, No Prediction (Missed Detection)""" + # Small object - area = 30*30 = 900 < 1024 + small_detection = Detections( + xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.95]), + data={"area": np.array([900])}, + ) -def test_mixed_classes_with_missing_detections(): - """Test mixed scenario with some classes having no detections""" - # Class 0: Perfect detection - class_0_det = Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float64), - class_id=np.array([0]), - confidence=np.array([0.9]), - ) + # Medium object - area = 50*50 = 2500 (between 1024 and 9216) - missed + medium_target = Detections( + xyxy=np.array([[10, 10, 60, 60]], dtype=np.float64), + class_id=np.array([1]), + data={"area": np.array([2500])}, + ) + no_medium_pred = Detections.empty() - # Class 1: GT exists but no prediction - class_1_target = Detections( - xyxy=np.array([[60, 60, 100, 100]], dtype=np.float64), - class_id=np.array([1]), - ) - class_1_pred = Detections.empty() + # Large objects - area = 100*100 = 10000 > 9216 + large_detections = Detections( + xyxy=np.array( + [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], + dtype=np.float64, + ), + class_id=np.array([2, 0, 1]), + confidence=np.array([0.9, 0.9, 0.9]), + data={"area": np.array([10000, 10000, 10000])}, + ) - # Class 2: Prediction exists but no GT (false positive) - class_2_pred = Detections( - xyxy=np.array([[110, 110, 150, 150]], dtype=np.float64), - class_id=np.array([2]), - confidence=np.array([0.8]), - ) - class_2_target = Detections.empty() + metric = MeanAveragePrecision() + metric.update([small_detection], [small_detection]) + metric.update([no_medium_pred], [medium_target]) + metric.update([large_detections], [large_detections]) + result = metric.compute() - metric = MeanAveragePrecision() - metric.update([class_0_det], [class_0_det]) - metric.update([class_1_pred], [class_1_target]) - metric.update([class_2_pred], [class_2_target]) - result = metric.compute() + # Medium objects should have 0.0 mAP (missed detection) + assert abs(result.medium_objects.map50_95 - 0.0) < 1e-6 - # Should not have negative mAP - assert result.map50_95 >= 0.0 - # Should be less than 1.0 due to missed detection and false positive - assert result.map50_95 < 1.0 + def test_scenario_3_false_positive(self): + """Scenario 3: No GT, Prediction Present (False Positive)""" + # Small object - area = 30*30 = 900 < 1024 + small_detection = Detections( + xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.95]), + data={"area": np.array([900])}, + ) + # Medium object - area = 50*50 = 2500 - false positive (no GT) + medium_pred = Detections( + xyxy=np.array([[12, 12, 62, 62]], dtype=np.float64), + class_id=np.array([1]), + confidence=np.array([0.9]), + data={"area": np.array([2500])}, + ) + no_medium_target = Detections.empty() -def test_empty_predictions_and_targets(): - """Test completely empty predictions and targets""" - metric = MeanAveragePrecision() - metric.update([Detections.empty()], [Detections.empty()]) - result = metric.compute() + # Large objects - area = 100*100 = 10000 > 9216 + large_detections = Detections( + xyxy=np.array( + [[10, 10, 110, 110], [120, 120, 220, 220], [230, 230, 330, 330]], + dtype=np.float64, + ), + class_id=np.array([2, 0, 1]), + confidence=np.array([0.9, 0.9, 0.9]), + data={"area": np.array([10000, 10000, 10000])}, + ) - # Should return -1 for no data (matching pycocotools behavior) - assert result.map50_95 == -1 - assert result.map50 == -1 - assert result.map75 == -1 + metric = MeanAveragePrecision() + metric.update([small_detection], [small_detection]) + metric.update([medium_pred], [no_medium_target]) + metric.update([large_detections], [large_detections]) + result = metric.compute() - # All object size categories should also be -1 - assert result.small_objects.map50_95 == -1 - assert result.medium_objects.map50_95 == -1 - assert result.large_objects.map50_95 == -1 + # Medium objects should have -1 mAP (false positive, matching pycocotools) + assert result.medium_objects.map50_95 == -1 + + def test_scenario_4_no_data(self): + """Scenario 4: No GT, No Prediction (Category has no data)""" + # Small object - area = 30*30 = 900 < 1024 + small_detection = Detections( + xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.95]), + data={"area": np.array([900])}, + ) + + # Medium object - no data at all + no_medium = Detections.empty() + + # Large objects - area = 100*100 = 10000 > 9216 + # only classes 0 and 2 (no class 1) + large_targets = Detections( + xyxy=np.array( + [ + [10, 10, 110, 110], + [120, 120, 220, 220], + ], + dtype=np.float64, + ), + class_id=np.array([2, 0]), + data={"area": np.array([10000, 10000])}, + ) + large_preds = Detections( + xyxy=np.array( + [ + [10, 10, 110, 110], + [120, 120, 220, 220], + ], + dtype=np.float64, + ), + class_id=np.array([2, 0]), + confidence=np.array([0.9, 0.9]), + data={"area": np.array([10000, 10000])}, + ) + + metric = MeanAveragePrecision() + metric.update([small_detection], [small_detection]) + metric.update([no_medium], [no_medium]) + metric.update([large_preds], [large_targets]) + result = metric.compute() + + # Should NOT have negative mAP values for overall + assert result.map50_95 >= 0.0 + # Medium objects should have -1 mAP (no data, matching pycocotools) + assert result.medium_objects.map50_95 == -1 + + def test_scenario_5_only_one_class_present(self): + """Scenario 5: Only 1 of 3 Classes Present (Perfect Match)""" + # Only class 0 objects with perfect matches + detections_class_0 = [ + Detections( + xyxy=np.array([[10, 10, 40, 40]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.95]), + ), + Detections( + xyxy=np.array([[20, 20, 230, 130]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.9]), + ), + ] + + metric = MeanAveragePrecision() + for det in detections_class_0: + metric.update([det], [det]) + + result = metric.compute() + + # Should be 1.0 mAP (perfect match for the only class present) + assert abs(result.map50_95 - 1.0) < 1e-6 + assert abs(result.map50 - 1.0) < 1e-6 + assert abs(result.map75 - 1.0) < 1e-6 + + def test_mixed_classes_with_missing_detections( + self, detections_50_50, targets_50_50 + ): + """Test mixed scenario with some classes having no detections""" + # Class 1: GT exists but no prediction + class_1_target = Detections( + xyxy=np.array([[60, 60, 100, 100]], dtype=np.float64), + class_id=np.array([1]), + ) + class_1_pred = Detections.empty() + + # Class 2: Prediction exists but no GT (false positive) + class_2_pred = Detections( + xyxy=np.array([[110, 110, 150, 150]], dtype=np.float64), + class_id=np.array([2]), + confidence=np.array([0.8]), + ) + class_2_target = Detections.empty() + + metric = MeanAveragePrecision() + metric.update([detections_50_50], [targets_50_50]) + metric.update([class_1_pred], [class_1_target]) + metric.update([class_2_pred], [class_2_target]) + result = metric.compute() + + # Should not have negative mAP + assert result.map50_95 >= 0.0 + # Should be less than 1.0 due to missed detection and false positive + assert result.map50_95 < 1.0 + + def test_empty_predictions_and_targets(self): + """Test completely empty predictions and targets""" + metric = MeanAveragePrecision() + metric.update([Detections.empty()], [Detections.empty()]) + result = metric.compute() + + # Should return -1 for no data (matching pycocotools behavior) + assert result.map50_95 == -1 + assert result.map50 == -1 + assert result.map75 == -1 + + # All object size categories should also be -1 + assert result.small_objects.map50_95 == -1 + assert result.medium_objects.map50_95 == -1 + assert result.large_objects.map50_95 == -1 diff --git a/test/metrics/test_precision.py b/test/metrics/test_precision.py index 2b3c69d4..f2c1fe9f 100644 --- a/test/metrics/test_precision.py +++ b/test/metrics/test_precision.py @@ -1,7 +1,3 @@ -""" -Tests for Precision metric -""" - import numpy as np import pytest @@ -10,130 +6,36 @@ from supervision.metrics.core import AveragingMethod, MetricTarget from supervision.metrics.precision import Precision -@pytest.fixture -def detections_50_50(): - return Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_50_50(): - return Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def dummy_prediction(): - return Detections( - xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32), - confidence=np.array([0.8]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def predictions_no_overlap(): - return Detections( - xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_no_overlap(): - return Detections( - xyxy=np.array([[100, 100, 110, 110]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def predictions_multiple_classes(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # class 0, matches target - [60, 60, 100, 100], # class 0, matches target - [200, 200, 240, 240], # class 1, matches target - ], - dtype=np.float32, - ), - confidence=np.array([0.9, 0.8, 0.7]), - class_id=np.array([0, 0, 1]), - ) - - -@pytest.fixture -def targets_multiple_classes(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # class 0 - [60, 60, 100, 100], # class 0 - [200, 200, 240, 240], # class 1 - ], - dtype=np.float32, - ), - class_id=np.array([0, 0, 1]), - ) - - -@pytest.fixture -def predictions_iou_064(): - return Detections( - xyxy=np.array([[15, 15, 55, 55]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_iou_064(): - return Detections( - xyxy=np.array([[10, 10, 60, 60]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def predictions_confidence_ranking(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # low confidence, perfect match - [11, 11, 49, 49], # high confidence, good match - ], - dtype=np.float32, - ), - confidence=np.array([0.6, 0.9]), # second has higher confidence - class_id=np.array([0, 0]), - ) - - -@pytest.fixture -def prediction_class_1(): - return Detections( - xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32), - confidence=np.array([0.8]), - class_id=np.array([1]), - ) - - -@pytest.fixture -def target_class_1(): - return Detections( - xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32), - class_id=np.array([1]), - ) - - class TestPrecision: + @pytest.fixture + def predictions_multiple_classes(self): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], # class 0, matches target + [60, 60, 100, 100], # class 0, matches target + [200, 200, 240, 240], # class 1, matches target + ], + dtype=np.float32, + ), + confidence=np.array([0.9, 0.8, 0.7]), + class_id=np.array([0, 0, 1]), + ) + + @pytest.fixture + def targets_multiple_classes(self): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], # class 0 + [60, 60, 100, 100], # class 0 + [200, 200, 240, 240], # class 1 + ], + dtype=np.float32, + ), + class_id=np.array([0, 0, 1]), + ) + def test_initialization_default(self): """Test that Precision can be initialized with default parameters""" metric = Precision() diff --git a/test/metrics/test_recall.py b/test/metrics/test_recall.py index 1c89298f..48ab16aa 100644 --- a/test/metrics/test_recall.py +++ b/test/metrics/test_recall.py @@ -1,7 +1,3 @@ -""" -Tests for Recall metric -""" - import numpy as np import pytest @@ -11,143 +7,35 @@ from supervision.metrics.recall import Recall from test.helpers import assert_almost_equal -@pytest.fixture -def detections_50_50(): - return Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_50_50(): - return Detections( - xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def dummy_prediction(): - return Detections( - xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32), - confidence=np.array([0.8]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def predictions_no_overlap(): - return Detections( - xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_no_overlap(): - return Detections( - xyxy=np.array([[100, 100, 110, 110]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_two_objects_class_0(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], - [100, 100, 110, 110], - ], - dtype=np.float32, - ), - class_id=np.array([0, 0]), - ) - - -@pytest.fixture -def predictions_multiple_classes(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # class 0, matches first target - [200, 200, 240, 240], # class 1, matches target - ], - dtype=np.float32, - ), - confidence=np.array([0.9, 0.8]), - class_id=np.array([0, 1]), - ) - - -@pytest.fixture -def targets_multiple_classes(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # class 0, matched - [60, 60, 100, 100], # class 0, missed - [200, 200, 240, 240], # class 1, matched - ], - dtype=np.float32, - ), - class_id=np.array([0, 0, 1]), - ) - - -@pytest.fixture -def predictions_iou_064(): - return Detections( - xyxy=np.array([[15, 15, 55, 55]], dtype=np.float32), - confidence=np.array([0.9]), - class_id=np.array([0]), - ) - - -@pytest.fixture -def targets_iou_064(): - return Detections( - xyxy=np.array([[10, 10, 60, 60]], dtype=np.float32), - class_id=np.array([0]), - ) - - -@pytest.fixture -def predictions_confidence_ranking(): - return Detections( - xyxy=np.array( - [ - [10, 10, 50, 50], # low confidence, perfect match - [11, 11, 49, 49], # high confidence, good match - ], - dtype=np.float32, - ), - confidence=np.array([0.6, 0.9]), # second has higher confidence - class_id=np.array([0, 0]), - ) - - -@pytest.fixture -def prediction_class_1(): - return Detections( - xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32), - confidence=np.array([0.8]), - class_id=np.array([1]), - ) - - -@pytest.fixture -def target_class_1(): - return Detections( - xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32), - class_id=np.array([1]), - ) - - class TestRecall: + @pytest.fixture + def predictions_multiple_classes(self): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], # class 0, matches first target + [200, 200, 240, 240], # class 1, matches target + ], + dtype=np.float32, + ), + confidence=np.array([0.9, 0.8]), + class_id=np.array([0, 1]), + ) + + @pytest.fixture + def targets_multiple_classes(self): + return Detections( + xyxy=np.array( + [ + [10, 10, 50, 50], # class 0, matched + [60, 60, 100, 100], # class 0, missed + [200, 200, 240, 240], # class 1, matched + ], + dtype=np.float32, + ), + class_id=np.array([0, 0, 1]), + ) + def test_initialization_default(self): """Test that Recall can be initialized with default parameters""" metric = Recall()