diff --git a/supervision/detection/core.py b/supervision/detection/core.py index f508dd62..9cda122a 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -40,6 +40,7 @@ from supervision.detection.vlm import ( from_paligemma, from_qwen_2_5_vl, validate_vlm_parameters, + from_deepseek_vl_2, ) from supervision.geometry.core import Position from supervision.utils.internal import deprecated, get_instance_variables @@ -849,6 +850,7 @@ class Detections: LMM.PALIGEMMA: VLM.PALIGEMMA, LMM.FLORENCE_2: VLM.FLORENCE_2, LMM.QWEN_2_5_VL: VLM.QWEN_2_5_VL, + LMM.DEEPSEEK_VL_2: VLM.DEEPSEEK_VL_2 } # (this works even if the LMM enum is wrapped by @deprecated) @@ -876,6 +878,41 @@ class Detections: def from_vlm( cls, vlm: Union[VLM, str], result: Union[str, dict], **kwargs: Any ) -> Detections: + """ + Creates a Detections object from the given result string based on the specified + Vision-Language Model (LMM). + + Args: + vlm (Union[VLM, str]): The type of VLM (Vision-Language Model) to use. + result (str): The result string containing the detection data. + **kwargs (Any): Additional keyword arguments required by the specified VLM. + + Returns: + Detections: A new Detections object. + + Raises: + ValueError: If the LMM is invalid, required arguments are missing, or + disallowed arguments are provided. + ValueError: If the specified LMM is not supported. + + Examples: + ```python + import supervision as sv + + paligemma_result = " cat" + detections = sv.Detections.from_vlm( + sv.VLM.PALIGEMMA, + paligemma_result, + resolution_wh=(1000, 1000), + classes=['cat', 'dog'] + ) + detections.xyxy + # array([[250., 250., 750., 750.]]) + + detections.class_id + # array([0]) + ``` + """ vlm = validate_vlm_parameters(vlm, result, kwargs) if vlm == VLM.PALIGEMMA: @@ -888,6 +925,11 @@ class Detections: data = {CLASS_NAME_DATA_FIELD: class_name} return cls(xyxy=xyxy, class_id=class_id, data=data) + if vlm == VLM.DEEPSEEK_VL_2: + xyxy, class_id, class_name = from_deepseek_vl_2(result, **kwargs) + data = {CLASS_NAME_DATA_FIELD: class_name} + return cls(xyxy=xyxy, class_id=class_id, data=data) + if vlm == VLM.FLORENCE_2: xyxy, labels, mask, xyxyxyxy = from_florence_2(result, **kwargs) if len(xyxy) == 0: diff --git a/supervision/detection/vlm.py b/supervision/detection/vlm.py index 719dc443..e9b72bef 100644 --- a/supervision/detection/vlm.py +++ b/supervision/detection/vlm.py @@ -1,5 +1,6 @@ import json import re +import ast from enum import Enum from typing import Any, Dict, List, Optional, Tuple, Union @@ -17,30 +18,35 @@ class LMM(Enum): PALIGEMMA = "paligemma" FLORENCE_2 = "florence_2" QWEN_2_5_VL = "qwen_2_5_vl" + DEEPSEEK_VL_2 = "deepseek_vl_2" class VLM(Enum): PALIGEMMA = "paligemma" FLORENCE_2 = "florence_2" QWEN_2_5_VL = "qwen_2_5_vl" + DEEPSEEK_VL_2 = "deepseek_vl_2" RESULT_TYPES: Dict[VLM, type] = { VLM.PALIGEMMA: str, VLM.FLORENCE_2: dict, VLM.QWEN_2_5_VL: str, + VLM.DEEPSEEK_VL_2: str, } REQUIRED_ARGUMENTS: Dict[VLM, List[str]] = { VLM.PALIGEMMA: ["resolution_wh"], VLM.FLORENCE_2: ["resolution_wh"], VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh"], + VLM.DEEPSEEK_VL_2: ["resolution_wh"], } ALLOWED_ARGUMENTS: Dict[VLM, List[str]] = { VLM.PALIGEMMA: ["resolution_wh", "classes"], VLM.FLORENCE_2: ["resolution_wh"], VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh", "classes"], + VLM.DEEPSEEK_VL_2: ["resolution_wh", "classes"], } SUPPORTED_TASKS_FLORENCE_2 = [ @@ -223,6 +229,92 @@ def from_qwen_2_5_vl( return xyxy, class_id, class_name +def from_deepseek_vl_2( + result: str, + resolution_wh: Tuple[int, int], + classes: Optional[List[str]] = None +) -> Tuple[np.ndarray, Optional[np.ndarray], np.ndarray]: + """ + Parse bounding boxes from deepseek-vl2-formatted text, scale them to the specified + resolution, and optionally filter by classes. + + The DeepSeek-VL2 output typically contains pairs of <|ref|> ... <|/ref|> labels + and <|det|> ... <|/det|> bounding box definitions. Each <|det|> section may + contain one or more bounding boxes in the form [[x1, y1, x2, y2], [x1, y1, x2, y2], ...] + (scaled to 0..999). However, other text (e.g. <｜end▁of▁sentence｜>) may appear + after the bracket, so we strip that out here. + + Args: + result: String containing deepseek-vl2-formatted locations and labels. + resolution_wh: Tuple (width, height) to which we scale the box coordinates. + classes: Optional list of valid class names. If provided, boxes and labels not + in this list are filtered out. + + Returns: + xyxy (np.ndarray): An array of shape `(n, 4)` containing + the bounding boxes coordinates in format `[x1, y1, x2, y2]`. + class_id (Optional[np.ndarray]): An array of shape `(n,)` containing + the class indices for each bounding box (or `None` if classes is not + provided). + class_name (np.ndarray): An array of shape `(n,)` containing + the class labels for each bounding box. + """ + + w, h = resolution_wh + if w <= 0 or h <= 0: + raise ValueError( + f"Both dimensions in resolution_wh must be positive. Got ({w}, {h})." + ) + + label_segments = re.findall(r'<\|ref\|>(.*?)<\|/ref\|>', result, flags=re.DOTALL) + bbox_segments = re.findall(r'<\|det\|>(.*?)<\|/det\|>', result, flags=re.DOTALL) + + if len(label_segments) != len(bbox_segments): + return np.empty((0, 4)), None, np.empty((0,), dtype=str) + + boxes, labels = [], [] + + for label_str, bbox_str in zip(label_segments, bbox_segments): + label_str = label_str.strip() + raw_box_groups = re.findall(r'\[\[.*?\]\]', bbox_str, flags=re.DOTALL) + + if not raw_box_groups: + continue + + for group_str in raw_box_groups: + try: + list_of_boxes = ast.literal_eval(group_str) + for box in list_of_boxes: + if len(box) != 4: + continue + + x1 = box[0] / 999.0 * w + y1 = box[1] / 999.0 * h + x2 = box[2] / 999.0 * w + y2 = box[3] / 999.0 * h + + boxes.append([x1, y1, x2, y2]) + labels.append(label_str) + + except (SyntaxError, ValueError): + continue + + if len(boxes) == 0: + return np.empty((0, 4)), None, np.empty((0,), dtype=str) + + xyxy = np.array(boxes, dtype=np.float32) + class_name = np.array(labels, dtype=str) + class_id = None + + if classes is not None: + mask = np.array([name in classes for name in class_name], dtype=bool) + xyxy = xyxy[mask] + class_name = class_name[mask] + class_id = np.array([classes.index(name) for name in class_name]) + + return xyxy, class_id, class_name + + def from_florence_2( result: dict, resolution_wh: Tuple[int, int] ) -> Tuple[