initial commit

supervision/detection/core.py

@@ -40,6 +40,7 @@ from supervision.detection.vlm import (
     from_paligemma,
     from_qwen_2_5_vl,
     validate_vlm_parameters,
+    from_deepseek_vl_2,
 )
 from supervision.geometry.core import Position
 from supervision.utils.internal import deprecated, get_instance_variables
@@ -849,6 +850,7 @@ class Detections:
             LMM.PALIGEMMA: VLM.PALIGEMMA,
             LMM.FLORENCE_2: VLM.FLORENCE_2,
             LMM.QWEN_2_5_VL: VLM.QWEN_2_5_VL,
+            LMM.DEEPSEEK_VL_2: VLM.DEEPSEEK_VL_2
         }
 
         # (this works even if the LMM enum is wrapped by @deprecated)
@@ -876,6 +878,41 @@ class Detections:
     def from_vlm(
         cls, vlm: Union[VLM, str], result: Union[str, dict], **kwargs: Any
     ) -> Detections:
+        """
+        Creates a Detections object from the given result string based on the specified
+        Vision-Language Model (LMM).
+
+        Args:
+            vlm (Union[VLM, str]): The type of VLM (Vision-Language Model) to use.
+            result (str): The result string containing the detection data.
+            **kwargs (Any): Additional keyword arguments required by the specified VLM.
+
+        Returns:
+            Detections: A new Detections object.
+
+        Raises:
+            ValueError: If the LMM is invalid, required arguments are missing, or
+                disallowed arguments are provided.
+            ValueError: If the specified LMM is not supported.
+
+        Examples:
+            ```python
+            import supervision as sv
+
+            paligemma_result = "<loc0256><loc0256><loc0768><loc0768> cat"
+            detections = sv.Detections.from_vlm(
+                sv.VLM.PALIGEMMA,
+                paligemma_result,
+                resolution_wh=(1000, 1000),
+                classes=['cat', 'dog']
+            )
+            detections.xyxy
+            # array([[250., 250., 750., 750.]])
+
+            detections.class_id
+            # array([0])
+            ```
+        """
         vlm = validate_vlm_parameters(vlm, result, kwargs)
 
         if vlm == VLM.PALIGEMMA:
@@ -888,6 +925,11 @@ class Detections:
             data = {CLASS_NAME_DATA_FIELD: class_name}
             return cls(xyxy=xyxy, class_id=class_id, data=data)
 
+        if vlm == VLM.DEEPSEEK_VL_2:
+            xyxy, class_id, class_name = from_deepseek_vl_2(result, **kwargs)
+            data = {CLASS_NAME_DATA_FIELD: class_name}
+            return cls(xyxy=xyxy, class_id=class_id, data=data)
+
         if vlm == VLM.FLORENCE_2:
             xyxy, labels, mask, xyxyxyxy = from_florence_2(result, **kwargs)
             if len(xyxy) == 0:

supervision/detection/vlm.py

@@ -1,5 +1,6 @@
 import json
 import re
+import ast
 from enum import Enum
 from typing import Any, Dict, List, Optional, Tuple, Union
 
@@ -17,30 +18,35 @@ class LMM(Enum):
     PALIGEMMA = "paligemma"
     FLORENCE_2 = "florence_2"
     QWEN_2_5_VL = "qwen_2_5_vl"
+    DEEPSEEK_VL_2 = "deepseek_vl_2"
 
 
 class VLM(Enum):
     PALIGEMMA = "paligemma"
     FLORENCE_2 = "florence_2"
     QWEN_2_5_VL = "qwen_2_5_vl"
+    DEEPSEEK_VL_2 = "deepseek_vl_2"
 
 
 RESULT_TYPES: Dict[VLM, type] = {
     VLM.PALIGEMMA: str,
     VLM.FLORENCE_2: dict,
     VLM.QWEN_2_5_VL: str,
+    VLM.DEEPSEEK_VL_2: str,
 }
 
 REQUIRED_ARGUMENTS: Dict[VLM, List[str]] = {
     VLM.PALIGEMMA: ["resolution_wh"],
     VLM.FLORENCE_2: ["resolution_wh"],
     VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh"],
+    VLM.DEEPSEEK_VL_2: ["resolution_wh"],
 }
 
 ALLOWED_ARGUMENTS: Dict[VLM, List[str]] = {
     VLM.PALIGEMMA: ["resolution_wh", "classes"],
     VLM.FLORENCE_2: ["resolution_wh"],
     VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh", "classes"],
+    VLM.DEEPSEEK_VL_2: ["resolution_wh", "classes"],
 }
 
 SUPPORTED_TASKS_FLORENCE_2 = [
@@ -223,6 +229,92 @@ def from_qwen_2_5_vl(
     return xyxy, class_id, class_name
 
 
+def from_deepseek_vl_2(
+    result: str,
+    resolution_wh: Tuple[int, int],
+    classes: Optional[List[str]] = None
+) -> Tuple[np.ndarray, Optional[np.ndarray], np.ndarray]:
+    """
+    Parse bounding boxes from deepseek-vl2-formatted text, scale them to the specified
+    resolution, and optionally filter by classes.
+
+    The DeepSeek-VL2 output typically contains pairs of <|ref|> ... <|/ref|> labels
+    and <|det|> ... <|/det|> bounding box definitions. Each <|det|> section may
+    contain one or more bounding boxes in the form [[x1, y1, x2, y2], [x1, y1, x2, y2], ...]
+    (scaled to 0..999). However, other text (e.g. <｜end▁of▁sentence｜>) may appear
+    after the bracket, so we strip that out here.
+
+    Args:
+        result: String containing deepseek-vl2-formatted locations and labels.
+        resolution_wh: Tuple (width, height) to which we scale the box coordinates.
+        classes: Optional list of valid class names. If provided, boxes and labels not
+            in this list are filtered out.
+
+    Returns:
+        xyxy (np.ndarray): An array of shape `(n, 4)` containing
+            the bounding boxes coordinates in format `[x1, y1, x2, y2]`.
+        class_id (Optional[np.ndarray]): An array of shape `(n,)` containing
+            the class indices for each bounding box (or `None` if classes is not
+            provided).
+        class_name (np.ndarray): An array of shape `(n,)` containing
+            the class labels for each bounding box.
+    """
+
+    w, h = resolution_wh
+    if w <= 0 or h <= 0:
+        raise ValueError(
+            f"Both dimensions in resolution_wh must be positive. Got ({w}, {h})."
+        )
+
+    label_segments = re.findall(r'<\|ref\|>(.*?)<\|/ref\|>', result, flags=re.DOTALL)
+    bbox_segments = re.findall(r'<\|det\|>(.*?)<\|/det\|>', result, flags=re.DOTALL)
+
+    if len(label_segments) != len(bbox_segments):
+        return np.empty((0, 4)), None, np.empty((0,), dtype=str)
+
+    boxes, labels = [], []
+
+    for label_str, bbox_str in zip(label_segments, bbox_segments):
+        label_str = label_str.strip()
+        raw_box_groups = re.findall(r'\[\[.*?\]\]', bbox_str, flags=re.DOTALL)
+
+        if not raw_box_groups:
+            continue
+
+        for group_str in raw_box_groups:
+            try:
+                list_of_boxes = ast.literal_eval(group_str)
+                for box in list_of_boxes:
+                    if len(box) != 4:
+                        continue
+
+                    x1 = box[0] / 999.0 * w
+                    y1 = box[1] / 999.0 * h
+                    x2 = box[2] / 999.0 * w
+                    y2 = box[3] / 999.0 * h
+
+                    boxes.append([x1, y1, x2, y2])
+                    labels.append(label_str)
+
+            except (SyntaxError, ValueError):
+                continue
+
+    if len(boxes) == 0:
+        return np.empty((0, 4)), None, np.empty((0,), dtype=str)
+
+    xyxy = np.array(boxes, dtype=np.float32)
+    class_name = np.array(labels, dtype=str)
+    class_id = None
+
+    if classes is not None:
+        mask = np.array([name in classes for name in class_name], dtype=bool)
+        xyxy = xyxy[mask]
+        class_name = class_name[mask]
+        class_id = np.array([classes.index(name) for name in class_name])
+
+    return xyxy, class_id, class_name
+
+
 def from_florence_2(
     result: dict, resolution_wh: Tuple[int, int]
 ) -> Tuple[