diff --git a/CHANGELOG.md b/CHANGELOG.md index 79bf2932..64316b45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.21.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.0) - 2026-08-19 + +### Added +- **`Model.create_run(name)` + `ModelRun.add_predictions(predictions, ...)`.** Create a model run with just a name, then attach predictions — no dataset needed up front: + ```python + run = model.create_run(name="my-run") + run.add_predictions(predictions) + ``` + Each prediction identifies its target item by `dataset_item_id` (the `di_*` id returned on exported items), so predictions can come from anywhere and a single run can cover items across multiple datasets. `add_predictions` posts to `POST /nucleus/modelRun/:modelRunId/uploadPredictions` and supports `update` / `batch_size` / file-batching arguments; `asynchronous=True` raises `NotImplementedError` for now. + - `create_run` still accepts the old `dataset=` / `predictions=` arguments for backwards compatibility (the deprecated dataset-bound path); omit them to use the flow above. +- **Per-prediction target.** Every prediction type (`box`, `line`, `polygon`, `keypoints`, `cuboid`, `category`, `scene_category`, `segmentation`) emits its `dataset_item_id` in `to_payload` (as `item_id`) when set, which is how the dataset-less upload route resolves each item. + +### Changed +- **`reference_id` is now optional on predictions.** A prediction can be constructed from its `dataset_item_id` alone (at least one of `reference_id` / `dataset_item_id` is required). Annotations still require `reference_id`. Existing prediction code that passes `reference_id` is unaffected. + +> **Server dependency:** requires the `POST /nucleus/model/:modelId/modelRun/create` and `POST /nucleus/modelRun/:modelRunId/uploadPredictions` routes in scaleapi. Unit tests pass regardless; live calls 404 until that deploys. + ## [0.20.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.20.2) - 2026-08-18 ### Added diff --git a/nucleus/annotation_uploader.py b/nucleus/annotation_uploader.py index c5768988..35a6fe6b 100644 --- a/nucleus/annotation_uploader.py +++ b/nucleus/annotation_uploader.py @@ -273,13 +273,21 @@ def __init__( if route is not None: self._route = route return - if dataset_id is None: - raise ValueError("dataset_id is required to upload predictions.") if model_run_id is not None and model_id is not None: raise ValueError("Pass either model_id or model_run_id, not both.") if model_run_id is not None: - self._route = f"dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions" + if dataset_id is not None: + self._route = f"dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions" + else: + # Dataset-less model run: predictions carry their own item_id + # (or dataset_id + reference_id); the server groups by dataset + # and widens the run. + self._route = f"modelRun/{model_run_id}/uploadPredictions" elif model_id is not None: + if dataset_id is None: + raise ValueError( + "dataset_id is required to upload predictions." + ) self._route = ( f"dataset/{dataset_id}/model/{model_id}/uploadPredictions" ) diff --git a/nucleus/constants.py b/nucleus/constants.py index 6ded5193..f2e0f18d 100644 --- a/nucleus/constants.py +++ b/nucleus/constants.py @@ -41,6 +41,7 @@ DATASET_IS_SCENE_KEY = "is_scene" DATASET_ITEM_ID_KEY = "dataset_item_id" DATASET_ITEM_IDS_KEY = "dataset_item_ids" +ITEM_ID_KEY = "item_id" DATASET_ITEMS_KEY = "dataset_items" DATASET_LENGTH_KEY = "length" DATASET_MODEL_RUNS_KEY = "model_run_ids" diff --git a/nucleus/model.py b/nucleus/model.py index 36659a73..18340571 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -151,41 +151,83 @@ def from_json(cls, payload: dict, client): def create_run( self, name: str, - dataset: Dataset, - predictions: List[ - Union[ - BoxPrediction, - PolygonPrediction, - CuboidPrediction, - SegmentationPrediction, + dataset: Optional[Dataset] = None, + predictions: Optional[ + List[ + Union[ + BoxPrediction, + PolygonPrediction, + CuboidPrediction, + SegmentationPrediction, + ] ] - ], + ] = None, metadata: Optional[Dict] = None, asynchronous: bool = False, + reference_id: Optional[str] = None, ) -> ModelRun: - # This method, as well as model runs in general are now deprecated. - - # Instead models will automatically generate a model run when applied to - # a dataset using dataset.upload_predictions(model, predictions). Therefore - # there is no longer any need to create a model run, since you can upload - # predictions without needing to explicitly create a model run. - - # When uploading to a dataset twice using the same model, the same model - # run will be reused by Nucleus. - - payload: dict = { - NAME_KEY: name, - REFERENCE_ID_KEY: self.reference_id, - } - if metadata: - payload[METADATA_KEY] = metadata - model_run: ModelRun = self._client.create_model_run( - dataset.id, payload - ) + """Creates a model run for this model. + + Call it with just a name to create a run, then attach predictions with + :meth:`ModelRun.add_predictions`:: + + run = model.create_run(name="my-run") + run.add_predictions(predictions) + + or pass ``predictions`` directly to create and upload in one call:: + + run = model.create_run(name="my-run", predictions=predictions) - model_run.predict(predictions, asynchronous=asynchronous) + Each prediction identifies its target item by ``dataset_item_id`` (the + ``di_*`` id returned on exported items), so predictions can come from + anywhere and a single run can cover items across different datasets. - return model_run + Args: + name: Human-readable name for the model run. + predictions: Optional predictions to attach to the run immediately. + metadata: Optional arbitrary metadata blob for the run. + reference_id: Optional user-defined reference id for the run. + dataset: Deprecated. Passing a dataset uses the legacy path that + binds the run to that one dataset and uploads ``predictions`` + to it. Omit it to use the recommended flow above. + asynchronous: Only used by the deprecated dataset-bound path. + + Returns: + The created :class:`ModelRun`. + """ + # Legacy path: an explicit dataset binds the run to that dataset and + # uploads predictions to it. Kept for backwards compatibility; new code + # should omit `dataset` and use `run.add_predictions(...)`. + if dataset is not None: + payload: dict = { + NAME_KEY: name, + REFERENCE_ID_KEY: self.reference_id, + } + if metadata: + payload[METADATA_KEY] = metadata + model_run: ModelRun = self._client.create_model_run( + dataset.id, payload + ) + model_run.predict(predictions or [], asynchronous=asynchronous) + return model_run + + response = self._client.make_request( + { + NAME_KEY: name, + REFERENCE_ID_KEY: reference_id, + METADATA_KEY: metadata or {}, + }, + route=f"model/{self.id}/modelRun/create", + requests_command=requests.post, + ) + run = ModelRun( + model_run_id=response["model_run_id"], + dataset_id=None, + client=self._client, + ) + if predictions: + run.add_predictions(predictions) + return run def evaluate(self, scenario_test_names: List[str]) -> AsyncJob: """Evaluates this on the specified Unit Tests. :: diff --git a/nucleus/model_run.py b/nucleus/model_run.py index d78b7336..aa626ca8 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -45,7 +45,12 @@ class ModelRun: This class is deprecated and will be removed from the python client. """ - def __init__(self, model_run_id: str, dataset_id: str, client): + def __init__( + self, + model_run_id: str, + dataset_id: Optional[str] = None, + client=None, + ): self.model_run_id = model_run_id self._client = client self.dataset_id = dataset_id @@ -173,7 +178,7 @@ def predict( "predictions_ignored": int, } """ - + uploader = PredictionUploader( client=self._client, dataset_id=self.dataset_id, @@ -184,6 +189,11 @@ def predict( if asynchronous: check_all_mask_paths_remote(annotations) + if self.dataset_id is None: + raise ValueError( + "Asynchronous predict() requires a dataset-bound model run. " + "Dataset-less runs must use add_predictions(...)." + ) request_id = serialize_and_write_to_presigned_url( annotations, self.dataset_id, self._client ) @@ -201,6 +211,73 @@ def predict( local_files_per_upload_request=local_files_per_upload_request, ) + def add_predictions( + self, + predictions: List[ + Union[ + BoxPrediction, + PolygonPrediction, + CuboidPrediction, + SegmentationPrediction, + ] + ], + update: bool = False, + asynchronous: bool = False, + batch_size: int = 5000, + remote_files_per_upload_request: int = 20, + local_files_per_upload_request: int = 10, + ) -> dict: + """Adds predictions to this model run. + + Each prediction identifies its target item by ``dataset_item_id`` (the + ``di_*`` id returned on exported items), so predictions can come from + anywhere and be added at any time — a single run can cover items that + live in different datasets. + + Args: + predictions: Predictions to upload. Each must set + ``dataset_item_id`` to identify its item. + update: If True, existing predictions for the same + (reference_id, annotation_id) will be overwritten. If False, + existing predictions will be skipped. + asynchronous: Not supported yet — passing True raises + :class:`NotImplementedError`. + batch_size: Number of predictions processed in each concurrent batch. + remote_files_per_upload_request: Number of remote segmentation files + to upload in each request. + local_files_per_upload_request: Number of local segmentation files + to upload in each request. The maximum is 10. + + Returns:: + + { + "model_run_id": str, + "predictions_processed": int, + "predictions_ignored": int, + } + """ + if asynchronous: + raise NotImplementedError( + "Asynchronous upload is not supported for dataset-less model " + "runs yet. Use " + "dataset.upload_predictions_for_model_run(model_run_id, " + "predictions, asynchronous=True) for the per-dataset route." + ) + + uploader = PredictionUploader( + client=self._client, + model_run_id=self.model_run_id, + ) + uploader.check_for_duplicate_ids(predictions) + + return uploader.upload( + annotations=predictions, + update=update, + batch_size=batch_size, + remote_files_per_upload_request=remote_files_per_upload_request, + local_files_per_upload_request=local_files_per_upload_request, + ) + def iloc(self, i: int): """Returns Model Run Info For Dataset Item by its number. diff --git a/nucleus/prediction.py b/nucleus/prediction.py index ae52c86f..0e44c29b 100644 --- a/nucleus/prediction.py +++ b/nucleus/prediction.py @@ -29,11 +29,13 @@ CLASS_PDF_KEY, CONFIDENCE_KEY, CUBOID_TYPE, + DATASET_ID_KEY, DATASET_ITEM_ID_KEY, DIMENSIONS_KEY, EMBEDDING_VECTOR_KEY, GEOMETRY_KEY, HEIGHT_KEY, + ITEM_ID_KEY, KEYPOINTS_KEY, KEYPOINTS_NAMES_KEY, KEYPOINTS_SKELETON_KEY, @@ -56,6 +58,32 @@ ) +def _add_prediction_target_ids( + payload: dict, + dataset_item_id: Optional[str], + dataset_id: Optional[str], +) -> dict: + """Emit the per-prediction upload target ids used by dataset-less model runs. + + The dataset-less ``modelRun/{run}/uploadPredictions`` endpoint resolves each + prediction's dataset item from either ``item_id`` (the dataset_item_id) or + ``dataset_id`` + ``reference_id``. Both keys are output-only and are only + included when set, keeping regular per-dataset uploads unchanged. + """ + if dataset_item_id is not None: + payload[ITEM_ID_KEY] = dataset_item_id + if dataset_id is not None: + payload[DATASET_ID_KEY] = dataset_id + return payload + + +def _require_prediction_target(reference_id, dataset_item_id): + if reference_id is None and dataset_item_id is None: + raise ValueError( + "A prediction must set reference_id or dataset_item_id." + ) + + def from_json(payload: dict): """Instantiates prediction object from schematized JSON dict payload.""" type_key_to_type: Dict[str, Type[Prediction]] = { @@ -112,7 +140,8 @@ class SegmentationPrediction(SegmentationAnnotation): example above these would map that 0 to background, 1 to car and 2 to pedestrian. In the instance segmentation example above, 0 and 1 would both be mapped to car, 2 and 3 would both be mapped to motorcycle - reference_id (str): User-defined ID of the image to which to apply this annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. annotation_id (Optional[str]): For segmentation predictions, this value is ignored because there can only be one segmentation prediction per dataset item. Therefore regardless of annotation ID, if there is an existing @@ -122,6 +151,34 @@ class SegmentationPrediction(SegmentationAnnotation): to an external database, and its value will be returned for any export. """ + def __init__( + self, + mask_url: str, + annotations: List[Segment], + reference_id: Optional[str] = None, + annotation_id: Optional[str] = None, + dataset_item_id: Optional[str] = None, + *, + dataset_id: Optional[str] = None, + ): + super().__init__( + mask_url=mask_url, + annotations=annotations, + reference_id=reference_id, + annotation_id=annotation_id, + dataset_item_id=dataset_item_id, + ) + _require_prediction_target(reference_id, dataset_item_id) + self.dataset_id = dataset_id + + def to_payload(self) -> dict: + payload = super().to_payload() + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) + + return payload + @classmethod def from_json(cls, payload: dict): return cls( @@ -130,8 +187,9 @@ def from_json(cls, payload: dict): Segment.from_json(ann) for ann in payload.get(ANNOTATIONS_KEY, []) ], - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), annotation_id=payload.get(ANNOTATION_ID_KEY, None), # metadata=payload.get(METADATA_KEY, None), # TODO(sc: 422637) ) @@ -148,8 +206,8 @@ class BoxPrediction(BoxAnnotation): of the bounding box and the top border of the image. width (Union[float, int]): The width in pixels of the annotation. height (Union[float, int]): The height in pixels of the annotation. - reference_id (str): User-defined ID of the image to which to apply this - annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, @@ -185,7 +243,7 @@ def __init__( y: Union[float, int], width: Union[float, int], height: Union[float, int], - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -194,6 +252,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -208,8 +267,10 @@ def __init__( embedding_vector=embedding_vector, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -217,6 +278,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -229,8 +293,9 @@ def from_json(cls, payload: dict): y=geometry.get(Y_KEY, 0), width=geometry.get(WIDTH_KEY, 0), height=geometry.get(HEIGHT_KEY, 0), - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -246,8 +311,8 @@ class LinePrediction(LineAnnotation): Parameters: label (str): The label for this prediction (e.g. car, pedestrian, bicycle). vertices (List[:class:`Point`]): The list of points making up the line. - reference_id (str): User-defined ID of the image to which to apply this - annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, a matching @@ -270,7 +335,7 @@ def __init__( self, label: str, vertices: List[Point], - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -278,6 +343,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -288,8 +354,10 @@ def __init__( metadata=metadata, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -297,6 +365,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -308,8 +379,9 @@ def from_json(cls, payload: dict): vertices=[ Point.from_json(_) for _ in geometry.get(VERTICES_KEY, []) ], - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -324,8 +396,8 @@ class PolygonPrediction(PolygonAnnotation): Parameters: label (str): The label for this annotation (e.g. car, pedestrian, bicycle). vertices (List[:class:`Point`]): The list of points making up the polygon. - reference_id (str): User-defined ID of the image to which to apply this - annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, a matching @@ -351,7 +423,7 @@ def __init__( self, label: str, vertices: List[Point], - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -360,6 +432,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -371,8 +444,10 @@ def __init__( embedding_vector=embedding_vector, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -380,6 +455,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -391,8 +469,9 @@ def from_json(cls, payload: dict): vertices=[ Point.from_json(_) for _ in geometry.get(VERTICES_KEY, []) ], - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -411,8 +490,8 @@ class KeypointsPrediction(KeypointsAnnotation): names (List[str]): A list that corresponds to the names of each keypoint. skeleton (List[List[int]]): A list of 2-length lists indicating a beginning and ending index for each line segment in the skeleton of this keypoint label. - reference_id (str): User-defined ID of the image to which to apply this - annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, a matching @@ -437,7 +516,7 @@ def __init__( keypoints: List[Keypoint], names: List[str], skeleton: List[List[int]], - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -445,6 +524,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -457,8 +537,10 @@ def __init__( metadata=metadata, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -466,6 +548,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -479,8 +564,9 @@ def from_json(cls, payload: dict): ], names=geometry[KEYPOINTS_NAMES_KEY], skeleton=geometry[KEYPOINTS_SKELETON_KEY], - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -497,7 +583,8 @@ class CuboidPrediction(CuboidAnnotation): position (:class:`Point3D`): The point at the center of the cuboid dimensions (:class:`Point3D`): The length (x), width (y), and height (z) of the cuboid yaw (float): The rotation, in radians, about the Z axis of the cuboid - reference_id (str): User-defined ID of the image to which to apply this annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, a matching @@ -522,7 +609,7 @@ def __init__( position: Point3D, dimensions: Point3D, yaw: float, - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -530,6 +617,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -542,8 +630,10 @@ def __init__( metadata=metadata, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -551,6 +641,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -562,8 +655,9 @@ def from_json(cls, payload: dict): position=Point3D.from_json(geometry.get(POSITION_KEY, {})), dimensions=Point3D.from_json(geometry.get(DIMENSIONS_KEY, {})), yaw=geometry.get(YAW_KEY, 0), - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -577,7 +671,8 @@ class CategoryPrediction(CategoryAnnotation): Parameters: label: The label for this annotation (e.g. car, pedestrian, bicycle). - reference_id: The reference ID of the image you wish to apply this annotation to. + reference_id: Optional user-defined ID of the item to which to apply this + prediction. Provide this or ``dataset_item_id``. taxonomy_name: The name of the taxonomy this annotation conforms to. See :meth:`Dataset.add_taxonomy`. confidence: 0-1 indicating the confidence of the prediction. @@ -597,7 +692,7 @@ class CategoryPrediction(CategoryAnnotation): def __init__( self, label: str, - reference_id: str, + reference_id: Optional[str] = None, taxonomy_name: Optional[str] = None, confidence: Optional[float] = None, metadata: Optional[Dict] = None, @@ -605,6 +700,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -614,8 +710,10 @@ def __init__( metadata=metadata, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -623,6 +721,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -631,8 +732,9 @@ def from_json(cls, payload: dict): return cls( label=payload.get(LABEL_KEY, 0), taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), @@ -659,7 +761,8 @@ class SceneCategoryPrediction(SceneCategoryAnnotation): Parameters: label: The label for this annotation (e.g. action, subject, scenario). - reference_id: The reference ID of the scene you wish to apply this annotation to. + reference_id: Optional user-defined ID of the item to which to apply this + prediction. Provide this or ``dataset_item_id``. taxonomy_name: The name of the taxonomy this annotation conforms to. See :meth:`Dataset.add_taxonomy`. confidence: 0-1 indicating the confidence of the prediction. @@ -672,12 +775,13 @@ class SceneCategoryPrediction(SceneCategoryAnnotation): def __init__( self, label: str, - reference_id: str, + reference_id: Optional[str] = None, taxonomy_name: Optional[str] = None, confidence: Optional[float] = None, metadata: Optional[Dict] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -686,12 +790,17 @@ def __init__( dataset_item_id=dataset_item_id, metadata=metadata, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() if self.confidence is not None: payload[CONFIDENCE_KEY] = self.confidence + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -700,8 +809,9 @@ def from_json(cls, payload: dict): return cls( label=payload.get(LABEL_KEY, 0), taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), metadata=payload.get(METADATA_KEY, {}), ) diff --git a/pyproject.toml b/pyproject.toml index 723d7d7e..23b970ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.20.2" +version = "0.21.0" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] diff --git a/tests/test_dataset_less_model_runs.py b/tests/test_dataset_less_model_runs.py new file mode 100644 index 00000000..d7df3c98 --- /dev/null +++ b/tests/test_dataset_less_model_runs.py @@ -0,0 +1,280 @@ +"""Unit tests for dataset-less model runs (no live API). + +A model run can now be created without naming a dataset up front. Its dataset +set starts empty and grows as predictions arrive, because each prediction +carries its own target item (``dataset_item_id``, or ``dataset_id`` + +``reference_id``). Uploads go to ``modelRun/{model_run_id}/uploadPredictions``, +where the server resolves each item, groups by dataset, and widens the run. + +These tests pin the routing, the create call, the ``add_predictions`` upload, +and the per-prediction target ids emitted in ``to_payload`` — all with mocks, +so they run offline. +""" + +from unittest.mock import MagicMock + +import pytest + +from nucleus import NucleusClient +from nucleus.annotation_uploader import PredictionUploader +from nucleus.errors import DuplicateIDError +from nucleus.model import Model +from nucleus.model_run import ModelRun +from nucleus.prediction import BoxPrediction, CategoryPrediction + + +def _client(): + return NucleusClient(api_key="test") + + +def _model(client=None): + return Model( + model_id="prj_1", + name="My Model", + reference_id="My-CNN", + metadata={}, + client=client or _client(), + ) + + +def _predictions(): + return [ + BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + reference_id="item_1", + confidence=0.9, + ) + ] + + +# --------------------------------------------------------------------------- # +# PredictionUploader routing +# --------------------------------------------------------------------------- # +def test_model_run_id_alone_routes_to_the_dataset_less_endpoint(): + uploader = PredictionUploader(client=_client(), model_run_id="run_1") + assert ( + uploader._route == "modelRun/run_1/uploadPredictions" + ) # noqa: SLF001 + + +def test_dataset_and_model_run_ids_still_route_to_the_widening_endpoint(): + uploader = PredictionUploader( + client=_client(), dataset_id="ds_1", model_run_id="run_1" + ) + assert ( + uploader._route + == "dataset/ds_1/modelRun/run_1/uploadPredictions" # noqa: SLF001 + ) + + +def test_dataset_and_model_ids_still_route_to_the_model_endpoint(): + uploader = PredictionUploader( + client=_client(), dataset_id="ds_1", model_id="prj_1" + ) + assert ( + uploader._route + == "dataset/ds_1/model/prj_1/uploadPredictions" # noqa: SLF001 + ) + + +def test_model_id_and_model_run_id_together_are_rejected(): + with pytest.raises(ValueError, match="not both"): + PredictionUploader( + client=_client(), + dataset_id="ds_1", + model_id="prj_1", + model_run_id="run_1", + ) + + +def test_neither_model_nor_model_run_is_rejected(): + with pytest.raises(ValueError, match="required"): + PredictionUploader(client=_client(), dataset_id="ds_1") + + +def test_model_id_without_dataset_id_is_rejected(): + with pytest.raises(ValueError, match="dataset_id is required"): + PredictionUploader(client=_client(), model_id="prj_1") + + +# --------------------------------------------------------------------------- # +# Model.create_run (dataset-less) +# --------------------------------------------------------------------------- # +def test_create_run_posts_to_the_create_route(): + client = _client() + client.make_request = MagicMock(return_value={"model_run_id": "run_1"}) + model = _model(client) + + run = model.create_run( + name="my run", metadata={"k": "v"}, reference_id="ref_1" + ) + + payload = client.make_request.call_args[0][0] + route = client.make_request.call_args[1]["route"] + assert route == "model/prj_1/modelRun/create" + assert payload == { + "name": "my run", + "reference_id": "ref_1", + "metadata": {"k": "v"}, + } + assert isinstance(run, ModelRun) + assert run.model_run_id == "run_1" + assert run.dataset_id is None + + +def test_create_run_defaults_metadata_to_empty_dict(): + client = _client() + client.make_request = MagicMock(return_value={"model_run_id": "run_1"}) + model = _model(client) + + model.create_run(name="my run") + + payload = client.make_request.call_args[0][0] + assert payload == { + "name": "my run", + "reference_id": None, + "metadata": {}, + } + + +def test_create_run_with_predictions_and_no_dataset_uploads_them(): + client = _client() + client.make_request = MagicMock(return_value={"model_run_id": "run_1"}) + model = _model(client) + predictions = _predictions() + + with pytest.MonkeyPatch.context() as mp: + captured = {} + mp.setattr( + ModelRun, + "add_predictions", + lambda self, preds, **kw: captured.update( + run_id=self.model_run_id, preds=preds + ), + ) + run = model.create_run(name="my run", predictions=predictions) + + # The run is created dataset-less, then the predictions are attached to it. + assert client.make_request.call_args[1]["route"] == ( + "model/prj_1/modelRun/create" + ) + assert run.model_run_id == "run_1" + assert run.dataset_id is None + assert captured == {"run_id": "run_1", "preds": predictions} + + +# --------------------------------------------------------------------------- # +# ModelRun.add_predictions +# --------------------------------------------------------------------------- # +def test_add_predictions_uses_the_dataset_less_route_and_forwards_update(): + run = ModelRun(model_run_id="run_1", client=_client()) + captured = {} + + with pytest.MonkeyPatch.context() as mp: + routes = [] + original_init = PredictionUploader.__init__ + + def _spy_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + routes.append(self._route) # noqa: SLF001 + + mp.setattr(PredictionUploader, "__init__", _spy_init) + mp.setattr( + PredictionUploader, + "upload", + lambda self, **kw: captured.update(kw) or {}, + ) + run.add_predictions(_predictions(), update=True) + + assert routes == ["modelRun/run_1/uploadPredictions"] + assert captured["update"] is True + + +def test_add_predictions_runs_the_duplicate_id_check(): + run = ModelRun(model_run_id="run_1", client=_client()) + duplicate = _predictions() * 2 + for pred in duplicate: + pred.annotation_id = "ann_1" + + with pytest.raises(DuplicateIDError): + run.add_predictions(duplicate) + + +def test_add_predictions_async_is_not_supported(): + run = ModelRun(model_run_id="run_1", client=_client()) + with pytest.raises(NotImplementedError): + run.add_predictions(_predictions(), asynchronous=True) + + +# --------------------------------------------------------------------------- # +# Per-prediction target ids in to_payload +# --------------------------------------------------------------------------- # +def test_box_prediction_emits_item_id_from_dataset_item_id(): + pred = BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + reference_id="item_1", + dataset_item_id="di_1", + ) + payload = pred.to_payload() + assert payload["item_id"] == "di_1" + assert "dataset_id" not in payload + + +def test_box_prediction_emits_dataset_id_and_reference_id(): + pred = BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + reference_id="r1", + dataset_id="ds_1", + ) + payload = pred.to_payload() + assert payload["dataset_id"] == "ds_1" + assert payload["reference_id"] == "r1" + assert "item_id" not in payload + + +def test_box_prediction_omits_target_ids_when_unset(): + payload = _predictions()[0].to_payload() + assert "item_id" not in payload + assert "dataset_id" not in payload + + +# --------------------------------------------------------------------------- # +# Predictions target an item by dataset_item_id alone (no reference_id) +# --------------------------------------------------------------------------- # +def test_box_prediction_builds_from_dataset_item_id_alone(): + pred = BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + dataset_item_id="di_1", + confidence=0.9, + ) + payload = pred.to_payload() + assert payload["item_id"] == "di_1" + assert payload.get("reference_id") is None + + +def test_prediction_with_no_target_is_rejected(): + with pytest.raises(ValueError, match="reference_id or dataset_item_id"): + BoxPrediction(label="car", x=0, y=0, width=10, height=10) + + +def test_category_prediction_builds_from_dataset_item_id_alone(): + pred = CategoryPrediction(label="car", dataset_item_id="di_1") + payload = pred.to_payload() + assert payload["item_id"] == "di_1" + assert payload.get("reference_id") is None diff --git a/tests/test_multi_dataset_model_runs.py b/tests/test_multi_dataset_model_runs.py index 80a27b25..2bf902ee 100644 --- a/tests/test_multi_dataset_model_runs.py +++ b/tests/test_multi_dataset_model_runs.py @@ -90,10 +90,13 @@ def test_neither_model_nor_model_run_is_rejected(): PredictionUploader(client=_client(), dataset_id="ds_1") -def test_dataset_id_is_required_for_id_based_routing(): - """model_run_id alone no longer selects the deprecated route.""" - with pytest.raises(ValueError, match="dataset_id is required"): - PredictionUploader(client=_client(), model_run_id="run_1") +def test_model_run_id_alone_routes_to_the_dataset_less_endpoint(): + """A bare model_run_id (no dataset_id, no explicit route) now targets the + dataset-less endpoint, where predictions carry their own item ids.""" + uploader = PredictionUploader(client=_client(), model_run_id="run_1") + assert ( + uploader._route == "modelRun/run_1/uploadPredictions" # noqa: SLF001 + ) # --------------------------------------------------------------------------- #