From d2c784a4e4cc5352c5b0b218cf8578f9266ccc85 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Wed, 19 Aug 2026 19:01:03 -0500 Subject: [PATCH 1/7] feat(model-runs): dataset-less create_model_run + add_predictions Add a create-run-then-add-predictions flow that does not require threading a dataset. A run can be created with no founding dataset; predictions then carry their own target (dataset_item_id, or dataset_id + reference_id) and the server groups by dataset and widens the run's dataset set. - Model.create_run_without_dataset(name, metadata=None, reference_id=None) -> POST model/{id}/modelRun/create, returns a ModelRun with no datasets. - ModelRun.add_predictions(...) -> POST modelRun/{id}/uploadPredictions. - Predictions emit item_id (from dataset_item_id) and/or dataset_id when set. - PredictionUploader: bare model_run_id routes to the dataset-less endpoint. Requires the matching scaleapi backend routes; async upload is not yet supported on this path. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 9 + nucleus/annotation_uploader.py | 14 +- nucleus/constants.py | 1 + nucleus/model.py | 39 +++++ nucleus/model_run.py | 79 ++++++++- nucleus/prediction.py | 91 ++++++++++ pyproject.toml | 2 +- tests/test_dataset_less_model_runs.py | 224 +++++++++++++++++++++++++ tests/test_multi_dataset_model_runs.py | 11 +- 9 files changed, 460 insertions(+), 10 deletions(-) create mode 100644 tests/test_dataset_less_model_runs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 79bf2932..5f9bbad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.21.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.0) - 2026-08-19 + +### Added +- **Dataset-less model runs.** `Model.create_run_without_dataset(name, metadata=None, reference_id=None)` creates a model run that starts with an empty dataset set and is not bound to any dataset up front. The run is not readable until predictions are added, and its dataset set grows as predictions arrive — letting a single run span multiple datasets without naming them ahead of time. +- **`ModelRun.add_predictions(predictions, ...)`** uploads predictions to a dataset-less run via `POST /nucleus/modelRun/:modelRunId/uploadPredictions`. The server resolves each prediction's target item, groups by dataset, and widens the run's dataset set. Supports `update` / `batch_size` / file-batching arguments; `asynchronous=True` raises `NotImplementedError` (use `Dataset.upload_predictions_for_model_run` for async per-dataset uploads). +- **Per-prediction upload targets.** Every prediction type (`box`, `line`, `polygon`, `keypoints`, `cuboid`, `category`, `scene_category`, `segmentation`) now accepts a `dataset_id` constructor kwarg alongside the existing `dataset_item_id`, and emits them in `to_payload` (as `item_id` and `dataset_id`) when set. A prediction targets its item by `dataset_item_id` (preferred) or `dataset_id` + `reference_id`, which is what the dataset-less upload route uses to resolve items. Both are optional and unset by default, so per-dataset uploads are unchanged. + +> **Server dependency:** requires the `POST /nucleus/model/:modelId/modelRun/create` and `POST /nucleus/modelRun/:modelRunId/uploadPredictions` routes in scaleapi. Unit tests pass regardless; live calls 404 until that deploys. + ## [0.20.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.20.2) - 2026-08-18 ### Added diff --git a/nucleus/annotation_uploader.py b/nucleus/annotation_uploader.py index c5768988..35a6fe6b 100644 --- a/nucleus/annotation_uploader.py +++ b/nucleus/annotation_uploader.py @@ -273,13 +273,21 @@ def __init__( if route is not None: self._route = route return - if dataset_id is None: - raise ValueError("dataset_id is required to upload predictions.") if model_run_id is not None and model_id is not None: raise ValueError("Pass either model_id or model_run_id, not both.") if model_run_id is not None: - self._route = f"dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions" + if dataset_id is not None: + self._route = f"dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions" + else: + # Dataset-less model run: predictions carry their own item_id + # (or dataset_id + reference_id); the server groups by dataset + # and widens the run. + self._route = f"modelRun/{model_run_id}/uploadPredictions" elif model_id is not None: + if dataset_id is None: + raise ValueError( + "dataset_id is required to upload predictions." + ) self._route = ( f"dataset/{dataset_id}/model/{model_id}/uploadPredictions" ) diff --git a/nucleus/constants.py b/nucleus/constants.py index 6ded5193..f2e0f18d 100644 --- a/nucleus/constants.py +++ b/nucleus/constants.py @@ -41,6 +41,7 @@ DATASET_IS_SCENE_KEY = "is_scene" DATASET_ITEM_ID_KEY = "dataset_item_id" DATASET_ITEM_IDS_KEY = "dataset_item_ids" +ITEM_ID_KEY = "item_id" DATASET_ITEMS_KEY = "dataset_items" DATASET_LENGTH_KEY = "length" DATASET_MODEL_RUNS_KEY = "model_run_ids" diff --git a/nucleus/model.py b/nucleus/model.py index 36659a73..acdabb70 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -187,6 +187,45 @@ def create_run( return model_run + def create_run_without_dataset( + self, + name: str, + metadata: Optional[Dict] = None, + reference_id: Optional[str] = None, + ) -> "ModelRun": + """Creates a model run that is not bound to any dataset up front. + + The run starts with an empty dataset set and is not readable until + predictions are added to it via :meth:`ModelRun.add_predictions`. Each + prediction names its own target item (``dataset_item_id``, or + ``dataset_id`` + ``reference_id``); the server groups them by dataset + and widens the run's dataset set as predictions arrive. This is what + lets a single run span multiple datasets. + + Args: + name: Human-readable name for the model run. + metadata: Optional arbitrary metadata blob for the run. + reference_id: Optional user-defined reference id for the run. + + Returns: + The created :class:`ModelRun` (with ``dataset_id`` unset). + """ + payload: dict = { + NAME_KEY: name, + REFERENCE_ID_KEY: reference_id, + METADATA_KEY: metadata or {}, + } + response = self._client.make_request( + payload, + route=f"model/{self.id}/modelRun/create", + requests_command=requests.post, + ) + return ModelRun( + model_run_id=response["model_run_id"], + dataset_id=None, + client=self._client, + ) + def evaluate(self, scenario_test_names: List[str]) -> AsyncJob: """Evaluates this on the specified Unit Tests. :: diff --git a/nucleus/model_run.py b/nucleus/model_run.py index d78b7336..54c91452 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -45,7 +45,12 @@ class ModelRun: This class is deprecated and will be removed from the python client. """ - def __init__(self, model_run_id: str, dataset_id: str, client): + def __init__( + self, + model_run_id: str, + dataset_id: Optional[str] = None, + client=None, + ): self.model_run_id = model_run_id self._client = client self.dataset_id = dataset_id @@ -173,7 +178,7 @@ def predict( "predictions_ignored": int, } """ - + uploader = PredictionUploader( client=self._client, dataset_id=self.dataset_id, @@ -201,6 +206,76 @@ def predict( local_files_per_upload_request=local_files_per_upload_request, ) + def add_predictions( + self, + predictions: List[ + Union[ + BoxPrediction, + PolygonPrediction, + CuboidPrediction, + SegmentationPrediction, + ] + ], + update: bool = False, + asynchronous: bool = False, + batch_size: int = 5000, + remote_files_per_upload_request: int = 20, + local_files_per_upload_request: int = 10, + ) -> dict: + """Uploads predictions to a dataset-less model run. + + Unlike :meth:`predict`, this run is not bound to a single dataset. Each + prediction names its own target item, and the server groups the + predictions by dataset and widens the run's dataset set accordingly. + Every prediction must carry either ``dataset_item_id`` (preferred) or + ``dataset_id`` + ``reference_id`` so the server can resolve its item. + + Args: + predictions: Predictions to upload. Each must carry + ``dataset_item_id`` or ``dataset_id`` + ``reference_id``. + update: If True, existing predictions for the same + (reference_id, annotation_id) will be overwritten. If False, + existing predictions will be skipped. + asynchronous: Not supported for dataset-less uploads yet — passing + True raises :class:`NotImplementedError`. Use + ``dataset.upload_predictions_for_model_run(...)`` for async + per-dataset uploads. + batch_size: Number of predictions processed in each concurrent batch. + remote_files_per_upload_request: Number of remote segmentation files + to upload in each request. + local_files_per_upload_request: Number of local segmentation files + to upload in each request. The maximum is 10. + + Returns:: + + { + "model_run_id": str, + "predictions_processed": int, + "predictions_ignored": int, + } + """ + if asynchronous: + raise NotImplementedError( + "Asynchronous upload is not supported for dataset-less model " + "runs yet. Use " + "dataset.upload_predictions_for_model_run(model_run_id, " + "predictions, asynchronous=True) for the per-dataset route." + ) + + uploader = PredictionUploader( + client=self._client, + model_run_id=self.model_run_id, + ) + uploader.check_for_duplicate_ids(predictions) + + return uploader.upload( + annotations=predictions, + update=update, + batch_size=batch_size, + remote_files_per_upload_request=remote_files_per_upload_request, + local_files_per_upload_request=local_files_per_upload_request, + ) + def iloc(self, i: int): """Returns Model Run Info For Dataset Item by its number. diff --git a/nucleus/prediction.py b/nucleus/prediction.py index ae52c86f..2bb9ff4a 100644 --- a/nucleus/prediction.py +++ b/nucleus/prediction.py @@ -29,11 +29,13 @@ CLASS_PDF_KEY, CONFIDENCE_KEY, CUBOID_TYPE, + DATASET_ID_KEY, DATASET_ITEM_ID_KEY, DIMENSIONS_KEY, EMBEDDING_VECTOR_KEY, GEOMETRY_KEY, HEIGHT_KEY, + ITEM_ID_KEY, KEYPOINTS_KEY, KEYPOINTS_NAMES_KEY, KEYPOINTS_SKELETON_KEY, @@ -56,6 +58,25 @@ ) +def _add_prediction_target_ids( + payload: dict, + dataset_item_id: Optional[str], + dataset_id: Optional[str], +) -> dict: + """Emit the per-prediction upload target ids used by dataset-less model runs. + + The dataset-less ``modelRun/{run}/uploadPredictions`` endpoint resolves each + prediction's dataset item from either ``item_id`` (the dataset_item_id) or + ``dataset_id`` + ``reference_id``. Both keys are output-only and are only + included when set, keeping regular per-dataset uploads unchanged. + """ + if dataset_item_id is not None: + payload[ITEM_ID_KEY] = dataset_item_id + if dataset_id is not None: + payload[DATASET_ID_KEY] = dataset_id + return payload + + def from_json(payload: dict): """Instantiates prediction object from schematized JSON dict payload.""" type_key_to_type: Dict[str, Type[Prediction]] = { @@ -122,6 +143,33 @@ class SegmentationPrediction(SegmentationAnnotation): to an external database, and its value will be returned for any export. """ + def __init__( + self, + mask_url: str, + annotations: List[Segment], + reference_id: str, + annotation_id: Optional[str] = None, + dataset_item_id: Optional[str] = None, + *, + dataset_id: Optional[str] = None, + ): + super().__init__( + mask_url=mask_url, + annotations=annotations, + reference_id=reference_id, + annotation_id=annotation_id, + dataset_item_id=dataset_item_id, + ) + self.dataset_id = dataset_id + + def to_payload(self) -> dict: + payload = super().to_payload() + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) + + return payload + @classmethod def from_json(cls, payload: dict): return cls( @@ -132,6 +180,7 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), annotation_id=payload.get(ANNOTATION_ID_KEY, None), # metadata=payload.get(METADATA_KEY, None), # TODO(sc: 422637) ) @@ -194,6 +243,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -210,6 +260,7 @@ def __init__( ) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -217,6 +268,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -231,6 +285,7 @@ def from_json(cls, payload: dict): height=geometry.get(HEIGHT_KEY, 0), reference_id=payload[REFERENCE_ID_KEY], dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -278,6 +333,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -290,6 +346,7 @@ def __init__( ) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -297,6 +354,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -310,6 +370,7 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -360,6 +421,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -373,6 +435,7 @@ def __init__( ) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -380,6 +443,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -393,6 +459,7 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -445,6 +512,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -459,6 +527,7 @@ def __init__( ) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -466,6 +535,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -481,6 +553,7 @@ def from_json(cls, payload: dict): skeleton=geometry[KEYPOINTS_SKELETON_KEY], reference_id=payload[REFERENCE_ID_KEY], dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -530,6 +603,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -544,6 +618,7 @@ def __init__( ) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -551,6 +626,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -564,6 +642,7 @@ def from_json(cls, payload: dict): yaw=geometry.get(YAW_KEY, 0), reference_id=payload[REFERENCE_ID_KEY], dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), @@ -605,6 +684,7 @@ def __init__( track_reference_id: Optional[str] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -616,6 +696,7 @@ def __init__( ) self.confidence = confidence self.class_pdf = class_pdf + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() @@ -623,6 +704,9 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -633,6 +717,7 @@ def from_json(cls, payload: dict): taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), reference_id=payload[REFERENCE_ID_KEY], dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), @@ -678,6 +763,7 @@ def __init__( metadata: Optional[Dict] = None, *, dataset_item_id: Optional[str] = None, + dataset_id: Optional[str] = None, ): super().__init__( label=label, @@ -687,11 +773,15 @@ def __init__( metadata=metadata, ) self.confidence = confidence + self.dataset_id = dataset_id def to_payload(self) -> dict: payload = super().to_payload() if self.confidence is not None: payload[CONFIDENCE_KEY] = self.confidence + _add_prediction_target_ids( + payload, self.dataset_item_id, self.dataset_id + ) return payload @@ -702,6 +792,7 @@ def from_json(cls, payload: dict): taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), reference_id=payload[REFERENCE_ID_KEY], dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), + dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), metadata=payload.get(METADATA_KEY, {}), ) diff --git a/pyproject.toml b/pyproject.toml index 723d7d7e..23b970ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.20.2" +version = "0.21.0" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] diff --git a/tests/test_dataset_less_model_runs.py b/tests/test_dataset_less_model_runs.py new file mode 100644 index 00000000..d98c4585 --- /dev/null +++ b/tests/test_dataset_less_model_runs.py @@ -0,0 +1,224 @@ +"""Unit tests for dataset-less model runs (no live API). + +A model run can now be created without naming a dataset up front. Its dataset +set starts empty and grows as predictions arrive, because each prediction +carries its own target item (``dataset_item_id``, or ``dataset_id`` + +``reference_id``). Uploads go to ``modelRun/{model_run_id}/uploadPredictions``, +where the server resolves each item, groups by dataset, and widens the run. + +These tests pin the routing, the create call, the ``add_predictions`` upload, +and the per-prediction target ids emitted in ``to_payload`` — all with mocks, +so they run offline. +""" + +from unittest.mock import MagicMock + +import pytest + +from nucleus import NucleusClient +from nucleus.annotation_uploader import PredictionUploader +from nucleus.errors import DuplicateIDError +from nucleus.model import Model +from nucleus.model_run import ModelRun +from nucleus.prediction import BoxPrediction + + +def _client(): + return NucleusClient(api_key="test") + + +def _model(client=None): + return Model( + model_id="prj_1", + name="My Model", + reference_id="My-CNN", + metadata={}, + client=client or _client(), + ) + + +def _predictions(): + return [ + BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + reference_id="item_1", + confidence=0.9, + ) + ] + + +# --------------------------------------------------------------------------- # +# PredictionUploader routing +# --------------------------------------------------------------------------- # +def test_model_run_id_alone_routes_to_the_dataset_less_endpoint(): + uploader = PredictionUploader(client=_client(), model_run_id="run_1") + assert ( + uploader._route == "modelRun/run_1/uploadPredictions" + ) # noqa: SLF001 + + +def test_dataset_and_model_run_ids_still_route_to_the_widening_endpoint(): + uploader = PredictionUploader( + client=_client(), dataset_id="ds_1", model_run_id="run_1" + ) + assert ( + uploader._route + == "dataset/ds_1/modelRun/run_1/uploadPredictions" # noqa: SLF001 + ) + + +def test_dataset_and_model_ids_still_route_to_the_model_endpoint(): + uploader = PredictionUploader( + client=_client(), dataset_id="ds_1", model_id="prj_1" + ) + assert ( + uploader._route + == "dataset/ds_1/model/prj_1/uploadPredictions" # noqa: SLF001 + ) + + +def test_model_id_and_model_run_id_together_are_rejected(): + with pytest.raises(ValueError, match="not both"): + PredictionUploader( + client=_client(), + dataset_id="ds_1", + model_id="prj_1", + model_run_id="run_1", + ) + + +def test_neither_model_nor_model_run_is_rejected(): + with pytest.raises(ValueError, match="required"): + PredictionUploader(client=_client(), dataset_id="ds_1") + + +def test_model_id_without_dataset_id_is_rejected(): + with pytest.raises(ValueError, match="dataset_id is required"): + PredictionUploader(client=_client(), model_id="prj_1") + + +# --------------------------------------------------------------------------- # +# Model.create_run_without_dataset +# --------------------------------------------------------------------------- # +def test_create_run_without_dataset_posts_to_the_create_route(): + client = _client() + client.make_request = MagicMock(return_value={"model_run_id": "run_1"}) + model = _model(client) + + run = model.create_run_without_dataset( + name="my run", metadata={"k": "v"}, reference_id="ref_1" + ) + + payload = client.make_request.call_args[0][0] + route = client.make_request.call_args[1]["route"] + assert route == "model/prj_1/modelRun/create" + assert payload == { + "name": "my run", + "reference_id": "ref_1", + "metadata": {"k": "v"}, + } + assert isinstance(run, ModelRun) + assert run.model_run_id == "run_1" + assert run.dataset_id is None + + +def test_create_run_without_dataset_defaults_metadata_to_empty_dict(): + client = _client() + client.make_request = MagicMock(return_value={"model_run_id": "run_1"}) + model = _model(client) + + model.create_run_without_dataset(name="my run") + + payload = client.make_request.call_args[0][0] + assert payload == { + "name": "my run", + "reference_id": None, + "metadata": {}, + } + + +# --------------------------------------------------------------------------- # +# ModelRun.add_predictions +# --------------------------------------------------------------------------- # +def test_add_predictions_uses_the_dataset_less_route_and_forwards_update(): + run = ModelRun(model_run_id="run_1", client=_client()) + captured = {} + + with pytest.MonkeyPatch.context() as mp: + routes = [] + original_init = PredictionUploader.__init__ + + def _spy_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + routes.append(self._route) # noqa: SLF001 + + mp.setattr(PredictionUploader, "__init__", _spy_init) + mp.setattr( + PredictionUploader, + "upload", + lambda self, **kw: captured.update(kw) or {}, + ) + run.add_predictions(_predictions(), update=True) + + assert routes == ["modelRun/run_1/uploadPredictions"] + assert captured["update"] is True + + +def test_add_predictions_runs_the_duplicate_id_check(): + run = ModelRun(model_run_id="run_1", client=_client()) + duplicate = _predictions() * 2 + for pred in duplicate: + pred.annotation_id = "ann_1" + + with pytest.raises(DuplicateIDError): + run.add_predictions(duplicate) + + +def test_add_predictions_async_is_not_supported(): + run = ModelRun(model_run_id="run_1", client=_client()) + with pytest.raises(NotImplementedError): + run.add_predictions(_predictions(), asynchronous=True) + + +# --------------------------------------------------------------------------- # +# Per-prediction target ids in to_payload +# --------------------------------------------------------------------------- # +def test_box_prediction_emits_item_id_from_dataset_item_id(): + pred = BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + reference_id="item_1", + dataset_item_id="di_1", + ) + payload = pred.to_payload() + assert payload["item_id"] == "di_1" + assert "dataset_id" not in payload + + +def test_box_prediction_emits_dataset_id_and_reference_id(): + pred = BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + reference_id="r1", + dataset_id="ds_1", + ) + payload = pred.to_payload() + assert payload["dataset_id"] == "ds_1" + assert payload["reference_id"] == "r1" + assert "item_id" not in payload + + +def test_box_prediction_omits_target_ids_when_unset(): + payload = _predictions()[0].to_payload() + assert "item_id" not in payload + assert "dataset_id" not in payload diff --git a/tests/test_multi_dataset_model_runs.py b/tests/test_multi_dataset_model_runs.py index 80a27b25..2bf902ee 100644 --- a/tests/test_multi_dataset_model_runs.py +++ b/tests/test_multi_dataset_model_runs.py @@ -90,10 +90,13 @@ def test_neither_model_nor_model_run_is_rejected(): PredictionUploader(client=_client(), dataset_id="ds_1") -def test_dataset_id_is_required_for_id_based_routing(): - """model_run_id alone no longer selects the deprecated route.""" - with pytest.raises(ValueError, match="dataset_id is required"): - PredictionUploader(client=_client(), model_run_id="run_1") +def test_model_run_id_alone_routes_to_the_dataset_less_endpoint(): + """A bare model_run_id (no dataset_id, no explicit route) now targets the + dataset-less endpoint, where predictions carry their own item ids.""" + uploader = PredictionUploader(client=_client(), model_run_id="run_1") + assert ( + uploader._route == "modelRun/run_1/uploadPredictions" # noqa: SLF001 + ) # --------------------------------------------------------------------------- # From 0b72ad5cc842b055bf0cc819dd43489b53d2b862 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Thu, 20 Aug 2026 01:06:01 +0000 Subject: [PATCH 2/7] fix(model-runs): guard async predict() against a None dataset_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Making dataset_id Optional[str] for dataset-less runs broke mypy on the deprecated predict() async path, which passes self.dataset_id straight to serialize_and_write_to_presigned_url (expects str). Async predict is inherently per-dataset, so raise a clear ValueError when dataset_id is None (dataset-less runs must use add_predictions) — this both fixes the type error and prevents a confusing runtime failure. Fixes the build_test MyPy failure. Co-Authored-By: Claude Opus 4.8 --- nucleus/model_run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nucleus/model_run.py b/nucleus/model_run.py index 54c91452..ac0b0698 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -189,6 +189,11 @@ def predict( if asynchronous: check_all_mask_paths_remote(annotations) + if self.dataset_id is None: + raise ValueError( + "Asynchronous predict() requires a dataset-bound model run. " + "Dataset-less runs must use add_predictions(...)." + ) request_id = serialize_and_write_to_presigned_url( annotations, self.dataset_id, self._client ) From 271dade9f337b64730feb5feeb12c0bbb659c47b Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Thu, 20 Aug 2026 10:48:17 -0500 Subject: [PATCH 3/7] refactor(model-runs): rename create_run_without_dataset -> create_run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge the dataset-less creation path into Model.create_run itself: create_run(name="my-run") now creates a run with no dataset, while the old create_run(name, dataset, predictions, ...) path is preserved for backwards compatibility (dataset/predictions are now optional). Drops the leaky create_run_without_dataset name and de-emphasizes dataset internals in the user-facing docstrings — callers identify a prediction's item by item_id/reference_id and never think about datasets. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 11 ++- nucleus/model.py | 101 +++++++++++++------------- nucleus/model_run.py | 21 +++--- tests/test_dataset_less_model_runs.py | 10 +-- 4 files changed, 72 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f9bbad5..7d3825d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.21.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.0) - 2026-08-19 ### Added -- **Dataset-less model runs.** `Model.create_run_without_dataset(name, metadata=None, reference_id=None)` creates a model run that starts with an empty dataset set and is not bound to any dataset up front. The run is not readable until predictions are added, and its dataset set grows as predictions arrive — letting a single run span multiple datasets without naming them ahead of time. -- **`ModelRun.add_predictions(predictions, ...)`** uploads predictions to a dataset-less run via `POST /nucleus/modelRun/:modelRunId/uploadPredictions`. The server resolves each prediction's target item, groups by dataset, and widens the run's dataset set. Supports `update` / `batch_size` / file-batching arguments; `asynchronous=True` raises `NotImplementedError` (use `Dataset.upload_predictions_for_model_run` for async per-dataset uploads). -- **Per-prediction upload targets.** Every prediction type (`box`, `line`, `polygon`, `keypoints`, `cuboid`, `category`, `scene_category`, `segmentation`) now accepts a `dataset_id` constructor kwarg alongside the existing `dataset_item_id`, and emits them in `to_payload` (as `item_id` and `dataset_id`) when set. A prediction targets its item by `dataset_item_id` (preferred) or `dataset_id` + `reference_id`, which is what the dataset-less upload route uses to resolve items. Both are optional and unset by default, so per-dataset uploads are unchanged. +- **`Model.create_run(name)` + `ModelRun.add_predictions(predictions, ...)`.** Create a model run with just a name, then attach predictions — no dataset needed up front: + ```python + run = model.create_run(name="my-run") + run.add_predictions(predictions) + ``` + Each prediction identifies the item it belongs to (by `item_id`, preferred, or `reference_id`), so predictions can come from anywhere and a single run can cover items across multiple datasets. `add_predictions` posts to `POST /nucleus/modelRun/:modelRunId/uploadPredictions` and supports `update` / `batch_size` / file-batching arguments; `asynchronous=True` raises `NotImplementedError` for now. + - `create_run` still accepts the old `dataset=` / `predictions=` arguments for backwards compatibility (the deprecated dataset-bound path); omit them to use the flow above. +- **Per-prediction targets.** Every prediction type (`box`, `line`, `polygon`, `keypoints`, `cuboid`, `category`, `scene_category`, `segmentation`) now accepts an optional `dataset_id` constructor kwarg alongside the existing `dataset_item_id`, emitted in `to_payload` (as `item_id` and `dataset_id`) when set. Both are optional and unset by default, so existing uploads are unchanged. > **Server dependency:** requires the `POST /nucleus/model/:modelId/modelRun/create` and `POST /nucleus/modelRun/:modelRunId/uploadPredictions` routes in scaleapi. Unit tests pass regardless; live calls 404 until that deploys. diff --git a/nucleus/model.py b/nucleus/model.py index acdabb70..049e4802 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -151,80 +151,79 @@ def from_json(cls, payload: dict, client): def create_run( self, name: str, - dataset: Dataset, - predictions: List[ - Union[ - BoxPrediction, - PolygonPrediction, - CuboidPrediction, - SegmentationPrediction, + dataset: Optional[Dataset] = None, + predictions: Optional[ + List[ + Union[ + BoxPrediction, + PolygonPrediction, + CuboidPrediction, + SegmentationPrediction, + ] ] - ], + ] = None, metadata: Optional[Dict] = None, asynchronous: bool = False, + reference_id: Optional[str] = None, ) -> ModelRun: - # This method, as well as model runs in general are now deprecated. - - # Instead models will automatically generate a model run when applied to - # a dataset using dataset.upload_predictions(model, predictions). Therefore - # there is no longer any need to create a model run, since you can upload - # predictions without needing to explicitly create a model run. - - # When uploading to a dataset twice using the same model, the same model - # run will be reused by Nucleus. - - payload: dict = { - NAME_KEY: name, - REFERENCE_ID_KEY: self.reference_id, - } - if metadata: - payload[METADATA_KEY] = metadata - model_run: ModelRun = self._client.create_model_run( - dataset.id, payload - ) + """Creates a model run for this model. - model_run.predict(predictions, asynchronous=asynchronous) + Call it with just a name to create a run, then attach predictions with + :meth:`ModelRun.add_predictions`:: - return model_run + run = model.create_run(name="my-run") + run.add_predictions(predictions) - def create_run_without_dataset( - self, - name: str, - metadata: Optional[Dict] = None, - reference_id: Optional[str] = None, - ) -> "ModelRun": - """Creates a model run that is not bound to any dataset up front. - - The run starts with an empty dataset set and is not readable until - predictions are added to it via :meth:`ModelRun.add_predictions`. Each - prediction names its own target item (``dataset_item_id``, or - ``dataset_id`` + ``reference_id``); the server groups them by dataset - and widens the run's dataset set as predictions arrive. This is what - lets a single run span multiple datasets. + Each prediction identifies the item it belongs to (by ``item_id``, or a + ``reference_id``), so you can add predictions from anywhere without + wiring anything up ahead of time. Args: name: Human-readable name for the model run. + predictions: Optional predictions to attach to the run immediately. metadata: Optional arbitrary metadata blob for the run. reference_id: Optional user-defined reference id for the run. + dataset: Deprecated. Passing a dataset uses the legacy path that + binds the run to that one dataset and uploads ``predictions`` + to it. Omit it to use the recommended flow above. + asynchronous: Only used by the deprecated dataset-bound path. Returns: - The created :class:`ModelRun` (with ``dataset_id`` unset). + The created :class:`ModelRun`. """ - payload: dict = { - NAME_KEY: name, - REFERENCE_ID_KEY: reference_id, - METADATA_KEY: metadata or {}, - } + # Legacy path: an explicit dataset binds the run to that dataset and + # uploads predictions to it. Kept for backwards compatibility; new code + # should omit `dataset` and use `run.add_predictions(...)`. + if dataset is not None: + payload: dict = { + NAME_KEY: name, + REFERENCE_ID_KEY: self.reference_id, + } + if metadata: + payload[METADATA_KEY] = metadata + model_run: ModelRun = self._client.create_model_run( + dataset.id, payload + ) + model_run.predict(predictions or [], asynchronous=asynchronous) + return model_run + response = self._client.make_request( - payload, + { + NAME_KEY: name, + REFERENCE_ID_KEY: reference_id, + METADATA_KEY: metadata or {}, + }, route=f"model/{self.id}/modelRun/create", requests_command=requests.post, ) - return ModelRun( + run = ModelRun( model_run_id=response["model_run_id"], dataset_id=None, client=self._client, ) + if predictions: + run.add_predictions(predictions) + return run def evaluate(self, scenario_test_names: List[str]) -> AsyncJob: """Evaluates this on the specified Unit Tests. :: diff --git a/nucleus/model_run.py b/nucleus/model_run.py index ac0b0698..4ccb0e79 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -227,24 +227,21 @@ def add_predictions( remote_files_per_upload_request: int = 20, local_files_per_upload_request: int = 10, ) -> dict: - """Uploads predictions to a dataset-less model run. + """Adds predictions to this model run. - Unlike :meth:`predict`, this run is not bound to a single dataset. Each - prediction names its own target item, and the server groups the - predictions by dataset and widens the run's dataset set accordingly. - Every prediction must carry either ``dataset_item_id`` (preferred) or - ``dataset_id`` + ``reference_id`` so the server can resolve its item. + Each prediction identifies the item it belongs to (by ``item_id``, + preferred, or ``reference_id``), so predictions can come from anywhere + and be added at any time — a single run can cover items that live in + different datasets. Args: - predictions: Predictions to upload. Each must carry - ``dataset_item_id`` or ``dataset_id`` + ``reference_id``. + predictions: Predictions to upload. Each must identify its item by + ``item_id`` (preferred) or ``reference_id``. update: If True, existing predictions for the same (reference_id, annotation_id) will be overwritten. If False, existing predictions will be skipped. - asynchronous: Not supported for dataset-less uploads yet — passing - True raises :class:`NotImplementedError`. Use - ``dataset.upload_predictions_for_model_run(...)`` for async - per-dataset uploads. + asynchronous: Not supported yet — passing True raises + :class:`NotImplementedError`. batch_size: Number of predictions processed in each concurrent batch. remote_files_per_upload_request: Number of remote segmentation files to upload in each request. diff --git a/tests/test_dataset_less_model_runs.py b/tests/test_dataset_less_model_runs.py index d98c4585..83071f15 100644 --- a/tests/test_dataset_less_model_runs.py +++ b/tests/test_dataset_less_model_runs.py @@ -102,14 +102,14 @@ def test_model_id_without_dataset_id_is_rejected(): # --------------------------------------------------------------------------- # -# Model.create_run_without_dataset +# Model.create_run (dataset-less) # --------------------------------------------------------------------------- # -def test_create_run_without_dataset_posts_to_the_create_route(): +def test_create_run_posts_to_the_create_route(): client = _client() client.make_request = MagicMock(return_value={"model_run_id": "run_1"}) model = _model(client) - run = model.create_run_without_dataset( + run = model.create_run( name="my run", metadata={"k": "v"}, reference_id="ref_1" ) @@ -126,12 +126,12 @@ def test_create_run_without_dataset_posts_to_the_create_route(): assert run.dataset_id is None -def test_create_run_without_dataset_defaults_metadata_to_empty_dict(): +def test_create_run_defaults_metadata_to_empty_dict(): client = _client() client.make_request = MagicMock(return_value={"model_run_id": "run_1"}) model = _model(client) - model.create_run_without_dataset(name="my run") + model.create_run(name="my run") payload = client.make_request.call_args[0][0] assert payload == { From 6316b35c31471c0e2ed87f9c8184f3995dec5cad Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Thu, 20 Aug 2026 10:49:57 -0500 Subject: [PATCH 4/7] test(model-runs): cover create_run(name, predictions=...) with no dataset create_run already creates a dataset-less run and forwards predictions to add_predictions when called without a dataset; pin that path with a test and document the one-call form. Co-Authored-By: Claude Opus 4.8 (1M context) --- nucleus/model.py | 4 ++++ tests/test_dataset_less_model_runs.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/nucleus/model.py b/nucleus/model.py index 049e4802..1dc241d0 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -174,6 +174,10 @@ def create_run( run = model.create_run(name="my-run") run.add_predictions(predictions) + or pass ``predictions`` directly to create and upload in one call:: + + run = model.create_run(name="my-run", predictions=predictions) + Each prediction identifies the item it belongs to (by ``item_id``, or a ``reference_id``), so you can add predictions from anywhere without wiring anything up ahead of time. diff --git a/tests/test_dataset_less_model_runs.py b/tests/test_dataset_less_model_runs.py index 83071f15..2b899ee5 100644 --- a/tests/test_dataset_less_model_runs.py +++ b/tests/test_dataset_less_model_runs.py @@ -141,6 +141,32 @@ def test_create_run_defaults_metadata_to_empty_dict(): } +def test_create_run_with_predictions_and_no_dataset_uploads_them(): + client = _client() + client.make_request = MagicMock(return_value={"model_run_id": "run_1"}) + model = _model(client) + predictions = _predictions() + + with pytest.MonkeyPatch.context() as mp: + captured = {} + mp.setattr( + ModelRun, + "add_predictions", + lambda self, preds, **kw: captured.update( + run_id=self.model_run_id, preds=preds + ), + ) + run = model.create_run(name="my run", predictions=predictions) + + # The run is created dataset-less, then the predictions are attached to it. + assert client.make_request.call_args[1]["route"] == ( + "model/prj_1/modelRun/create" + ) + assert run.model_run_id == "run_1" + assert run.dataset_id is None + assert captured == {"run_id": "run_1", "preds": predictions} + + # --------------------------------------------------------------------------- # # ModelRun.add_predictions # --------------------------------------------------------------------------- # From 5711ca738be77b5f013c6a82f4586447214d9ed2 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Thu, 20 Aug 2026 10:54:24 -0500 Subject: [PATCH 5/7] docs(model-runs): predictions target items by dataset_item_id, not reference_id Correct the docstrings/CHANGELOG: on the dataset-less route a bare reference_id can't resolve an item (it's only unique within a dataset). Each prediction must carry its dataset_item_id (the di_* id exposed on exported items). Drop the misleading "item_id or reference_id" framing. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++-- nucleus/model.py | 6 +++--- nucleus/model_run.py | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d3825d5..903c7146 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 run = model.create_run(name="my-run") run.add_predictions(predictions) ``` - Each prediction identifies the item it belongs to (by `item_id`, preferred, or `reference_id`), so predictions can come from anywhere and a single run can cover items across multiple datasets. `add_predictions` posts to `POST /nucleus/modelRun/:modelRunId/uploadPredictions` and supports `update` / `batch_size` / file-batching arguments; `asynchronous=True` raises `NotImplementedError` for now. + Each prediction identifies its target item by `dataset_item_id` (the `di_*` id returned on exported items), so predictions can come from anywhere and a single run can cover items across multiple datasets. `add_predictions` posts to `POST /nucleus/modelRun/:modelRunId/uploadPredictions` and supports `update` / `batch_size` / file-batching arguments; `asynchronous=True` raises `NotImplementedError` for now. - `create_run` still accepts the old `dataset=` / `predictions=` arguments for backwards compatibility (the deprecated dataset-bound path); omit them to use the flow above. -- **Per-prediction targets.** Every prediction type (`box`, `line`, `polygon`, `keypoints`, `cuboid`, `category`, `scene_category`, `segmentation`) now accepts an optional `dataset_id` constructor kwarg alongside the existing `dataset_item_id`, emitted in `to_payload` (as `item_id` and `dataset_id`) when set. Both are optional and unset by default, so existing uploads are unchanged. +- **Per-prediction target.** Every prediction type (`box`, `line`, `polygon`, `keypoints`, `cuboid`, `category`, `scene_category`, `segmentation`) emits its `dataset_item_id` in `to_payload` (as `item_id`) when set, which is how the dataset-less upload route resolves each item. It is optional and unset by default, so existing uploads are unchanged. > **Server dependency:** requires the `POST /nucleus/model/:modelId/modelRun/create` and `POST /nucleus/modelRun/:modelRunId/uploadPredictions` routes in scaleapi. Unit tests pass regardless; live calls 404 until that deploys. diff --git a/nucleus/model.py b/nucleus/model.py index 1dc241d0..18340571 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -178,9 +178,9 @@ def create_run( run = model.create_run(name="my-run", predictions=predictions) - Each prediction identifies the item it belongs to (by ``item_id``, or a - ``reference_id``), so you can add predictions from anywhere without - wiring anything up ahead of time. + Each prediction identifies its target item by ``dataset_item_id`` (the + ``di_*`` id returned on exported items), so predictions can come from + anywhere and a single run can cover items across different datasets. Args: name: Human-readable name for the model run. diff --git a/nucleus/model_run.py b/nucleus/model_run.py index 4ccb0e79..aa626ca8 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -229,14 +229,14 @@ def add_predictions( ) -> dict: """Adds predictions to this model run. - Each prediction identifies the item it belongs to (by ``item_id``, - preferred, or ``reference_id``), so predictions can come from anywhere - and be added at any time — a single run can cover items that live in - different datasets. + Each prediction identifies its target item by ``dataset_item_id`` (the + ``di_*`` id returned on exported items), so predictions can come from + anywhere and be added at any time — a single run can cover items that + live in different datasets. Args: - predictions: Predictions to upload. Each must identify its item by - ``item_id`` (preferred) or ``reference_id``. + predictions: Predictions to upload. Each must set + ``dataset_item_id`` to identify its item. update: If True, existing predictions for the same (reference_id, annotation_id) will be overwritten. If False, existing predictions will be skipped. From 97cced44db8f3616eef8de053afc2f94f529af76 Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Thu, 20 Aug 2026 11:00:10 -0500 Subject: [PATCH 6/7] feat(prediction): make reference_id optional when dataset_item_id is set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Predictions can now be built from a dataset_item_id alone — no reference_id required. A prediction must set at least one of reference_id / dataset_item_id. Annotations (ground truth) still require reference_id. from_json reads reference_id via .get() so item-only predictions round-trip. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 +- nucleus/prediction.py | 75 +++++++++++++++++---------- tests/test_dataset_less_model_runs.py | 32 +++++++++++- 3 files changed, 82 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 903c7146..64316b45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ``` Each prediction identifies its target item by `dataset_item_id` (the `di_*` id returned on exported items), so predictions can come from anywhere and a single run can cover items across multiple datasets. `add_predictions` posts to `POST /nucleus/modelRun/:modelRunId/uploadPredictions` and supports `update` / `batch_size` / file-batching arguments; `asynchronous=True` raises `NotImplementedError` for now. - `create_run` still accepts the old `dataset=` / `predictions=` arguments for backwards compatibility (the deprecated dataset-bound path); omit them to use the flow above. -- **Per-prediction target.** Every prediction type (`box`, `line`, `polygon`, `keypoints`, `cuboid`, `category`, `scene_category`, `segmentation`) emits its `dataset_item_id` in `to_payload` (as `item_id`) when set, which is how the dataset-less upload route resolves each item. It is optional and unset by default, so existing uploads are unchanged. +- **Per-prediction target.** Every prediction type (`box`, `line`, `polygon`, `keypoints`, `cuboid`, `category`, `scene_category`, `segmentation`) emits its `dataset_item_id` in `to_payload` (as `item_id`) when set, which is how the dataset-less upload route resolves each item. + +### Changed +- **`reference_id` is now optional on predictions.** A prediction can be constructed from its `dataset_item_id` alone (at least one of `reference_id` / `dataset_item_id` is required). Annotations still require `reference_id`. Existing prediction code that passes `reference_id` is unaffected. > **Server dependency:** requires the `POST /nucleus/model/:modelId/modelRun/create` and `POST /nucleus/modelRun/:modelRunId/uploadPredictions` routes in scaleapi. Unit tests pass regardless; live calls 404 until that deploys. diff --git a/nucleus/prediction.py b/nucleus/prediction.py index 2bb9ff4a..0e44c29b 100644 --- a/nucleus/prediction.py +++ b/nucleus/prediction.py @@ -77,6 +77,13 @@ def _add_prediction_target_ids( return payload +def _require_prediction_target(reference_id, dataset_item_id): + if reference_id is None and dataset_item_id is None: + raise ValueError( + "A prediction must set reference_id or dataset_item_id." + ) + + def from_json(payload: dict): """Instantiates prediction object from schematized JSON dict payload.""" type_key_to_type: Dict[str, Type[Prediction]] = { @@ -133,7 +140,8 @@ class SegmentationPrediction(SegmentationAnnotation): example above these would map that 0 to background, 1 to car and 2 to pedestrian. In the instance segmentation example above, 0 and 1 would both be mapped to car, 2 and 3 would both be mapped to motorcycle - reference_id (str): User-defined ID of the image to which to apply this annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. annotation_id (Optional[str]): For segmentation predictions, this value is ignored because there can only be one segmentation prediction per dataset item. Therefore regardless of annotation ID, if there is an existing @@ -147,7 +155,7 @@ def __init__( self, mask_url: str, annotations: List[Segment], - reference_id: str, + reference_id: Optional[str] = None, annotation_id: Optional[str] = None, dataset_item_id: Optional[str] = None, *, @@ -160,6 +168,7 @@ def __init__( annotation_id=annotation_id, dataset_item_id=dataset_item_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.dataset_id = dataset_id def to_payload(self) -> dict: @@ -178,7 +187,7 @@ def from_json(cls, payload: dict): Segment.from_json(ann) for ann in payload.get(ANNOTATIONS_KEY, []) ], - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), dataset_id=payload.get(DATASET_ID_KEY), annotation_id=payload.get(ANNOTATION_ID_KEY, None), @@ -197,8 +206,8 @@ class BoxPrediction(BoxAnnotation): of the bounding box and the top border of the image. width (Union[float, int]): The width in pixels of the annotation. height (Union[float, int]): The height in pixels of the annotation. - reference_id (str): User-defined ID of the image to which to apply this - annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, @@ -234,7 +243,7 @@ def __init__( y: Union[float, int], width: Union[float, int], height: Union[float, int], - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -258,6 +267,7 @@ def __init__( embedding_vector=embedding_vector, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf self.dataset_id = dataset_id @@ -283,7 +293,7 @@ def from_json(cls, payload: dict): y=geometry.get(Y_KEY, 0), width=geometry.get(WIDTH_KEY, 0), height=geometry.get(HEIGHT_KEY, 0), - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), @@ -301,8 +311,8 @@ class LinePrediction(LineAnnotation): Parameters: label (str): The label for this prediction (e.g. car, pedestrian, bicycle). vertices (List[:class:`Point`]): The list of points making up the line. - reference_id (str): User-defined ID of the image to which to apply this - annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, a matching @@ -325,7 +335,7 @@ def __init__( self, label: str, vertices: List[Point], - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -344,6 +354,7 @@ def __init__( metadata=metadata, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf self.dataset_id = dataset_id @@ -368,7 +379,7 @@ def from_json(cls, payload: dict): vertices=[ Point.from_json(_) for _ in geometry.get(VERTICES_KEY, []) ], - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), @@ -385,8 +396,8 @@ class PolygonPrediction(PolygonAnnotation): Parameters: label (str): The label for this annotation (e.g. car, pedestrian, bicycle). vertices (List[:class:`Point`]): The list of points making up the polygon. - reference_id (str): User-defined ID of the image to which to apply this - annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, a matching @@ -412,7 +423,7 @@ def __init__( self, label: str, vertices: List[Point], - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -433,6 +444,7 @@ def __init__( embedding_vector=embedding_vector, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf self.dataset_id = dataset_id @@ -457,7 +469,7 @@ def from_json(cls, payload: dict): vertices=[ Point.from_json(_) for _ in geometry.get(VERTICES_KEY, []) ], - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), @@ -478,8 +490,8 @@ class KeypointsPrediction(KeypointsAnnotation): names (List[str]): A list that corresponds to the names of each keypoint. skeleton (List[List[int]]): A list of 2-length lists indicating a beginning and ending index for each line segment in the skeleton of this keypoint label. - reference_id (str): User-defined ID of the image to which to apply this - annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, a matching @@ -504,7 +516,7 @@ def __init__( keypoints: List[Keypoint], names: List[str], skeleton: List[List[int]], - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -525,6 +537,7 @@ def __init__( metadata=metadata, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf self.dataset_id = dataset_id @@ -551,7 +564,7 @@ def from_json(cls, payload: dict): ], names=geometry[KEYPOINTS_NAMES_KEY], skeleton=geometry[KEYPOINTS_SKELETON_KEY], - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), @@ -570,7 +583,8 @@ class CuboidPrediction(CuboidAnnotation): position (:class:`Point3D`): The point at the center of the cuboid dimensions (:class:`Point3D`): The length (x), width (y), and height (z) of the cuboid yaw (float): The rotation, in radians, about the Z axis of the cuboid - reference_id (str): User-defined ID of the image to which to apply this annotation. + reference_id (Optional[str]): Optional user-defined ID of the item to which to apply + this prediction. Provide this or ``dataset_item_id``. confidence: 0-1 indicating the confidence of the prediction. annotation_id (Optional[str]): The annotation ID that uniquely identifies this annotation within its target dataset item. Upon ingest, a matching @@ -595,7 +609,7 @@ def __init__( position: Point3D, dimensions: Point3D, yaw: float, - reference_id: str, + reference_id: Optional[str] = None, confidence: Optional[float] = None, annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, @@ -616,6 +630,7 @@ def __init__( metadata=metadata, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf self.dataset_id = dataset_id @@ -640,7 +655,7 @@ def from_json(cls, payload: dict): position=Point3D.from_json(geometry.get(POSITION_KEY, {})), dimensions=Point3D.from_json(geometry.get(DIMENSIONS_KEY, {})), yaw=geometry.get(YAW_KEY, 0), - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), @@ -656,7 +671,8 @@ class CategoryPrediction(CategoryAnnotation): Parameters: label: The label for this annotation (e.g. car, pedestrian, bicycle). - reference_id: The reference ID of the image you wish to apply this annotation to. + reference_id: Optional user-defined ID of the item to which to apply this + prediction. Provide this or ``dataset_item_id``. taxonomy_name: The name of the taxonomy this annotation conforms to. See :meth:`Dataset.add_taxonomy`. confidence: 0-1 indicating the confidence of the prediction. @@ -676,7 +692,7 @@ class CategoryPrediction(CategoryAnnotation): def __init__( self, label: str, - reference_id: str, + reference_id: Optional[str] = None, taxonomy_name: Optional[str] = None, confidence: Optional[float] = None, metadata: Optional[Dict] = None, @@ -694,6 +710,7 @@ def __init__( metadata=metadata, track_reference_id=track_reference_id, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.class_pdf = class_pdf self.dataset_id = dataset_id @@ -715,7 +732,7 @@ def from_json(cls, payload: dict): return cls( label=payload.get(LABEL_KEY, 0), taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), @@ -744,7 +761,8 @@ class SceneCategoryPrediction(SceneCategoryAnnotation): Parameters: label: The label for this annotation (e.g. action, subject, scenario). - reference_id: The reference ID of the scene you wish to apply this annotation to. + reference_id: Optional user-defined ID of the item to which to apply this + prediction. Provide this or ``dataset_item_id``. taxonomy_name: The name of the taxonomy this annotation conforms to. See :meth:`Dataset.add_taxonomy`. confidence: 0-1 indicating the confidence of the prediction. @@ -757,7 +775,7 @@ class SceneCategoryPrediction(SceneCategoryAnnotation): def __init__( self, label: str, - reference_id: str, + reference_id: Optional[str] = None, taxonomy_name: Optional[str] = None, confidence: Optional[float] = None, metadata: Optional[Dict] = None, @@ -772,6 +790,7 @@ def __init__( dataset_item_id=dataset_item_id, metadata=metadata, ) + _require_prediction_target(reference_id, dataset_item_id) self.confidence = confidence self.dataset_id = dataset_id @@ -790,7 +809,7 @@ def from_json(cls, payload: dict): return cls( label=payload.get(LABEL_KEY, 0), taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), - reference_id=payload[REFERENCE_ID_KEY], + reference_id=payload.get(REFERENCE_ID_KEY), dataset_item_id=payload.get(DATASET_ITEM_ID_KEY), dataset_id=payload.get(DATASET_ID_KEY), confidence=payload.get(CONFIDENCE_KEY, None), diff --git a/tests/test_dataset_less_model_runs.py b/tests/test_dataset_less_model_runs.py index 2b899ee5..d7df3c98 100644 --- a/tests/test_dataset_less_model_runs.py +++ b/tests/test_dataset_less_model_runs.py @@ -20,7 +20,7 @@ from nucleus.errors import DuplicateIDError from nucleus.model import Model from nucleus.model_run import ModelRun -from nucleus.prediction import BoxPrediction +from nucleus.prediction import BoxPrediction, CategoryPrediction def _client(): @@ -248,3 +248,33 @@ def test_box_prediction_omits_target_ids_when_unset(): payload = _predictions()[0].to_payload() assert "item_id" not in payload assert "dataset_id" not in payload + + +# --------------------------------------------------------------------------- # +# Predictions target an item by dataset_item_id alone (no reference_id) +# --------------------------------------------------------------------------- # +def test_box_prediction_builds_from_dataset_item_id_alone(): + pred = BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + dataset_item_id="di_1", + confidence=0.9, + ) + payload = pred.to_payload() + assert payload["item_id"] == "di_1" + assert payload.get("reference_id") is None + + +def test_prediction_with_no_target_is_rejected(): + with pytest.raises(ValueError, match="reference_id or dataset_item_id"): + BoxPrediction(label="car", x=0, y=0, width=10, height=10) + + +def test_category_prediction_builds_from_dataset_item_id_alone(): + pred = CategoryPrediction(label="car", dataset_item_id="di_1") + payload = pred.to_payload() + assert payload["item_id"] == "di_1" + assert payload.get("reference_id") is None From 039a8cee0d3433ff6b14086a01ee087c657589cc Mon Sep 17 00:00:00 2001 From: Luke Schaefer Date: Thu, 20 Aug 2026 13:22:47 -0500 Subject: [PATCH 7/7] fix(prediction): silence mypy on optional reference_id forward Predictions may omit reference_id (when dataset_item_id is set) but forward it to annotation __init__s typed reference_id: str. Annotations still require it, so keep the base type str and mark the 8 prediction forwards with type: ignore[arg-type] rather than weakening the annotation type. Co-Authored-By: Claude Opus 4.8 (1M context) --- nucleus/prediction.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/nucleus/prediction.py b/nucleus/prediction.py index 0e44c29b..b8dd3575 100644 --- a/nucleus/prediction.py +++ b/nucleus/prediction.py @@ -164,7 +164,9 @@ def __init__( super().__init__( mask_url=mask_url, annotations=annotations, - reference_id=reference_id, + # Predictions may omit reference_id when dataset_item_id is set; + # annotations still require it, hence the base type is str. + reference_id=reference_id, # type: ignore[arg-type] annotation_id=annotation_id, dataset_item_id=dataset_item_id, ) @@ -260,7 +262,9 @@ def __init__( y=y, width=width, height=height, - reference_id=reference_id, + # Predictions may omit reference_id when dataset_item_id is set; + # annotations still require it, hence the base type is str. + reference_id=reference_id, # type: ignore[arg-type] dataset_item_id=dataset_item_id, annotation_id=annotation_id, metadata=metadata, @@ -348,7 +352,9 @@ def __init__( super().__init__( label=label, vertices=vertices, - reference_id=reference_id, + # Predictions may omit reference_id when dataset_item_id is set; + # annotations still require it, hence the base type is str. + reference_id=reference_id, # type: ignore[arg-type] dataset_item_id=dataset_item_id, annotation_id=annotation_id, metadata=metadata, @@ -437,7 +443,9 @@ def __init__( super().__init__( label=label, vertices=vertices, - reference_id=reference_id, + # Predictions may omit reference_id when dataset_item_id is set; + # annotations still require it, hence the base type is str. + reference_id=reference_id, # type: ignore[arg-type] dataset_item_id=dataset_item_id, annotation_id=annotation_id, metadata=metadata, @@ -531,7 +539,9 @@ def __init__( keypoints=keypoints, names=names, skeleton=skeleton, - reference_id=reference_id, + # Predictions may omit reference_id when dataset_item_id is set; + # annotations still require it, hence the base type is str. + reference_id=reference_id, # type: ignore[arg-type] dataset_item_id=dataset_item_id, annotation_id=annotation_id, metadata=metadata, @@ -624,7 +634,9 @@ def __init__( position=position, dimensions=dimensions, yaw=yaw, - reference_id=reference_id, + # Predictions may omit reference_id when dataset_item_id is set; + # annotations still require it, hence the base type is str. + reference_id=reference_id, # type: ignore[arg-type] dataset_item_id=dataset_item_id, annotation_id=annotation_id, metadata=metadata, @@ -705,7 +717,9 @@ def __init__( super().__init__( label=label, taxonomy_name=taxonomy_name, - reference_id=reference_id, + # Predictions may omit reference_id when dataset_item_id is set; + # annotations still require it, hence the base type is str. + reference_id=reference_id, # type: ignore[arg-type] dataset_item_id=dataset_item_id, metadata=metadata, track_reference_id=track_reference_id, @@ -786,7 +800,9 @@ def __init__( super().__init__( label=label, taxonomy_name=taxonomy_name, - reference_id=reference_id, + # Predictions may omit reference_id when dataset_item_id is set; + # annotations still require it, hence the base type is str. + reference_id=reference_id, # type: ignore[arg-type] dataset_item_id=dataset_item_id, metadata=metadata, )