How to upload predictions on image data in a model run and sample upload formats.
You can use the Python SDK to upload predictions on images.
This page shows how to declare the predictions and demonstrates the upload process.
A Python notebook demonstrates these steps and can be run directly with Google CoLab.
Supported predictions
To upload predictions in Labelbox, you need to create prediction payloads. This section shows how to create payloads for each supported prediction type. You can declare payloads as Python annotation types (preferred) or NDJSON objects.
Confidence scores are optional. Predictions uploaded without confidence score are treated as if they have a score of one (1).
Classification: Radio (single-choice)
radio_prediction = lb_types.ClassificationAnnotation(
name="radio_question",
value=lb_types.Radio(answer = lb_types.ClassificationAnswer(name = "second_radio_answer", confidence=0.5))
)
radio_prediction_ndjson = {
"name": "radio_question",
"answer": {"name": "second_radio_answer", "confidence": 0.5}
}
Classification: Nested radio
nested_radio_prediction = lb_types.ClassificationAnnotation(
name="nested_radio_question",
value=lb_types.Radio(
answer=lb_types.ClassificationAnswer(
name="first_radio_answer",
confidence=0.5,
classifications=[
lb_types.ClassificationAnnotation(
name="sub_radio_question",
value=lb_types.Radio(
answer=lb_types.ClassificationAnswer(
name="first_sub_radio_answer",
confidence=0.5
)
)
)
]
)
)
)
nested_radio_prediction_ndjson = {
"name": "nested_radio_question",
"answer": {
"name": "first_radio_answer",
"confidence": 0.5,
"classifications": [{
"name": "sub_radio_question",
"answer": {
"name": "first_sub_radio_answer",
"confidence": 0.5
}
}]
}
}
Classification: Nested checklist
nested_checklist_prediction = lb_types.ClassificationAnnotation(
name="nested_checklist_question",
value=lb_types.Checklist(
answer=[lb_types.ClassificationAnswer(
name="first_checklist_answer",
confidence=0.5,
classifications=[
lb_types.ClassificationAnnotation(
name="sub_checklist_question",
value=lb_types.Checklist(
answer=[lb_types.ClassificationAnswer(
name="first_sub_checklist_answer",
confidence=0.5
)]
))
]
)]
)
)
nested_checklist_prediction_ndjson = {
"name": "nested_checklist_question",
"answer": [{
"name": "first_checklist_answer",
"confidence": 0.5,
"classifications": [{
"name": "sub_checklist_question",
"answer": {
"name": "first_sub_checklist_answer",
"confidence": 0.5
}
}]
}]
}
Classification: Checklist (multiple choice)
checklist_prediction = lb_types.ClassificationAnnotation(
name="checklist_question",
value=lb_types.Checklist(
answer = [
lb_types.ClassificationAnswer(
name = "first_checklist_answer",
confidence=0.5
),
lb_types.ClassificationAnswer(
name = "second_checklist_answer",
confidence=0.5
)
]
)
)
checklist_prediction_ndjson = {
"name": "checklist_question",
"answer": [
{"name": "first_checklist_answer" , "confidence": 0.5},
{"name": "second_checklist_answer", "confidence": 0.5}
]
}
Bounding box
bbox_prediction = lb_types.ObjectAnnotation(
name = "bounding_box",
confidence=0.5,
value=lb_types.Rectangle(
start=lb_types.Point(x=1690, y=977), # x = left, y = top
end=lb_types.Point(x=1915, y=1307), # x= left + width , y = top + height
)
)
bbox_prediction_ndjson = {
"name": "bounding_box",
"confidence": 0.5,
"bbox": {
"top": 977,
"left": 1690,
"height": 330,
"width": 225
}
}
Bounding box with nested classification
bbox_with_radio_subclass_prediction = lb_types.ObjectAnnotation(
name="bbox_with_radio_subclass",
confidence=0.5,
value=lb_types.Rectangle(
start=lb_types.Point(x=541, y=933), # x = left, y = top
end=lb_types.Point(x=871, y=1124), # x= left + width , y = top + height
),
classifications=[
lb_types.ClassificationAnnotation(
name="sub_radio_question",
value=lb_types.Radio(answer=lb_types.ClassificationAnswer(name="first_sub_radio_answer", confidence=0.5))
)
]
)
bbox_with_radio_subclass_prediction_ndjson = {
"name": "bbox_with_radio_subclass",
"confidence": 0.5
"classifications": [{
"name": "sub_radio_question",
"answer":
{ "name":"first_sub_radio_answer", "confidence": 0.5}
}],
"bbox": {
"top": 933,
"left": 541,
"height": 191,
"width": 330
}
}
Polygon
polygon_prediction = lb_types.ObjectAnnotation(
name = "polygon",
confidence = 0.5,
value=lb_types.Polygon(
points=[lb_types.Point(x=1489.581,y=183.934), lb_types.Point(x=2278.306,y=256.885), lb_types.Point(x=2428.197,y=200.437), lb_types.Point(x=2560.0,y=335.419),
lb_types.Point(x=2557.386,y=503.165), lb_types.Point(x=2320.596,y=503.103), lb_types.Point(x=2156.083, y=628.943), lb_types.Point(x=2161.111,y=785.519),
lb_types.Point(x=2002.115, y=894.647), lb_types.Point(x=1838.456,y=877.874), lb_types.Point(x=1436.53,y=874.636), lb_types.Point(x=1411.403,y=758.579),
lb_types.Point(x=1353.853,y=751.74), lb_types.Point(x=1345.264, y=453.461), lb_types.Point(x=1426.011,y=421.129)]
),
)
polygon_prediction_ndjson = {
"name": "polygon",
"confidence": 0.5,
"polygon": [
{"x": 1489.581, "y": 183.934},
{"x": 2278.306, "y": 256.885},
{"x": 2428.197, "y": 200.437},
{"x": 2560.0, "y": 335.419},
{"x": 2557.386, "y": 503.165},
{"x": 2320.596, "y": 503.103},
{"x": 2156.083, "y": 628.943},
{"x": 2161.111, "y": 785.519},
{"x": 2002.115, "y": 894.647},
{"x": 1838.456, "y": 877.874},
{"x": 1436.53, "y": 874.636},
{"x": 1411.403, "y": 758.579},
{"x": 1353.853, "y": 751.74},
{"x": 1345.264, "y": 453.461},
{"x": 1426.011, "y": 421.129},
{"x": 1489.581, "y": 183.934}
]
}
Classification: free-form text
text_annotation = lb_types.ClassificationAnnotation(
name="free_text",
value=lb_types.Text(answer="sample text", confidence=0.5)
)
text_annotation_ndjson = {
"name": "free_text",
"confidence": 0.5,
"answer": "sample text",
}
Segmentation mask
# We recommend to use byte array
url = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg.png"
response = requests.get(url)
mask_data = lb.types.MaskData(im_bytes=response.content) # You can also use "url" instead of img_bytes to pass the PNG mask url.
mask_prediction = lb_types.ObjectAnnotation(
name="mask",
confidence=0.5,
value=lb_types.Mask(
mask=mask_data,
color=(255, 255, 255))
)
# using instanceURI, bytes array is not fully supported.
mask_prediction_ndjson = {
"name": "mask",
"confidence": 0.5,
"classifications": [],
"mask": {"instanceURI": url,
"colorRGB": (255, 255, 255)}
}
Segmentation mask with nested classification
url_2 = "https://storage.googleapis.com/labelbox-datasets/image_sample_data/raster_seg_with_subclass.png"
response_2 = requests.get(url_2)
mask_data_2 = lb_types.MaskData(im_bytes=response_2.content)
# Python annotation
mask_with_text_subclass_prediction = lb_types.ObjectAnnotation(
name = "mask_with_text_subclass", # must match your ontology feature"s name
value=lb_types.Mask(
mask=mask_data_2,
confidence=0.5,
color=(255, 255, 255)),
classifications=[
lb_types.ClassificationAnnotation(
name="sub_free_text",
value=lb_types.Text(answer="free text answer")
)]
)
mask_with_text_subclass_prediction_ndjson = {
"name": "mask_with_text_subclass",
"mask": {"instanceURI": url_2,
"colorRGB": (255, 255, 255)},
"classifications":[{
"name": "sub_free_text",
"answer": "free text answer"
}]
}
Point
point_prediction = lb_types.ObjectAnnotation(
name = "point",
confidence=0.5,
value = lb_types.Point(x=1166.606, y=1441.768),
)
point_prediction_ndjson = {
"name": "point",
"confidence": 0.5,
"classifications": [],
"point": {"x": 1166.606, "y": 1441.768}
}
Polyline
polyline_prediction = lb_types.ObjectAnnotation(
name = "polyline",
confidence=0.5,
value=lb_types.Line(
points=[lb_types.Point(x=2534.353, y=249.471), lb_types.Point(x=2429.492, y=182.092), lb_types.Point(x=2294.322, y=221.962), lb_types.Point(x=2224.491, y=180.463), lb_types.Point(x=2136.123, y=204.716),
lb_types.Point(x=1712.247, y=173.949), lb_types.Point(x=1703.838, y=84.438), lb_types.Point(x=1579.772, y=82.61), lb_types.Point(x=1583.442, y=167.552),
lb_types.Point(x=1478.869, y=164.903), lb_types.Point(x=1418.941, y=318.149), lb_types.Point(x=1243.128, y=400.815), lb_types.Point(x=1022.067, y=319.007),
lb_types.Point(x=892.367, y=379.216), lb_types.Point(x=670.273, y=364.408), lb_types.Point(x=613.114, y=288.16), lb_types.Point(x=377.559, y=238.251),
lb_types.Point(x=368.087, y=185.064), lb_types.Point(x=246.557, y=167.286), lb_types.Point(x=236.648, y=285.61), lb_types.Point(x=90.929, y=326.412)]
),
)
polyline_prediction_ndjson = {
"name": "polyline",
"confidence":0.5,
"classifications": [],
"line": [
{"x": 2534.353, "y": 249.471},
{"x": 2429.492, "y": 182.092},
{"x": 2294.322, "y": 221.962},
{"x": 2224.491, "y": 180.463},
{"x": 2136.123, "y": 204.716},
{"x": 1712.247, "y": 173.949},
{"x": 1703.838, "y": 84.438},
{"x": 1579.772, "y": 82.61},
{"x": 1583.442, "y": 167.552},
{"x": 1478.869, "y": 164.903},
{"x": 1418.941, "y": 318.149},
{"x": 1243.128, "y": 400.815},
{"x": 1022.067, "y": 319.007},
{"x": 892.367, "y": 379.216},
{"x": 670.273, "y": 364.408},
{"x": 613.114, "y": 288.16},
{"x": 377.559, "y": 238.251},
{"x": 368.087, "y": 185.064},
{"x": 246.557, "y": 167.286},
{"x": 236.648, "y": 285.61},
{"x": 90.929, "y": 326.412}
]
}
Example: Upload predictions to model run
To upload predictions to a model run:
Before you start
Thse examples require the following libraries:
import labelbox as lb
import labelbox.types as lb_types
import uuid
import requests
Replace API key
Paste your API key as the value of the API_KEY variable:
API_KEY = ""
client = lb.Client(API_KEY)
Step 1: Import data rows into Catalog
# send a sample image as batch to the project
global_key = "2560px-Kitano_Street_Kobe01s5s4110.jpeg"
test_img_url = {
"row_data": "https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg",
"global_key": global_key
}
dataset = client.create_dataset(name="image_prediction_demo")
task = dataset.create_data_rows([test_img_url])
print("Errors:",task.errors)
print("Failed data rows:", task.failed_data_rows)
Step 2: Set up ontology for predictions
Your model run ontology should support all tools and classifications used in your predictions.
This example shows how to create an ontology with all supported prediction types .
ontology_builder = lb.OntologyBuilder(
classifications=[ # List of Classification objects
lb.Classification(
class_type=lb.Classification.Type.RADIO,
name="radio_question",
options=[
lb.Option(value="first_radio_answer"),
lb.Option(value="second_radio_answer")
]
),
lb.Classification(
class_type=lb.Classification.Type.CHECKLIST,
name="checklist_question",
options=[
lb.Option(value="first_checklist_answer"),
lb.Option(value="second_checklist_answer")
]
),
lb.Classification(
class_type=lb.Classification.Type.TEXT,
name="free_text"
),
lb.Classification(
class_type=lb.Classification.Type.RADIO,
name="nested_radio_question",
options=[
lb.Option("first_radio_answer",
options=[
lb.Classification(
class_type=lb.Classification.Type.RADIO,
name="sub_radio_question",
options=[lb.Option("first_sub_radio_answer")]
)
]
)
]
),
lb.Classification(
class_type=lb.Classification.Type.CHECKLIST,
name="nested_checklist_question",
options=[
lb.Option("first_checklist_answer",
options=[
lb.Classification(
class_type=lb.Classification.Type.CHECKLIST,
name="sub_checklist_question",
options=[lb.Option("first_sub_checklist_answer")]
)
]
)
]
),
],
tools=[ # List of tools
lb.Tool(
tool=lb.Tool.Type.BBOX,
name="bounding_box"),
lb.Tool(
tool=lb.Tool.Type.BBOX,
name="bbox_with_radio_subclass",
classifications=[
lb.Classification(
class_type=lb.Classification.Type.RADIO,
name="sub_radio_question",
options=[
lb.Option(value="first_sub_radio_answer")
]
),
]
),
lb.Tool(
tool=lb.Tool.Type.POLYGON,
name="polygon"),
lb.Tool(
tool=lb.Tool.Type.RASTER_SEGMENTATION,
name="mask"),
lb.Tool(
tool=lb.Tool.Type.RASTER_SEGMENTATION,
name="mask_with_text_subclass",
classifications=[
lb.Classification(
class_type=lb.Classification.Type.TEXT,
name="sub_free_text")
]
),
lb.Tool(
tool=lb.Tool.Type.POINT,
name="point"),
lb.Tool(
tool=lb.Tool.Type.LINE,
name="polyline")]
)
ontology = client.create_ontology("Image Prediction Import Demo",
ontology_builder.asdict(),
media_type=lb.MediaType.Image
)
Step 3: Create model and model run
Create a Model using the ontology and a model run.
# create Model
model = client.create_model(name="image_model_run_" + str(uuid.uuid4()),
ontology_id=ontology.uid)
# create Model Run
model_run = model.create_model_run("iteration 1")
Step 4: Send data rows to model run
model_run.upsert_data_rows(global_keys=[global_key])
Step 5: Create prediction payloads
For help creating prediction payloads, see supported predictions. You can declare payloads as Python annotation types (preferred) or as NDJSON objects. This example demonstrates each format and shows how to compose annotations into labels attached to the data rows.
The resulting label_prediction_ndjson and label_prediction payloads should have exactly the same prediction content (except for the value of generated uuid string values).
label_prediction = []
label_prediction.append(lb_types.Label(
data=lb_types.ImageData(global_key=global_key),
annotations = [
radio_prediction,
nested_radio_prediction,
checklist_prediction,
nested_checklist_prediction,
bbox_prediction,
bbox_with_radio_subclass_prediction,
polyline_prediction,
polygon_prediction,
mask_prediction,
mask_with_text_subclass_prediction,
point_prediction,
text_annotation
]
)
)
label_prediction_ndjson = []
for annot in [
radio_prediction_ndjson,
checklist_prediction_ndjson,
bbox_prediction_ndjson,
bbox_with_radio_subclass_prediction_ndjson,
polygon_prediction_ndjson,
mask_prediction_ndjson,
mask_with_text_subclass_prediction_ndjson,
point_prediction_ndjson,
polyline_prediction_ndjson,
text_annotation_ndjson,
nested_radio_prediction_ndjson,
nested_checklist_prediction_ndjson
]:
annot.update({
"dataRow": {"globalKey": global_key}
})
label_prediction_ndjson.append(annot)
Step 6: Upload prediction payloads to model run
# Upload the prediction label to the Model Run
upload_job_prediction = model_run.add_predictions(
name="prediction_upload_job"+str(uuid.uuid4()),
predictions=label_prediction)
# Errors will appear for prediction uploads that failed.
print("Errors:", upload_job_prediction.errors)
print("Status of uploads: ", upload_job_prediction.statuses)
Step 7: Send annotations to a model run
This step is optional.
This example creates a project with ground truth annotations to visualize both annotations and predictions in the model run.
To send annotations to a model run:
- Import them into a project
- Create a label payload
- Send them to the model run.
# 7.1 Create a labelbox project
project = client.create_project(name="Image Prediction Demo",
media_type=lb.MediaType.Image)
project.setup_editor(ontology)
# 7.2 Create a batch
project.create_batch(
"batch_predictions_demo", # Each batch in a project must have a unique name
global_keys=[global_key], # Paginated collection of data row objects, list of data row ids or global keys
priority=5 # priority between 1(Highest) - 5(lowest)
)
# 7.3 Create your annotation payload as explained in:
# https://docs.labelbox.com/reference/import-image-annotations#supported-annotations
radio_annotation ...
nested_radio_annotation ...
nested_checklist_annotation ...
checklist_annotation ...
bbox_annotation ...
bbox_with_radio_subclass_annotation ...
polygon_annotation ...
text_annotation ...
mask_annotation ...
mask_with_text_subclass_annotation ...
point_annotation ...
polyline_annotation ...
# 7.4 Create a Label object by identifying the applicable data row in Labelbox and providing a list of annotations
labels = []
annotations = [
radio_annotation,
nested_radio_annotation,
checklist_annotation,
nested_checklist_annotation,
text_annotation,
bbox_annotation,
bbox_with_radio_subclass_annotation,
polygon_annotation,
mask_annotation,
mask_with_text_subclass_annotation,
point_annotation,
polyline_annotation
]
labels.append(
lb_types.Label(
data=lb_types.ImageData(global_key=global_key),
annotations=annotations))
# 7.5 Upload annotations to the project using Label import
upload_job_annotation = lb.LabelImport.create_from_objects(
client = client,
project_id = project.uid,
name="annotation_import_" + str(uuid.uuid4()),
labels=labels)
upload_job_annotation.wait_until_done()
# Errors will appear for annotation uploads that failed.
print("Errors:", upload_job_annotation.errors)
print("Status of uploads: ", upload_job_annotation.statuses)
# 7.6 Send the annotations to the model run
model_run.upsert_labels(project_id=project.uid)