├── .dockerignore ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md ├── actions │ ├── lbox-matrix │ │ ├── action.yml │ │ └── index.js │ ├── provenance │ │ └── action.yml │ └── python-package-shared-setup │ │ └── action.yml ├── pull_request_template.md └── workflows │ ├── lbox-develop.yml │ ├── lbox-publish.yml │ ├── notebooks.yml │ ├── publish.yml │ ├── python-package-develop.yml │ ├── python-package-prod.yml │ ├── python-package-shared.yml │ └── secrets_scan.yml ├── .gitignore ├── .python-version ├── .readthedocs.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── docs ├── conf.py ├── github-workflow.svg ├── labelbox │ ├── annotation-import.rst │ ├── app.rst │ ├── asset-attachment.rst │ ├── batch.rst │ ├── benchmark.rst │ ├── client.rst │ ├── conflict-resolution-strategy.rst │ ├── datarow-metadata.rst │ ├── datarow.rst │ ├── datarow_payload_templates.rst │ ├── dataset.rst │ ├── embedding.rst │ ├── enums.rst │ ├── exceptions.rst │ ├── export-task.rst │ ├── fact-checking-tool.rst │ ├── foundry-client.rst │ ├── foundry-model.rst │ ├── identifiable.rst │ ├── identifiables.rst │ ├── index.rst │ ├── label-score.rst │ ├── label.rst │ ├── labeling-frontend-options.rst │ ├── labeling-frontend.rst │ ├── labeling-parameter-override.rst │ ├── labeling-service-dashboard.rst │ ├── labeling-service-status.rst │ ├── labeling-service.rst │ ├── model-config.rst │ ├── model-run.rst │ ├── model.rst │ ├── ontology-kind.rst │ ├── ontology.rst │ ├── organization.rst │ ├── pagination.rst │ ├── project-model-config.rst │ ├── project-overview.rst │ ├── project.rst │ ├── prompt-issue-tool.rst │ ├── quality-mode.rst │ ├── request-client.rst │ ├── resource-tag.rst │ ├── review.rst │ ├── search-filters.rst │ ├── send-to-annotate-params.rst │ ├── slice.rst │ ├── step-reasoning-tool.rst │ ├── task-queue.rst │ ├── task.rst │ ├── user-group-v2.rst │ ├── user-group.rst │ ├── user.rst │ └── webhook.rst └── logo-full-black.svg ├── examples ├── CONTRIBUTING.md ├── README.md ├── annotation_import │ ├── audio.ipynb │ ├── conversational.ipynb │ ├── conversational_LLM.ipynb │ ├── conversational_LLM_data_generation.ipynb │ ├── html.ipynb │ ├── image.ipynb │ ├── pdf.ipynb │ ├── text.ipynb │ ├── tiled.ipynb │ └── video.ipynb ├── basics │ ├── basics.ipynb │ ├── batches.ipynb │ ├── custom_embeddings.ipynb │ ├── data_row_metadata.ipynb │ ├── data_rows.ipynb │ ├── ontologies.ipynb │ ├── projects.ipynb │ ├── quick_start.ipynb │ └── user_management.ipynb ├── exports │ ├── composite_mask_export.ipynb │ ├── export_data.ipynb │ ├── export_v1_to_v2_migration_support.ipynb │ └── exporting_to_csv.ipynb ├── foundry │ └── object_detection.ipynb ├── integrations │ ├── huggingface │ │ └── huggingface_custom_embeddings.ipynb │ ├── langchain │ │ └── langchain.ipynb │ ├── sam │ │ ├── meta_sam.ipynb │ │ └── meta_sam_video.ipynb │ └── yolo │ │ └── import_yolov8_annotations.ipynb ├── model_experiments │ ├── custom_metrics_basics.ipynb │ ├── custom_metrics_demo.ipynb │ ├── model_predictions_to_project.ipynb │ └── model_slices.ipynb ├── prediction_upload │ ├── conversational_LLM_predictions.ipynb │ ├── conversational_predictions.ipynb │ ├── geospatial_predictions.ipynb │ ├── html_predictions.ipynb │ ├── image_predictions.ipynb │ ├── pdf_predictions.ipynb │ ├── text_predictions.ipynb │ └── video_predictions.ipynb ├── project_configuration │ ├── multimodal_chat_project.ipynb │ ├── project_setup.ipynb │ ├── queue_management.ipynb │ └── webhooks.ipynb ├── pyproject.toml ├── scripts │ ├── format_notebooks.py │ └── generate_readme.py └── template.ipynb ├── libs ├── labelbox │ ├── .yapfignore │ ├── CHANGELOG.md │ ├── Dockerfile │ ├── README.md │ ├── mypy.ini │ ├── pyproject.toml │ ├── src │ │ └── labelbox │ │ │ ├── __init__.py │ │ │ ├── adv_client.py │ │ │ ├── annotated_types.py │ │ │ ├── client.py │ │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── annotation_types │ │ │ │ ├── __init__.py │ │ │ │ ├── annotation.py │ │ │ │ ├── base_annotation.py │ │ │ │ ├── classification │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── classification.py │ │ │ │ ├── collection.py │ │ │ │ ├── data │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base_data.py │ │ │ │ │ ├── generic_data_row_data.py │ │ │ │ │ ├── raster.py │ │ │ │ │ └── tiled_image.py │ │ │ │ ├── feature.py │ │ │ │ ├── geometry │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── geometry.py │ │ │ │ │ ├── line.py │ │ │ │ │ ├── mask.py │ │ │ │ │ ├── point.py │ │ │ │ │ ├── polygon.py │ │ │ │ │ └── rectangle.py │ │ │ │ ├── label.py │ │ │ │ ├── llm_prompt_response │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── prompt.py │ │ │ │ ├── metrics │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base.py │ │ │ │ │ ├── confusion_matrix.py │ │ │ │ │ └── scalar.py │ │ │ │ ├── mmc.py │ │ │ │ ├── ner │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── conversation_entity.py │ │ │ │ │ ├── document_entity.py │ │ │ │ │ └── text_entity.py │ │ │ │ ├── relationship.py │ │ │ │ ├── types.py │ │ │ │ └── video.py │ │ │ ├── generator.py │ │ │ ├── metrics │ │ │ │ ├── __init__.py │ │ │ │ ├── confusion_matrix │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── calculation.py │ │ │ │ │ └── confusion_matrix.py │ │ │ │ ├── group.py │ │ │ │ └── iou │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── calculation.py │ │ │ │ │ └── iou.py │ │ │ ├── mixins.py │ │ │ ├── ontology.py │ │ │ └── serialization │ │ │ │ ├── __init__.py │ │ │ │ └── ndjson │ │ │ │ ├── __init__.py │ │ │ │ ├── base.py │ │ │ │ ├── classification.py │ │ │ │ ├── converter.py │ │ │ │ ├── label.py │ │ │ │ ├── metric.py │ │ │ │ ├── mmc.py │ │ │ │ ├── objects.py │ │ │ │ └── relationship.py │ │ │ ├── orm │ │ │ ├── __init__.py │ │ │ ├── comparison.py │ │ │ ├── db_object.py │ │ │ ├── model.py │ │ │ └── query.py │ │ │ ├── pagination.py │ │ │ ├── parser.py │ │ │ ├── project_validation.py │ │ │ ├── schema │ │ │ ├── __init__.py │ │ │ ├── annotation_import.py │ │ │ ├── api_key.py │ │ │ ├── asset_attachment.py │ │ │ ├── batch.py │ │ │ ├── benchmark.py │ │ │ ├── catalog.py │ │ │ ├── confidence_presence_checker.py │ │ │ ├── conflict_resolution_strategy.py │ │ │ ├── consensus_settings.py │ │ │ ├── create_batches_task.py │ │ │ ├── data_row.py │ │ │ ├── data_row_metadata.py │ │ │ ├── data_row_payload_templates.py │ │ │ ├── dataset.py │ │ │ ├── embedding.py │ │ │ ├── enums.py │ │ │ ├── export_filters.py │ │ │ ├── export_params.py │ │ │ ├── export_task.py │ │ │ ├── foundry │ │ │ │ ├── __init__.py │ │ │ │ ├── app.py │ │ │ │ ├── foundry_client.py │ │ │ │ └── model.py │ │ │ ├── iam_integration.py │ │ │ ├── id_type.py │ │ │ ├── identifiable.py │ │ │ ├── identifiables.py │ │ │ ├── internal │ │ │ │ ├── data_row_uploader.py │ │ │ │ ├── data_row_upsert_item.py │ │ │ │ ├── datarow_upload_constants.py │ │ │ │ └── descriptor_file_creator.py │ │ │ ├── invite.py │ │ │ ├── label.py │ │ │ ├── label_score.py │ │ │ ├── labeling_frontend.py │ │ │ ├── labeling_service.py │ │ │ ├── labeling_service_dashboard.py │ │ │ ├── labeling_service_status.py │ │ │ ├── media_type.py │ │ │ ├── model.py │ │ │ ├── model_config.py │ │ │ ├── model_run.py │ │ │ ├── ontology.py │ │ │ ├── ontology_kind.py │ │ │ ├── organization.py │ │ │ ├── project.py │ │ │ ├── project_model_config.py │ │ │ ├── project_overview.py │ │ │ ├── project_resource_tag.py │ │ │ ├── quality_mode.py │ │ │ ├── resource_tag.py │ │ │ ├── review.py │ │ │ ├── role.py │ │ │ ├── search_filters.py │ │ │ ├── send_to_annotate_params.py │ │ │ ├── serialization.py │ │ │ ├── slice.py │ │ │ ├── task.py │ │ │ ├── task_queue.py │ │ │ ├── taskstatus.py │ │ │ ├── timeunit.py │ │ │ ├── tool_building │ │ │ │ ├── __init__.py │ │ │ │ ├── base_step_reasoning_tool.py │ │ │ │ ├── classification.py │ │ │ │ ├── fact_checking_tool.py │ │ │ │ ├── prompt_issue_tool.py │ │ │ │ ├── step_reasoning_tool.py │ │ │ │ ├── tool_type.py │ │ │ │ ├── tool_type_mapping.py │ │ │ │ └── types.py │ │ │ ├── user.py │ │ │ ├── user_group.py │ │ │ ├── user_group_v2.py │ │ │ └── webhook.py │ │ │ ├── types.py │ │ │ └── utils.py │ └── tests │ │ ├── assets │ │ ├── arxiv-pdf_data_99-word-token-pdfs_0801.3483-lb-textlayer.json │ │ ├── arxiv-pdf_data_99-word-token-pdfs_0801.3483.pdf │ │ ├── conversation-1.json │ │ └── loremipsum.pdf │ │ ├── conftest.py │ │ ├── data │ │ ├── annotation_import │ │ │ ├── conftest.py │ │ │ ├── test_annotation_import_limit.py │ │ │ ├── test_generic_data_types.py │ │ │ ├── test_label_import.py │ │ │ ├── test_mal_prediction_import.py │ │ │ ├── test_mea_prediction_import.py │ │ │ ├── test_model.py │ │ │ ├── test_model_run.py │ │ │ ├── test_relationships.py │ │ │ ├── test_send_to_annotate_mea.py │ │ │ └── test_upsert_prediction_import.py │ │ ├── annotation_types │ │ │ ├── __init__.py │ │ │ ├── classification │ │ │ │ ├── __init__.py │ │ │ │ └── test_classification.py │ │ │ ├── data │ │ │ │ ├── __init__.py │ │ │ │ └── test_raster.py │ │ │ ├── geometry │ │ │ │ ├── __init__.py │ │ │ │ ├── test_line.py │ │ │ │ ├── test_mask.py │ │ │ │ ├── test_point.py │ │ │ │ ├── test_polygon.py │ │ │ │ └── test_rectangle.py │ │ │ ├── test_annotation.py │ │ │ ├── test_collection.py │ │ │ ├── test_label.py │ │ │ ├── test_metrics.py │ │ │ ├── test_ner.py │ │ │ ├── test_text.py │ │ │ ├── test_tiled_image.py │ │ │ └── test_video.py │ │ ├── assets │ │ │ ├── coco │ │ │ │ ├── images │ │ │ │ │ ├── 000000129054.jpg │ │ │ │ │ ├── 000000141328.jpg │ │ │ │ │ ├── 000000147498.jpg │ │ │ │ │ ├── 000000184324.jpg │ │ │ │ │ ├── 000000188465.jpg │ │ │ │ │ ├── 000000245764.jpg │ │ │ │ │ ├── 000000311303.jpg │ │ │ │ │ ├── 000000343706.jpg │ │ │ │ │ ├── 000000356387.jpg │ │ │ │ │ ├── 000000370711.jpg │ │ │ │ │ ├── 000000385029.jpg │ │ │ │ │ ├── 000000393226.jpg │ │ │ │ │ ├── 000000434230.jpg │ │ │ │ │ ├── 000000458755.jpg │ │ │ │ │ ├── 000000458768.jpg │ │ │ │ │ ├── 000000462904.jpg │ │ │ │ │ ├── 000000475150.jpg │ │ │ │ │ ├── 000000532481.jpg │ │ │ │ │ ├── 000000532493.jpg │ │ │ │ │ └── 000000546823.jpg │ │ │ │ ├── instances.json │ │ │ │ ├── masks │ │ │ │ │ ├── 000000129054.png │ │ │ │ │ ├── 000000141328.png │ │ │ │ │ ├── 000000147498.png │ │ │ │ │ ├── 000000184324.png │ │ │ │ │ ├── 000000188465.png │ │ │ │ │ ├── 000000245764.png │ │ │ │ │ ├── 000000311303.png │ │ │ │ │ ├── 000000343706.png │ │ │ │ │ ├── 000000356387.png │ │ │ │ │ ├── 000000370711.png │ │ │ │ │ ├── 000000385029.png │ │ │ │ │ ├── 000000393226.png │ │ │ │ │ ├── 000000434230.png │ │ │ │ │ ├── 000000458755.png │ │ │ │ │ ├── 000000458768.png │ │ │ │ │ ├── 000000462904.png │ │ │ │ │ ├── 000000475150.png │ │ │ │ │ ├── 000000532481.png │ │ │ │ │ ├── 000000532493.png │ │ │ │ │ └── 000000546823.png │ │ │ │ ├── panoptic.json │ │ │ │ └── rle.json │ │ │ ├── labelbox_v1 │ │ │ │ ├── highly_nested_image.json │ │ │ │ ├── image_export.json │ │ │ │ ├── pdf_export.json │ │ │ │ ├── text_export.json │ │ │ │ ├── tiled_image_export.json │ │ │ │ ├── unkown_media_type_export.json │ │ │ │ └── video_export.json │ │ │ ├── line.png │ │ │ ├── mask.png │ │ │ ├── ndjson │ │ │ │ ├── classification_import.json │ │ │ │ ├── classification_import_name_only.json │ │ │ │ ├── conversation_entity_import.json │ │ │ │ ├── conversation_entity_without_confidence_import.json │ │ │ │ ├── custom_confusion_matrix_import.json │ │ │ │ ├── custom_scalar_import.json │ │ │ │ ├── image_import.json │ │ │ │ ├── image_import_name_only.json │ │ │ │ ├── metric_import.json │ │ │ │ ├── mmc_import.json │ │ │ │ ├── nested_import.json │ │ │ │ ├── nested_import_name_only.json │ │ │ │ ├── pdf_import.json │ │ │ │ ├── pdf_import_name_only.json │ │ │ │ ├── polyline_import.json │ │ │ │ ├── polyline_without_confidence_import.json │ │ │ │ ├── rectangle_import.json │ │ │ │ ├── relationship_import.json │ │ │ │ ├── text_entity_import.json │ │ │ │ ├── text_entity_without_confidence_import.json │ │ │ │ ├── video_import.json │ │ │ │ └── video_import_name_only.json │ │ │ ├── point.png │ │ │ ├── polygon.png │ │ │ ├── rectangle.png │ │ │ └── sample_text.txt │ │ ├── conftest.py │ │ ├── export │ │ │ ├── conftest.py │ │ │ └── streamable │ │ │ │ ├── test_export_data_rows_streamable.py │ │ │ │ ├── test_export_dataset_streamable.py │ │ │ │ ├── test_export_embeddings_streamable.py │ │ │ │ ├── test_export_model_run_streamable.py │ │ │ │ ├── test_export_project_streamable.py │ │ │ │ └── test_export_video_streamable.py │ │ ├── metrics │ │ │ ├── confusion_matrix │ │ │ │ ├── conftest.py │ │ │ │ ├── test_confusion_matrix_data_row.py │ │ │ │ └── test_confusion_matrix_feature.py │ │ │ └── iou │ │ │ │ ├── data_row │ │ │ │ └── conftest.py │ │ │ │ └── feature │ │ │ │ ├── conftest.py │ │ │ │ └── test_feature_iou.py │ │ ├── serialization │ │ │ ├── __init__.py │ │ │ └── ndjson │ │ │ │ ├── __init__.py │ │ │ │ ├── test_checklist.py │ │ │ │ ├── test_classification.py │ │ │ │ ├── test_conversation.py │ │ │ │ ├── test_data_gen.py │ │ │ │ ├── test_document.py │ │ │ │ ├── test_export_video_objects.py │ │ │ │ ├── test_free_text.py │ │ │ │ ├── test_generic_data_row_data.py │ │ │ │ ├── test_global_key.py │ │ │ │ ├── test_image.py │ │ │ │ ├── test_metric.py │ │ │ │ ├── test_mmc.py │ │ │ │ ├── test_nested.py │ │ │ │ ├── test_polyline.py │ │ │ │ ├── test_radio.py │ │ │ │ ├── test_rectangle.py │ │ │ │ ├── test_relationship.py │ │ │ │ ├── test_text.py │ │ │ │ ├── test_text_entity.py │ │ │ │ └── test_video.py │ │ ├── test_data_row_metadata.py │ │ └── test_prefetch_generator.py │ │ ├── integration │ │ ├── conftest.py │ │ ├── media │ │ │ ├── bulk_conversation.json │ │ │ ├── cat.mp4 │ │ │ ├── sample_image.jpg │ │ │ └── sample_pdf.pdf │ │ ├── schema │ │ │ └── test_user_group.py │ │ ├── snapshots │ │ │ ├── test_data_rows.test_data_row_bulk_creation_with_same_global_keys.failed_data_rows.json │ │ │ ├── test_task.test_task_errors.errors.json │ │ │ ├── test_task.test_task_errors.failed_data_rows.json │ │ │ └── test_task.test_task_success_json.json │ │ ├── test_api_keys.py │ │ ├── test_batch.py │ │ ├── test_batches.py │ │ ├── test_benchmark.py │ │ ├── test_chat_evaluation_ontology_project.py │ │ ├── test_client_errors.py │ │ ├── test_data_row_delete_metadata.py │ │ ├── test_data_rows.py │ │ ├── test_data_rows_upsert.py │ │ ├── test_data_upload.py │ │ ├── test_dataset.py │ │ ├── test_dates.py │ │ ├── test_delegated_access.py │ │ ├── test_embedding.py │ │ ├── test_ephemeral.py │ │ ├── test_feature_schema.py │ │ ├── test_filtering.py │ │ ├── test_foundry.py │ │ ├── test_global_keys.py │ │ ├── test_invite.py │ │ ├── test_label.py │ │ ├── test_labeling_dashboard.py │ │ ├── test_labeling_frontend.py │ │ ├── test_labeling_parameter_overrides.py │ │ ├── test_labeling_service.py │ │ ├── test_legacy_project.py │ │ ├── test_mmc_data_rows.py │ │ ├── test_model_config.py │ │ ├── test_offline_chat_evaluation_project.py │ │ ├── test_ontology.py │ │ ├── test_pagination.py │ │ ├── test_project.py │ │ ├── test_project_model_config.py │ │ ├── test_project_set_model_setup_complete.py │ │ ├── test_project_setup.py │ │ ├── test_prompt_response_generation_project.py │ │ ├── test_response_creation_project.py │ │ ├── test_send_to_annotate.py │ │ ├── test_slice.py │ │ ├── test_sorting.py │ │ ├── test_task.py │ │ ├── test_task_queue.py │ │ ├── test_toggle_mal.py │ │ ├── test_user_and_org.py │ │ ├── test_user_management.py │ │ └── test_webhook.py │ │ ├── unit │ │ ├── conftest.py │ │ ├── export_task │ │ │ └── test_export_task.py │ │ ├── schema │ │ │ ├── test_user_group.py │ │ │ └── test_user_group_v2.py │ │ ├── test_annotation_import.py │ │ ├── test_client.py │ │ ├── test_data_row_upsert_data.py │ │ ├── test_exceptions.py │ │ ├── test_label_data_type.py │ │ ├── test_labeling_service_dashboard.py │ │ ├── test_mal_import.py │ │ ├── test_ndjson_parsing.py │ │ ├── test_project.py │ │ ├── test_unit_case_change.py │ │ ├── test_unit_delete_batch_data_row_metadata.py │ │ ├── test_unit_descriptor_file_creator.py │ │ ├── test_unit_entity_meta.py │ │ ├── test_unit_export_filters.py │ │ ├── test_unit_fact_checking_tool.py │ │ ├── test_unit_filter.py │ │ ├── test_unit_identifiables.py │ │ ├── test_unit_label_import.py │ │ ├── test_unit_ontology.py │ │ ├── test_unit_ontology_kind.py │ │ ├── test_unit_project_validate_labeling_parameter_overrides.py │ │ ├── test_unit_prompt_issue_tool.py │ │ ├── test_unit_query.py │ │ ├── test_unit_rand_gen.py │ │ ├── test_unit_search_filters.py │ │ ├── test_unit_step_ontology_variants.py │ │ ├── test_unit_step_reasoning_tool.py │ │ ├── test_unit_webhook.py │ │ └── test_utils.py │ │ └── utils.py ├── lbox-clients │ ├── .gitignore │ ├── .python-version │ ├── Dockerfile │ ├── README.md │ ├── pyproject.toml │ ├── src │ │ └── lbox │ │ │ ├── call_info.py │ │ │ ├── exceptions.py │ │ │ └── request_client.py │ └── tests │ │ └── unit │ │ └── lbox │ │ └── test_client.py └── lbox-example │ ├── .gitignore │ ├── Dockerfile │ ├── README.md │ ├── pyproject.toml │ ├── src │ └── lbox │ │ └── example │ │ └── art.py │ └── tests │ └── unit │ └── lbox │ └── example │ └── test_art.py ├── pyproject.toml ├── requirements-dev.lock └── requirements.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.py[cod] 3 | *$py.class 4 | .Python 5 | env 6 | pip-log.txt 7 | pip-delete-this-directory.txt 8 | .tox 9 | .coverage 10 | .coverage.* 11 | .cache 12 | nosetests.xml 13 | coverage.xml 14 | *.cover 15 | *.log 16 | .git 17 | .mypy_cache 18 | .pytest_cache 19 | .hypothesis -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # SDK team's approval is required for all code changes (excluding notebook changes) 2 | * @labelbox/engineers 3 | *.ipynb -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Report a bug 4 | title: '' 5 | labels: 'bug-report' 6 | assignees: '' 7 | --- 8 | 9 | # Describe the bug 10 | A clear and concise description of what the bug is. 11 | 12 | # To reproduce 13 | Steps to reproduce the behavior: 14 | 1. Use method '...' 15 | 2. Pass parameters '...' 16 | 3. See error 17 | 18 | # Expected behavior 19 | A clear and concise description of what you expected to happen. 20 | 21 | # Screenshots / Videos 22 | If applicable, add screenshots or videos to help explain your problem. 23 | 24 | # Environment (please complete the following information 25 | - OS: [e.g. iOS] 26 | - Python Version: [e.g. 3.8.12] 27 | - SDK + SDK Version: [e.g. labelbox@3.70.0 / labelbox[data]@3.71.0] 28 | - Python Environment [e.g. pip freeze > requirements.txt] 29 | 30 | # Additional context 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: New Feature Request 3 | about: Request a new feature 4 | title: '' 5 | labels: 'feature-request' 6 | assignees: '' 7 | --- 8 | 9 | # Is your feature request related to a problem? Please describe. 10 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 11 | 12 | # Describe the solution you'd like 13 | A clear and concise description of what you want to happen. 14 | 15 | # Describe alternatives you've considered 16 | A clear and concise description of any alternative solutions or features you've considered. 17 | 18 | # Additional context 19 | Add any other context or screenshots about the feature request here. -------------------------------------------------------------------------------- /.github/actions/lbox-matrix/action.yml: -------------------------------------------------------------------------------- 1 | name: 'lbox-matrix' 2 | description: 'Generate running matrix for an lbox package' 3 | inputs: 4 | files-changed: 5 | description: 'Files changed as json array' 6 | required: true 7 | default: '[]' 8 | outputs: 9 | matrix: 10 | description: 'Matrix to run lbox packages tests against' 11 | runs: 12 | using: 'node20' 13 | main: 'index.js' -------------------------------------------------------------------------------- /.github/actions/provenance/action.yml: -------------------------------------------------------------------------------- 1 | name: Labelbox Python SDK Provenance Generation 2 | 3 | inputs: 4 | subjects: 5 | required: true 6 | type: string 7 | runs: 8 | using: "composite" 9 | steps: 10 | - name: upload 11 | uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 12 | with: 13 | base64-subjects: "${{ inputs.subjects }}" 14 | upload-assets: true 15 | upload-tag-name: v.6.0.0 # Tag from the initiation of the workflow -------------------------------------------------------------------------------- /.github/actions/python-package-shared-setup/action.yml: -------------------------------------------------------------------------------- 1 | name: Labelbox Python SDK Shared Setup 2 | 3 | inputs: 4 | rye-version: 5 | required: true 6 | python-version: 7 | required: true 8 | 9 | runs: 10 | using: "composite" 11 | steps: 12 | - name: Install the latest version of rye 13 | uses: eifinger/setup-rye@v2 14 | with: 15 | version: ${{ inputs.rye-version }} 16 | enable-cache: true 17 | - name: Rye Setup 18 | shell: bash 19 | run: | 20 | rye config --set-bool behavior.use-uv=true 21 | - name: Python setup 22 | shell: bash 23 | run: rye pin ${{ inputs.python-version }} 24 | - name: Environment setup 25 | working-directory: libs/labelbox 26 | shell: bash 27 | run: | 28 | rye sync -f --update-all -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Description 2 | 3 | Please include a summary of the changes and the related issue. Please also include relevant motivation and context. 4 | 5 | Fixes # (issue) 6 | 7 | ## Type of change 8 | 9 | Please delete options that are not relevant. 10 | 11 | - [ ] Bug fix (non-breaking change which fixes an issue) 12 | - [ ] New feature (non-breaking change which adds functionality) 13 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 14 | - [ ] Document change (fix typo or modifying any markdown files, code comments or anything in the examples folder only) 15 | 16 | ## All Submissions 17 | 18 | - [ ] Have you followed the guidelines in our Contributing document? 19 | - [ ] Have you provided a description? 20 | - [ ] Are your changes properly formatted? 21 | 22 | ## New Feature Submissions 23 | 24 | - [ ] Does your submission pass tests? 25 | - [ ] Have you added thorough tests for your new feature? 26 | - [ ] Have you commented your code, particularly in hard-to-understand areas? 27 | - [ ] Have you added a Docstring? 28 | 29 | ## Changes to Core Features 30 | 31 | - [ ] Have you written new tests for your core changes, as applicable? 32 | - [ ] Have you successfully run tests with your changes locally? 33 | - [ ] Have you updated any code comments, as applicable? 34 | -------------------------------------------------------------------------------- /.github/workflows/python-package-prod.yml: -------------------------------------------------------------------------------- 1 | name: Labelbox Python SDK Prod (Develop) 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | concurrency: 7 | group: ${{ github.workflow }}-${{ github.ref }} 8 | cancel-in-progress: true 9 | 10 | jobs: 11 | build: 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | include: 16 | - python-version: 3.9 17 | api-key: PROD_LABELBOX_API_KEY_3 18 | da-test-key: DA_GCP_LABELBOX_API_KEY 19 | - python-version: "3.10" 20 | api-key: PROD_LABELBOX_API_KEY_4 21 | da-test-key: DA_GCP_LABELBOX_API_KEY 22 | - python-version: 3.11 23 | api-key: LABELBOX_API_KEY 24 | da-test-key: DA_GCP_LABELBOX_API_KEY 25 | - python-version: 3.12 26 | api-key: PROD_LABELBOX_API_KEY_5 27 | da-test-key: DA_GCP_LABELBOX_API_KEY 28 | uses: ./.github/workflows/python-package-shared.yml 29 | with: 30 | python-version: ${{ matrix.python-version }} 31 | api-key: ${{ matrix.api-key }} 32 | da-test-key: ${{ matrix.da-test-key }} 33 | fixture-profile: false 34 | test-env: 'prod' 35 | secrets: inherit -------------------------------------------------------------------------------- /.github/workflows/secrets_scan.yml: -------------------------------------------------------------------------------- 1 | name: Secret_Scan 2 | 3 | on: 4 | pull_request: 5 | 6 | jobs: 7 | TruffleHog: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout code 11 | uses: actions/checkout@v4 12 | with: 13 | fetch-depth: 0 14 | - name: Secret Scanning 15 | uses: trufflesecurity/trufflehog@main 16 | with: 17 | extra_args: --only-verified 18 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.9.18 2 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Read the docs config file version (Required) 6 | version: 2 7 | 8 | # https://github.com/wemake-services/wemake-django-template/blob/master/.readthedocs.yml 9 | build: 10 | os: ubuntu-lts-latest 11 | commands: 12 | - asdf plugin add rye 13 | - asdf install rye latest 14 | - asdf global rye latest 15 | - rye config --set-bool behavior.global-python=true 16 | - rye config --set-bool behavior.use-uv=true 17 | - rye sync 18 | - rye run sphinx-build ./docs $READTHEDOCS_OUTPUT/html 19 | 20 | # Build documentation in the docs/ directory with Sphinx 21 | sphinx: 22 | configuration: docs/conf.py 23 | 24 | # Not building additional formats, as PDF build is failing 25 | # formats: 26 | # - pdf -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security 2 | 3 | ## Reporting a Vulnerability 4 | 5 | Please contact security@labelbox.com for any vulnerability that needs to be reported 6 | with labelbox-python. We will get back to you as soon as we can. 7 | -------------------------------------------------------------------------------- /docs/labelbox/annotation-import.rst: -------------------------------------------------------------------------------- 1 | Annotation Import 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.annotation_import 5 | :members: 6 | :show-inheritance: 7 | 8 | -------------------------------------------------------------------------------- /docs/labelbox/app.rst: -------------------------------------------------------------------------------- 1 | App 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.foundry.app 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/asset-attachment.rst: -------------------------------------------------------------------------------- 1 | Asset Attachment 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.asset_attachment 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/batch.rst: -------------------------------------------------------------------------------- 1 | Batch 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.batch 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/benchmark.rst: -------------------------------------------------------------------------------- 1 | Benchmark 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.benchmark 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/client.rst: -------------------------------------------------------------------------------- 1 | Client 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.client 5 | :members: 6 | :special-members: __init__ 7 | :exclude-members: upload_data, upload_file 8 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/conflict-resolution-strategy.rst: -------------------------------------------------------------------------------- 1 | Conflict Resolution Strategy 2 | =============================================================================================== 3 | .. automodule:: labelbox.schema.conflict_resolution_strategy 4 | :members: 5 | :show-inheritance: 6 | -------------------------------------------------------------------------------- /docs/labelbox/datarow-metadata.rst: -------------------------------------------------------------------------------- 1 | DataRow Metadata 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.data_row_metadata 5 | :members: 6 | :exclude-members: _DeleteBatchDataRowMetadata 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/labelbox/datarow.rst: -------------------------------------------------------------------------------- 1 | DataRow 2 | =============================================================================================== 3 | .. automodule:: labelbox.schema.data_row 4 | :members: 5 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/datarow_payload_templates.rst: -------------------------------------------------------------------------------- 1 | Datarow payload templates 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.data_row_payload_templates 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/dataset.rst: -------------------------------------------------------------------------------- 1 | Dataset 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.dataset 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/embedding.rst: -------------------------------------------------------------------------------- 1 | Embedding 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.embedding 5 | :members: 6 | :exclude-members: EmbeddingVector 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/labelbox/enums.rst: -------------------------------------------------------------------------------- 1 | Enums 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.enums 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/exceptions.rst: -------------------------------------------------------------------------------- 1 | Exceptions 2 | =============================================================================================== 3 | 4 | .. automodule:: lbox.exceptions 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/export-task.rst: -------------------------------------------------------------------------------- 1 | Export Task 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.export_task 5 | :members: 6 | :exclude-members: FileRetrieverByLine, FileRetrieverByOffset, FileRetrieverStrategy, Range, Converter 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/labelbox/fact-checking-tool.rst: -------------------------------------------------------------------------------- 1 | Fact Checking Tool 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.tool_building.fact_checking_tool 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/foundry-client.rst: -------------------------------------------------------------------------------- 1 | Foundry Client 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.foundry.foundry_client 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/foundry-model.rst: -------------------------------------------------------------------------------- 1 | Foundry Model 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.foundry.model 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/identifiable.rst: -------------------------------------------------------------------------------- 1 | Identifiable 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.identifiable 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/identifiables.rst: -------------------------------------------------------------------------------- 1 | Identifiables 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.identifiables 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/index.rst: -------------------------------------------------------------------------------- 1 | Labelbox Python SDK Documentation 2 | =============================================================================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 3 6 | :caption: Contents: 7 | 8 | annotation-import 9 | app 10 | asset-attachment 11 | batch 12 | benchmark 13 | client 14 | conflict-resolution-strategy 15 | datarow 16 | datarow-metadata 17 | dataset 18 | embedding 19 | enums 20 | exceptions 21 | export-task 22 | fact-checking-tool 23 | foundry-client 24 | foundry-model 25 | identifiable 26 | identifiables 27 | label 28 | label-score 29 | labeling-frontend 30 | labeling-frontend-options 31 | labeling-parameter-override 32 | labeling-service 33 | labeling-service-dashboard 34 | model 35 | model-config 36 | model-run 37 | ontology 38 | ontology_kind 39 | organization 40 | pagination 41 | project 42 | project-model-config 43 | prompt-issue-tool 44 | quality-mode 45 | request-client 46 | resource-tag 47 | review 48 | search-filters 49 | send-to-annotate-params 50 | slice 51 | step-reasoning-tool 52 | task 53 | task-queue 54 | user 55 | user-group-v2 56 | webhook 57 | -------------------------------------------------------------------------------- /docs/labelbox/label-score.rst: -------------------------------------------------------------------------------- 1 | Label Score 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.label_score 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/label.rst: -------------------------------------------------------------------------------- 1 | Label 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.label 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/labeling-frontend-options.rst: -------------------------------------------------------------------------------- 1 | Labeling Frontend Options 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.labeling_frontend 5 | :members: LabelingFrontendOptions 6 | :exclude-members: LabelingFrontend 7 | :show-inheritance: 8 | :noindex: -------------------------------------------------------------------------------- /docs/labelbox/labeling-frontend.rst: -------------------------------------------------------------------------------- 1 | Labeling Frontend 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.labeling_frontend 5 | :members: LabelingFrontend 6 | :exclude-members: LabelingFrontendOptions 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/labeling-parameter-override.rst: -------------------------------------------------------------------------------- 1 | Labeling Parameter Override 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.project 5 | :members: LabelingParameterOverride 6 | :show-inheritance: 7 | :noindex: -------------------------------------------------------------------------------- /docs/labelbox/labeling-service-dashboard.rst: -------------------------------------------------------------------------------- 1 | Labeling Service Dashboard 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.labeling_service_dashboard 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/labeling-service-status.rst: -------------------------------------------------------------------------------- 1 | Labeling Service Status 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.labeling_service_status 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/labeling-service.rst: -------------------------------------------------------------------------------- 1 | Labeling Service 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.labeling_service 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/model-config.rst: -------------------------------------------------------------------------------- 1 | Model Config 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.model_config 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/model-run.rst: -------------------------------------------------------------------------------- 1 | Model Run 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.model_run 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/model.rst: -------------------------------------------------------------------------------- 1 | Model 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.model 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/ontology-kind.rst: -------------------------------------------------------------------------------- 1 | OntologyKind 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.ontology_kind 5 | :members: 6 | :exclude-members: EditorTaskType 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/ontology.rst: -------------------------------------------------------------------------------- 1 | Ontology 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.ontology 5 | :members: 6 | :exclude-members: OntologyEntity, Classification, Tool, Option 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/organization.rst: -------------------------------------------------------------------------------- 1 | Organization 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.organization 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/pagination.rst: -------------------------------------------------------------------------------- 1 | Pagination 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.pagination 5 | :members: 6 | :special-members: __init__ 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/project-model-config.rst: -------------------------------------------------------------------------------- 1 | Project Model Config 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.project_model_config 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/project-overview.rst: -------------------------------------------------------------------------------- 1 | Project Overview 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.project-overview 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/project.rst: -------------------------------------------------------------------------------- 1 | Project 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.project 5 | :members: 6 | :exclude-members: LabelerPerformance, LabelingParameterOverride 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/prompt-issue-tool.rst: -------------------------------------------------------------------------------- 1 | Step Reasoning Tool 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.tool_building.prompt_issue_tool 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/quality-mode.rst: -------------------------------------------------------------------------------- 1 | Quality Mode 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.quality_mode 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/request-client.rst: -------------------------------------------------------------------------------- 1 | Request Client 2 | =============================================================================================== 3 | 4 | .. automodule:: lbox.request_client 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/resource-tag.rst: -------------------------------------------------------------------------------- 1 | Resource Tag 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.resource_tag 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/review.rst: -------------------------------------------------------------------------------- 1 | Review 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.review 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/search-filters.rst: -------------------------------------------------------------------------------- 1 | Search Filters 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.search_filters 5 | :members: 6 | :exclude-members: _dict_to_graphql_string 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /docs/labelbox/send-to-annotate-params.rst: -------------------------------------------------------------------------------- 1 | Send To Annotate Params 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.send_to_annotate_params 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/slice.rst: -------------------------------------------------------------------------------- 1 | Slice 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.slice 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/step-reasoning-tool.rst: -------------------------------------------------------------------------------- 1 | Step Reasoning Tool 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.tool_building.step_reasoning_tool 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/task-queue.rst: -------------------------------------------------------------------------------- 1 | Task Queue 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.task_queue 5 | :members: 6 | :show-inheritance: 7 | -------------------------------------------------------------------------------- /docs/labelbox/task.rst: -------------------------------------------------------------------------------- 1 | Task 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.task 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/user-group-v2.rst: -------------------------------------------------------------------------------- 1 | User Group 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.user_group_v2 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/user-group.rst: -------------------------------------------------------------------------------- 1 | User Group 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.user_group 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/user.rst: -------------------------------------------------------------------------------- 1 | User 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.user 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /docs/labelbox/webhook.rst: -------------------------------------------------------------------------------- 1 | Webhook 2 | =============================================================================================== 3 | 4 | .. automodule:: labelbox.schema.webhook 5 | :members: 6 | :show-inheritance: -------------------------------------------------------------------------------- /examples/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "examples" 3 | version = "0.1.0" 4 | description = "Labelbox Python Example Notebooks" 5 | authors = [{ name = "Labelbox", email = "docs@labelbox.com" }] 6 | readme = "README.md" 7 | # Python version matches labelbox SDK 8 | requires-python = ">=3.9" 9 | dependencies = [] 10 | 11 | [project.urls] 12 | Homepage = "https://labelbox.com/" 13 | Documentation = "https://docs.labelbox.com/" 14 | Tutorials = "https://docs.labelbox.com/page/tutorials" 15 | Repository = "https://github.com/Labelbox/labelbox-python/examples" 16 | Issues = "https://github.com/Labelbox/labelbox-python/issues" 17 | 18 | [tool.rye] 19 | managed = true 20 | virtual = true 21 | dev-dependencies = [ 22 | "yapf>=0.40.2", 23 | "black[jupyter]>=24.4.2", 24 | "databooks>=1.3.10", 25 | # higher versions dont support python 3.9 26 | "pandas>=2.2.3", 27 | ] 28 | 29 | [tool.rye.scripts] 30 | clean = { chain = ["clean:black", "clean:format", "clean:databooks"] } 31 | "clean:databooks" = 'databooks meta --rm-outs . --yes' 32 | "clean:black" = 'black --line-length 80 . --quiet' 33 | "clean:format" = { call = "scripts.format_notebooks" } 34 | "create-readme" = { call = "scripts.generate_readme:main(True)" } 35 | "create-doc-readme" = { call = "scripts.generate_readme:main(False)" } 36 | -------------------------------------------------------------------------------- /libs/labelbox/.yapfignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | *.txt 3 | *.ini 4 | *.lock -------------------------------------------------------------------------------- /libs/labelbox/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/ucyo/python-package-template/blob/master/Dockerfile 2 | FROM python:3.9-slim as rye 3 | 4 | ENV LANG="C.UTF-8" \ 5 | LC_ALL="C.UTF-8" \ 6 | PATH="/home/python/.local/bin:/home/python/.rye/shims:$PATH" \ 7 | PIP_NO_CACHE_DIR="false" \ 8 | RYE_VERSION="0.43.0" \ 9 | RYE_INSTALL_OPTION="--yes" \ 10 | LABELBOX_TEST_ENVIRON="prod" 11 | 12 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 13 | ca-certificates \ 14 | curl \ 15 | inotify-tools \ 16 | make \ 17 | # cv2 18 | libsm6 \ 19 | libxext6 \ 20 | ffmpeg \ 21 | libfontconfig1 \ 22 | libxrender1 \ 23 | libgl1-mesa-glx \ 24 | libgeos-dev \ 25 | gcc \ 26 | && rm -rf /var/lib/apt/lists/* 27 | 28 | RUN groupadd --gid 1000 python && \ 29 | useradd --uid 1000 --gid python --shell /bin/bash --create-home python 30 | 31 | USER 1000 32 | WORKDIR /home/python/ 33 | 34 | RUN curl -sSf https://rye.astral.sh/get | bash - 35 | 36 | COPY --chown=python:python . /home/python/labelbox-python/ 37 | WORKDIR /home/python/labelbox-python 38 | 39 | RUN rye config --set-bool behavior.global-python=true && \ 40 | rye config --set-bool behavior.use-uv=true && \ 41 | rye pin 3.9 && \ 42 | rye sync 43 | 44 | CMD cd libs/labelbox && rye run integration && rye sync -f --features labelbox/data && rye run unit && rye run data 45 | -------------------------------------------------------------------------------- /libs/labelbox/mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | 3 | [mypy-google.*] 4 | ignore_missing_imports = True 5 | 6 | [mypy-labelbox.data.*] 7 | ignore_errors = True 8 | 9 | [mypy-labelbox] 10 | ignore_errors = True 11 | 12 | [mypy-lbox.exceptions] 13 | ignore_missing_imports = True 14 | 15 | [mypy-lbox.call_info] 16 | ignore_missing_imports = True 17 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/annotated_types.py: -------------------------------------------------------------------------------- 1 | from typing import Annotated 2 | 3 | from pydantic import Field 4 | 5 | Cuid = Annotated[str, Field(min_length=25, max_length=25)] 6 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/src/labelbox/data/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/__init__.py: -------------------------------------------------------------------------------- 1 | from .geometry import Line 2 | from .geometry import Point 3 | from .geometry import Mask 4 | from .geometry import Polygon 5 | from .geometry import Rectangle 6 | from .geometry import Geometry 7 | from .geometry import DocumentRectangle 8 | from .geometry import RectangleUnit 9 | 10 | from .annotation import ClassificationAnnotation 11 | from .annotation import ObjectAnnotation 12 | 13 | from .relationship import RelationshipAnnotation 14 | from .relationship import Relationship 15 | 16 | from .video import VideoClassificationAnnotation 17 | from .video import VideoObjectAnnotation 18 | from .video import MaskFrame 19 | from .video import MaskInstance 20 | from .video import VideoMaskAnnotation 21 | 22 | from .ner import ConversationEntity 23 | from .ner import DocumentEntity 24 | from .ner import DocumentTextSelection 25 | from .ner import TextEntity 26 | 27 | from .classification import Checklist 28 | from .classification import ClassificationAnswer 29 | from .classification import Radio 30 | from .classification import Text 31 | 32 | from .data import GenericDataRowData 33 | from .data import MaskData 34 | 35 | from .label import Label 36 | from .collection import LabelGenerator 37 | 38 | from .metrics import ScalarMetric 39 | from .metrics import ScalarMetricAggregation 40 | from .metrics import ConfusionMatrixMetric 41 | from .metrics import ConfusionMatrixAggregation 42 | from .metrics import ScalarMetricValue 43 | from .metrics import ConfusionMatrixMetricValue 44 | 45 | from .data.tiled_image import EPSG 46 | from .data.tiled_image import EPSGTransformer 47 | from .data.tiled_image import TiledBounds 48 | from .data.tiled_image import TiledImageData 49 | from .data.tiled_image import TileLayer 50 | 51 | from .llm_prompt_response.prompt import PromptText 52 | from .llm_prompt_response.prompt import PromptClassificationAnnotation 53 | 54 | from .mmc import ( 55 | MessageInfo, 56 | OrderedMessageInfo, 57 | MessageSingleSelectionTask, 58 | MessageMultiSelectionTask, 59 | MessageRankingTask, 60 | MessageEvaluationTaskAnnotation, 61 | ) 62 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/annotation.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union 2 | 3 | from labelbox.data.annotation_types.base_annotation import BaseAnnotation 4 | from labelbox.data.annotation_types.geometry.geometry import Geometry 5 | 6 | from labelbox.data.mixins import ConfidenceMixin, CustomMetricsMixin 7 | 8 | from labelbox.data.annotation_types.classification.classification import ( 9 | ClassificationAnnotation, 10 | ) 11 | from .ner import DocumentEntity, TextEntity, ConversationEntity 12 | from typing import Optional 13 | 14 | 15 | class ObjectAnnotation(BaseAnnotation, ConfidenceMixin, CustomMetricsMixin): 16 | """Generic localized annotation (non classifications) 17 | 18 | >>> ObjectAnnotation( 19 | >>> value=Rectangle( 20 | >>> start=Point(x=0, y=0), 21 | >>> end=Point(x=1, y=1) 22 | >>> ), 23 | >>> feature_schema_id="my-feature-schema-id" 24 | >>> ) 25 | 26 | Args: 27 | name (Optional[str]) 28 | feature_schema_id (Optional[Cuid]) 29 | value (Union[TextEntity, Geometry]): Localization of the annotation 30 | classifications (Optional[List[ClassificationAnnotation]]): Optional sub classification of the annotation 31 | extra (Dict[str, Any]) 32 | """ 33 | 34 | value: Union[TextEntity, ConversationEntity, DocumentEntity, Geometry] 35 | classifications: Optional[List[ClassificationAnnotation]] = [] 36 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/base_annotation.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from uuid import UUID, uuid4 3 | from typing import Any, Dict, Optional 4 | 5 | from .feature import FeatureSchema 6 | from pydantic import PrivateAttr, ConfigDict 7 | 8 | 9 | class BaseAnnotation(FeatureSchema, abc.ABC): 10 | """Base annotation class. Shouldn't be directly instantiated""" 11 | 12 | _uuid: Optional[UUID] = PrivateAttr() 13 | extra: Dict[str, Any] = {} 14 | 15 | model_config = ConfigDict(extra="allow") 16 | 17 | def __init__(self, **data): 18 | super().__init__(**data) 19 | extra_uuid = data.get("extra", {}).get("uuid") 20 | self._uuid = data.get("_uuid") or extra_uuid or uuid4() 21 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/classification/__init__.py: -------------------------------------------------------------------------------- 1 | from .classification import Checklist, ClassificationAnswer, Radio, Text 2 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/data/__init__.py: -------------------------------------------------------------------------------- 1 | from .raster import MaskData 2 | from .generic_data_row_data import GenericDataRowData 3 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/data/base_data.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from typing import Optional, Dict, List, Any 3 | 4 | from pydantic import BaseModel 5 | 6 | 7 | class BaseData(BaseModel, ABC): 8 | """ 9 | Base class for objects representing data. 10 | This class shouldn't directly be used 11 | """ 12 | 13 | external_id: Optional[str] = None 14 | uid: Optional[str] = None 15 | global_key: Optional[str] = None 16 | media_attributes: Optional[Dict[str, Any]] = None 17 | metadata: Optional[List[Dict[str, Any]]] = None 18 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/data/generic_data_row_data.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Literal, Optional 2 | 3 | from labelbox.data.annotation_types.data.base_data import BaseData 4 | from labelbox.utils import _NoCoercionMixin 5 | from pydantic import model_validator 6 | 7 | 8 | class GenericDataRowData(BaseData, _NoCoercionMixin): 9 | """Generic data row data. This is replacing all other DataType passed into Label""" 10 | 11 | url: Optional[str] = None 12 | class_name: Literal["GenericDataRowData"] = "GenericDataRowData" 13 | 14 | def create_url(self, signer: Callable[[bytes], str]) -> Optional[str]: 15 | return self.url 16 | 17 | @model_validator(mode="before") 18 | @classmethod 19 | def validate_one_datarow_key_present(cls, data): 20 | keys = ["external_id", "global_key", "uid"] 21 | count = sum([key in data for key in keys]) 22 | 23 | if count < 1: 24 | raise ValueError(f"Exactly one of {keys} must be present.") 25 | if count > 1: 26 | raise ValueError(f"Only one of {keys} can be present.") 27 | return data 28 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/feature.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from pydantic import BaseModel, model_validator 3 | 4 | from ...annotated_types import Cuid 5 | 6 | 7 | class FeatureSchema(BaseModel): 8 | """ 9 | Class that represents a feature schema. 10 | Could be a annotation, a subclass, or an option. 11 | Schema ids might not be known when constructing these objects so both a name and schema id are valid. 12 | """ 13 | 14 | name: Optional[str] = None 15 | feature_schema_id: Optional[Cuid] = None 16 | 17 | @model_validator(mode="after") 18 | def must_set_one(self): 19 | if self.feature_schema_id is None and self.name is None: 20 | raise ValueError( 21 | "Must set either feature_schema_id or name for all feature schemas" 22 | ) 23 | return self 24 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/geometry/__init__.py: -------------------------------------------------------------------------------- 1 | from .line import Line 2 | from .point import Point 3 | from .mask import Mask 4 | from .polygon import Polygon 5 | from .rectangle import Rectangle 6 | from .rectangle import DocumentRectangle 7 | from .rectangle import RectangleUnit 8 | from .geometry import Geometry 9 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/geometry/geometry.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any, Optional, Union, Tuple 2 | from abc import ABC, abstractmethod 3 | 4 | import geojson 5 | import numpy as np 6 | 7 | from shapely import geometry as geom 8 | from pydantic import BaseModel 9 | 10 | 11 | class Geometry(BaseModel, ABC): 12 | """Abstract base class for geometry objects""" 13 | 14 | extra: Dict[str, Any] = {} 15 | 16 | @property 17 | def shapely( 18 | self, 19 | ) -> Union[ 20 | geom.Point, 21 | geom.LineString, 22 | geom.Polygon, 23 | geom.MultiPoint, 24 | geom.MultiLineString, 25 | geom.MultiPolygon, 26 | ]: 27 | return geom.shape(self.geometry) 28 | 29 | def get_or_create_canvas( 30 | self, 31 | height: Optional[int], 32 | width: Optional[int], 33 | canvas: Optional[np.ndarray], 34 | ) -> np.ndarray: 35 | if canvas is None: 36 | if height is None or width is None: 37 | raise ValueError( 38 | "Must either provide canvas or height and width" 39 | ) 40 | canvas = np.zeros((height, width, 3), dtype=np.uint8) 41 | canvas = np.ascontiguousarray(canvas) 42 | return canvas 43 | 44 | @property 45 | @abstractmethod 46 | def geometry(self) -> geojson: 47 | pass 48 | 49 | @abstractmethod 50 | def draw( 51 | self, 52 | height: Optional[int] = None, 53 | width: Optional[int] = None, 54 | canvas: Optional[np.ndarray] = None, 55 | color: Optional[Union[int, Tuple[int, int, int]]] = None, 56 | thickness: Optional[int] = 1, 57 | ) -> np.ndarray: 58 | pass 59 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/geometry/point.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Tuple, Union 2 | 3 | import geojson 4 | import numpy as np 5 | import cv2 6 | from shapely.geometry import Point as SPoint 7 | 8 | from .geometry import Geometry 9 | 10 | 11 | class Point(Geometry): 12 | """Point geometry 13 | 14 | >>> Point(x=0, y=0) 15 | 16 | Args: 17 | x (float) 18 | y (float) 19 | 20 | """ 21 | 22 | x: float 23 | y: float 24 | 25 | @property 26 | def geometry(self) -> geojson.Point: 27 | return geojson.Point((self.x, self.y)) 28 | 29 | @classmethod 30 | def from_shapely(cls, shapely_obj: SPoint) -> "Point": 31 | """Transforms a shapely object.""" 32 | if not isinstance(shapely_obj, SPoint): 33 | raise TypeError( 34 | f"Expected Shapely Point. Got {shapely_obj.geom_type}" 35 | ) 36 | 37 | obj_coords = shapely_obj.__geo_interface__["coordinates"] 38 | return Point(x=obj_coords[0], y=obj_coords[1]) 39 | 40 | def draw( 41 | self, 42 | height: Optional[int] = None, 43 | width: Optional[int] = None, 44 | canvas: Optional[np.ndarray] = None, 45 | color: Union[int, Tuple[int, int, int]] = (255, 255, 255), 46 | thickness: int = 10, 47 | ) -> np.ndarray: 48 | """ 49 | Draw the point onto a 3d mask 50 | Args: 51 | height (int): height of the mask 52 | width (int): width of the mask 53 | thickness (int): pixel radius of the point 54 | color (int): color for the point. 55 | RGB values by default but if a 2D canvas is provided this can set this to an int. 56 | canvas (np.ndarray): Canvas to draw the point on 57 | Returns: 58 | numpy array representing the mask with the point drawn on it. 59 | """ 60 | canvas = self.get_or_create_canvas(height, width, canvas) 61 | return cv2.circle( 62 | canvas, 63 | (int(self.x), int(self.y)), 64 | radius=thickness, 65 | color=color, 66 | thickness=-1, 67 | ) 68 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/llm_prompt_response/__init__.py: -------------------------------------------------------------------------------- 1 | from .prompt import PromptText 2 | from .prompt import PromptClassificationAnnotation 3 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/llm_prompt_response/prompt.py: -------------------------------------------------------------------------------- 1 | from labelbox.data.annotation_types.base_annotation import BaseAnnotation 2 | from labelbox.data.mixins import ConfidenceMixin, CustomMetricsMixin 3 | from pydantic import BaseModel 4 | 5 | 6 | class PromptText(ConfidenceMixin, CustomMetricsMixin, BaseModel): 7 | """Prompt text for LLM data generation 8 | 9 | >>> PromptText(answer = "some text answer", 10 | >>> confidence = 0.5, 11 | >>> custom_metrics = [ 12 | >>> { 13 | >>> "name": "iou", 14 | >>> "value": 0.1 15 | >>> }]) 16 | """ 17 | 18 | answer: str 19 | 20 | 21 | class PromptClassificationAnnotation( 22 | BaseAnnotation, ConfidenceMixin, CustomMetricsMixin 23 | ): 24 | """Prompt annotation (non localized) 25 | 26 | >>> PromptClassificationAnnotation( 27 | >>> value=PromptText(answer="my caption message"), 28 | >>> feature_schema_id="my-feature-schema-id" 29 | >>> ) 30 | 31 | Args: 32 | name (Optional[str]) 33 | feature_schema_id (Optional[Cuid]) 34 | value (Union[Text]) 35 | """ 36 | 37 | value: PromptText 38 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .scalar import ScalarMetric, ScalarMetricAggregation, ScalarMetricValue 2 | from .confusion_matrix import ( 3 | ConfusionMatrixMetric, 4 | ConfusionMatrixAggregation, 5 | ConfusionMatrixMetricValue, 6 | ) 7 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/metrics/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from typing import Dict, Optional, Any, Union 3 | 4 | from pydantic import ( 5 | confloat, 6 | BaseModel, 7 | model_serializer, 8 | field_validator, 9 | ) 10 | 11 | ConfidenceValue = confloat(ge=0, le=1) 12 | 13 | MIN_CONFIDENCE_SCORES = 2 14 | MAX_CONFIDENCE_SCORES = 15 15 | 16 | 17 | class BaseMetric(BaseModel, ABC): 18 | value: Union[Any, Dict[float, Any]] 19 | feature_name: Optional[str] = None 20 | subclass_name: Optional[str] = None 21 | extra: Dict[str, Any] = {} 22 | 23 | @model_serializer(mode="wrap") 24 | def serialize_model(self, handler): 25 | res = handler(self) 26 | return {k: v for k, v in res.items() if v is not None} 27 | 28 | @field_validator("value") 29 | def validate_value(cls, value): 30 | if isinstance(value, Dict): 31 | if not ( 32 | MIN_CONFIDENCE_SCORES <= len(value) <= MAX_CONFIDENCE_SCORES 33 | ): 34 | raise ValueError( 35 | f"Number of confidence scores must be greater than\n \ 36 | or equal to {MIN_CONFIDENCE_SCORES} and less than\n \ 37 | or equal to {MAX_CONFIDENCE_SCORES}. Found {len(value)}" 38 | ) 39 | return value 40 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/metrics/confusion_matrix.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Optional, Tuple, Dict, Union 3 | 4 | from pydantic import conint 5 | 6 | from .base import ConfidenceValue, BaseMetric 7 | 8 | Count = conint(ge=0, le=1e10) 9 | 10 | ConfusionMatrixMetricValue = Tuple[Count, Count, Count, Count] 11 | ConfusionMatrixMetricConfidenceValue = Dict[ 12 | ConfidenceValue, ConfusionMatrixMetricValue 13 | ] 14 | 15 | 16 | class ConfusionMatrixAggregation(Enum): 17 | CONFUSION_MATRIX = "CONFUSION_MATRIX" 18 | 19 | 20 | class ConfusionMatrixMetric(BaseMetric): 21 | """Class representing confusion matrix metrics. 22 | 23 | In the editor, this provides precision, recall, and f-scores. 24 | This should be used over multiple scalar metrics so that aggregations are accurate. 25 | 26 | Value should be a tuple representing: 27 | [True Positive Count, False Positive Count, True Negative Count, False Negative Count] 28 | 29 | aggregation cannot be adjusted for confusion matrix metrics. 30 | """ 31 | 32 | metric_name: str 33 | value: Union[ 34 | ConfusionMatrixMetricValue, ConfusionMatrixMetricConfidenceValue 35 | ] 36 | aggregation: Optional[ConfusionMatrixAggregation] = ( 37 | ConfusionMatrixAggregation.CONFUSION_MATRIX 38 | ) 39 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/metrics/scalar.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Annotated, Dict, Optional, Union 3 | 4 | from pydantic import field_validator 5 | from pydantic.types import confloat 6 | 7 | from .base import BaseMetric, ConfidenceValue 8 | 9 | ScalarMetricValue = Annotated[float, confloat(ge=0, le=100_000_000)] 10 | ScalarMetricConfidenceValue = Dict[ConfidenceValue, ScalarMetricValue] 11 | 12 | 13 | class ScalarMetricAggregation(Enum): 14 | ARITHMETIC_MEAN = "ARITHMETIC_MEAN" 15 | GEOMETRIC_MEAN = "GEOMETRIC_MEAN" 16 | HARMONIC_MEAN = "HARMONIC_MEAN" 17 | SUM = "SUM" 18 | 19 | 20 | RESERVED_METRIC_NAMES = ( 21 | "true_positive_count", 22 | "false_positive_count", 23 | "true_negative_count", 24 | "false_negative_count", 25 | "precision", 26 | "recall", 27 | "f1", 28 | "iou", 29 | ) 30 | 31 | 32 | class ScalarMetric(BaseMetric): 33 | """Class representing scalar metrics 34 | 35 | For backwards compatibility, metric_name is optional. 36 | The metric_name will be set to a default name in the editor if it is not set. 37 | This is not recommended and support for empty metric_name fields will be removed. 38 | aggregation will be ignored without providing a metric name. 39 | """ 40 | 41 | metric_name: Optional[str] = None 42 | value: Union[ScalarMetricValue, ScalarMetricConfidenceValue] 43 | aggregation: Optional[ScalarMetricAggregation] = ( 44 | ScalarMetricAggregation.ARITHMETIC_MEAN 45 | ) 46 | 47 | @field_validator("metric_name") 48 | def validate_metric_name(cls, name: Union[str, None]): 49 | if name is None: 50 | return None 51 | clean_name = name.lower().strip() 52 | if clean_name in RESERVED_METRIC_NAMES: 53 | raise ValueError( 54 | f"`{clean_name}` is a reserved metric name. " 55 | "Please provide another value for `metric_name`." 56 | ) 57 | return name 58 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/mmc.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from typing import ClassVar, List, Union 3 | 4 | from pydantic import ConfigDict, field_validator 5 | 6 | from labelbox.utils import _CamelCaseMixin 7 | from labelbox.data.annotation_types.annotation import BaseAnnotation 8 | 9 | 10 | class MessageInfo(_CamelCaseMixin): 11 | message_id: str 12 | model_config_name: str 13 | 14 | model_config = ConfigDict(protected_namespaces=()) 15 | 16 | 17 | class OrderedMessageInfo(MessageInfo): 18 | order: int 19 | 20 | 21 | class _BaseMessageEvaluationTask(_CamelCaseMixin, ABC): 22 | format: ClassVar[str] 23 | parent_message_id: str 24 | 25 | model_config = ConfigDict(protected_namespaces=()) 26 | 27 | 28 | class MessageSingleSelectionTask(_BaseMessageEvaluationTask, MessageInfo): 29 | format: ClassVar[str] = "message-single-selection" 30 | 31 | 32 | class MessageMultiSelectionTask(_BaseMessageEvaluationTask): 33 | format: ClassVar[str] = "message-multi-selection" 34 | selected_messages: List[MessageInfo] 35 | 36 | 37 | class MessageRankingTask(_BaseMessageEvaluationTask): 38 | format: ClassVar[str] = "message-ranking" 39 | ranked_messages: List[OrderedMessageInfo] 40 | 41 | @field_validator("ranked_messages") 42 | def _validate_ranked_messages(cls, v: List[OrderedMessageInfo]): 43 | if not {msg.order for msg in v} == set(range(1, len(v) + 1)): 44 | raise ValueError( 45 | "Messages must be ordered by unique and consecutive natural numbers starting from 1" 46 | ) 47 | return v 48 | 49 | 50 | class MessageEvaluationTaskAnnotation(BaseAnnotation): 51 | value: Union[ 52 | MessageSingleSelectionTask, 53 | MessageMultiSelectionTask, 54 | MessageRankingTask, 55 | ] 56 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/ner/__init__.py: -------------------------------------------------------------------------------- 1 | from .conversation_entity import ConversationEntity 2 | from .document_entity import DocumentEntity, DocumentTextSelection 3 | from .text_entity import TextEntity 4 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/ner/conversation_entity.py: -------------------------------------------------------------------------------- 1 | from labelbox.data.annotation_types.ner.text_entity import TextEntity 2 | from labelbox.utils import _CamelCaseMixin 3 | 4 | 5 | class ConversationEntity(TextEntity, _CamelCaseMixin): 6 | """Represents a text entity""" 7 | 8 | message_id: str 9 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/ner/document_entity.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from labelbox.utils import _CamelCaseMixin 4 | from pydantic import BaseModel, field_validator 5 | 6 | 7 | class DocumentTextSelection(_CamelCaseMixin, BaseModel): 8 | token_ids: List[str] 9 | group_id: str 10 | page: int 11 | 12 | @field_validator("page") 13 | def validate_page(cls, v): 14 | if v < 1: 15 | raise ValueError("Page must be greater than 1") 16 | return v 17 | 18 | 19 | class DocumentEntity(_CamelCaseMixin, BaseModel): 20 | """Represents a text entity""" 21 | 22 | text_selections: List[DocumentTextSelection] 23 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/ner/text_entity.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any 2 | 3 | from pydantic import BaseModel, model_validator 4 | 5 | 6 | class TextEntity(BaseModel): 7 | """Represents a text entity""" 8 | 9 | start: int 10 | end: int 11 | extra: Dict[str, Any] = {} 12 | 13 | @model_validator(mode="after") 14 | def validate_start_end(self, values): 15 | if hasattr(self, "start") and hasattr(self, "end"): 16 | if isinstance(self.start, int) and self.start > self.end: 17 | raise ValueError( 18 | "Location end must be greater or equal to start" 19 | ) 20 | return self 21 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/relationship.py: -------------------------------------------------------------------------------- 1 | from typing import Union, Optional 2 | from pydantic import BaseModel, model_validator 3 | from enum import Enum 4 | import warnings 5 | from labelbox.data.annotation_types.annotation import ( 6 | BaseAnnotation, 7 | ObjectAnnotation, 8 | ClassificationAnnotation, 9 | ) 10 | 11 | 12 | class Relationship(BaseModel): 13 | class Type(Enum): 14 | UNIDIRECTIONAL = "unidirectional" 15 | BIDIRECTIONAL = "bidirectional" 16 | 17 | source: Optional[Union[ObjectAnnotation, ClassificationAnnotation]] = None 18 | source_ontology_name: Optional[str] = None 19 | target: ObjectAnnotation 20 | type: Type = Type.UNIDIRECTIONAL 21 | readonly: Optional[bool] = None 22 | 23 | @model_validator(mode="after") 24 | def check_readonly(self): 25 | if self.readonly is True: 26 | warnings.warn( 27 | "Creating a relationship with readonly=True is in beta and its behavior may change in future releases.", 28 | ) 29 | return self 30 | 31 | @model_validator(mode="after") 32 | def validate_source_fields(self): 33 | if self.source is None and self.source_ontology_name is None: 34 | raise ValueError( 35 | "Either source or source_ontology_name must be provided" 36 | ) 37 | return self 38 | 39 | @model_validator(mode="after") 40 | def validate_source_consistency(self): 41 | if self.source is not None and self.source_ontology_name is not None: 42 | raise ValueError( 43 | "Only one of 'source' or 'source_ontology_name' may be provided" 44 | ) 45 | return self 46 | 47 | 48 | class RelationshipAnnotation(BaseAnnotation): 49 | value: Relationship 50 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/annotation_types/types.py: -------------------------------------------------------------------------------- 1 | from typing import Generic, TypeVar 2 | import numpy as np 3 | from pydantic_core import core_schema 4 | 5 | DType = TypeVar("DType") 6 | DShape = TypeVar("DShape") 7 | 8 | 9 | class TypedArray(np.ndarray, Generic[DType, DShape]): 10 | @classmethod 11 | def __get_pydantic_core_schema__( 12 | cls, _source_type: type, _model: type 13 | ) -> core_schema.CoreSchema: 14 | return core_schema.no_info_plain_validator_function(cls.validate) 15 | 16 | @classmethod 17 | def validate(cls, val): 18 | if not isinstance(val, np.ndarray): 19 | raise TypeError(f"Expected numpy array. Found {type(val)}") 20 | return val 21 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .confusion_matrix import ( 2 | confusion_matrix_metric, 3 | feature_confusion_matrix_metric, 4 | ) 5 | from .iou import miou_metric, feature_miou_metric 6 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/metrics/confusion_matrix/__init__.py: -------------------------------------------------------------------------------- 1 | from .calculation import * 2 | from .confusion_matrix import * 3 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/metrics/iou/__init__.py: -------------------------------------------------------------------------------- 1 | from .calculation import * 2 | from .iou import * 3 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/mixins.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | from lbox.exceptions import ( 4 | ConfidenceNotSupportedException, 5 | CustomMetricsNotSupportedException, 6 | ) 7 | from pydantic import BaseModel, field_validator 8 | 9 | 10 | class ConfidenceMixin(BaseModel): 11 | confidence: Optional[float] = None 12 | 13 | @field_validator("confidence") 14 | def confidence_valid_float(cls, value): 15 | if value is None: 16 | return value 17 | if not isinstance(value, (int, float)) or not 0 <= value <= 1: 18 | raise ValueError("must be a number within [0,1] range") 19 | return value 20 | 21 | 22 | class ConfidenceNotSupportedMixin: 23 | def __new__(cls, *args, **kwargs): 24 | if "confidence" in kwargs: 25 | raise ConfidenceNotSupportedException( 26 | "Confidence is not supported for this annotation type yet" 27 | ) 28 | return super().__new__(cls) 29 | 30 | 31 | class CustomMetric(BaseModel): 32 | name: str 33 | value: float 34 | 35 | @field_validator("name") 36 | def confidence_valid_float(cls, value): 37 | if not isinstance(value, str): 38 | raise ValueError("Name must be a string") 39 | return value 40 | 41 | @field_validator("value") 42 | def value_valid_float(cls, value): 43 | if not isinstance(value, (int, float)): 44 | raise ValueError("Value must be a number") 45 | return value 46 | 47 | 48 | class CustomMetricsMixin(BaseModel): 49 | custom_metrics: Optional[List[CustomMetric]] = None 50 | 51 | 52 | class CustomMetricsNotSupportedMixin: 53 | def __new__(cls, *args, **kwargs): 54 | if "custom_metrics" in kwargs: 55 | raise CustomMetricsNotSupportedException( 56 | "Custom metrics is not supported for this annotation type yet" 57 | ) 58 | return super().__new__(cls) 59 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/serialization/__init__.py: -------------------------------------------------------------------------------- 1 | from .ndjson import NDJsonConverter 2 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/serialization/ndjson/__init__.py: -------------------------------------------------------------------------------- 1 | from .converter import NDJsonConverter 2 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/serialization/ndjson/base.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from uuid import uuid4 3 | 4 | from labelbox.utils import _CamelCaseMixin, is_exactly_one_set 5 | from pydantic import model_validator, Field 6 | from uuid import uuid4 7 | 8 | from ....annotated_types import Cuid 9 | 10 | 11 | class DataRow(_CamelCaseMixin): 12 | id: Optional[str] = None 13 | global_key: Optional[str] = None 14 | 15 | @model_validator(mode="after") 16 | def must_set_one(self): 17 | if not is_exactly_one_set(self.id, self.global_key): 18 | raise ValueError("Must set either id or global_key") 19 | return self 20 | 21 | 22 | class NDJsonBase(_CamelCaseMixin): 23 | uuid: Optional[str] = Field(default_factory=lambda: str(uuid4())) 24 | data_row: DataRow 25 | 26 | 27 | class NDAnnotation(NDJsonBase): 28 | name: Optional[str] = None 29 | schema_id: Optional[Cuid] = None 30 | message_id: Optional[str] = None 31 | page: Optional[int] = None 32 | unit: Optional[str] = None 33 | 34 | @model_validator(mode="after") 35 | def must_set_one(self): 36 | if (not hasattr(self, "schema_id") or self.schema_id is None) and ( 37 | not hasattr(self, "name") or self.name is None 38 | ): 39 | raise ValueError("Schema id or name are not set. Set either one.") 40 | return self 41 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/serialization/ndjson/converter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Any, Dict, Generator 3 | 4 | 5 | from ...annotation_types.collection import LabelCollection 6 | from .label import NDLabel 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | IGNORE_IF_NONE = ["page", "unit", "messageId"] 11 | 12 | 13 | class NDJsonConverter: 14 | @staticmethod 15 | def serialize( 16 | labels: LabelCollection, 17 | ) -> Generator[Dict[str, Any], None, None]: 18 | """ 19 | Converts a labelbox common object to the labelbox ndjson format (prediction import format) 20 | 21 | Note that this function might fail for objects that are not supported by mal. 22 | Not all edge cases are handling by custom exceptions, if you get a cryptic pydantic error message it is probably due to this. 23 | We will continue to improve the error messages and add helper functions to deal with this. 24 | 25 | Args: 26 | labels: Either a list of Label objects or a LabelGenerator 27 | Returns: 28 | A generator for accessing the ndjson representation of the data 29 | """ 30 | 31 | for label in labels: 32 | for example in NDLabel.from_common([label]): 33 | annotation_uuid = getattr(example, "uuid", None) 34 | res = example.model_dump( 35 | exclude_none=True, 36 | by_alias=True, 37 | exclude={"uuid"} if annotation_uuid == "None" else None, 38 | ) 39 | for k, v in list(res.items()): 40 | if k in IGNORE_IF_NONE and v is None: 41 | del res[k] 42 | if getattr(label, "is_benchmark_reference"): 43 | res["isBenchmarkReferenceLabel"] = True 44 | yield res 45 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/data/serialization/ndjson/mmc.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from labelbox.utils import _CamelCaseMixin 4 | 5 | from .base import DataRow, NDAnnotation 6 | from ...annotation_types.mmc import ( 7 | MessageSingleSelectionTask, 8 | MessageMultiSelectionTask, 9 | MessageRankingTask, 10 | MessageEvaluationTaskAnnotation, 11 | ) 12 | from ...annotation_types import GenericDataRowData 13 | 14 | 15 | class MessageTaskData(_CamelCaseMixin): 16 | format: str 17 | data: Union[ 18 | MessageSingleSelectionTask, 19 | MessageMultiSelectionTask, 20 | MessageRankingTask, 21 | ] 22 | 23 | 24 | class NDMessageTask(NDAnnotation): 25 | message_evaluation_task: MessageTaskData 26 | 27 | def to_common(self) -> MessageEvaluationTaskAnnotation: 28 | return MessageEvaluationTaskAnnotation( 29 | name=self.name, 30 | feature_schema_id=self.schema_id, 31 | value=self.message_evaluation_task.data, 32 | extra={"uuid": self.uuid}, 33 | ) 34 | 35 | @classmethod 36 | def from_common( 37 | cls, 38 | annotation: MessageEvaluationTaskAnnotation, 39 | data: GenericDataRowData, 40 | ) -> "NDMessageTask": 41 | return cls( 42 | uuid=str(annotation._uuid), 43 | name=annotation.name, 44 | schema_id=annotation.feature_schema_id, 45 | data_row=DataRow(id=data.uid, global_key=data.global_key), 46 | message_evaluation_task=MessageTaskData( 47 | format=annotation.value.format, data=annotation.value 48 | ), 49 | ) 50 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/orm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/src/labelbox/orm/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/parser.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | class NdjsonDecoder(json.JSONDecoder): 5 | def __init__(self, *args, **kwargs): 6 | super().__init__(*args, **kwargs) 7 | 8 | def decode(self, s: str, *args, **kwargs): 9 | lines = ",".join(s.splitlines()) 10 | text = f"[{lines}]" # NOTE: this is a hack to make json.loads work for ndjson 11 | return super().decode(text, *args, **kwargs) 12 | 13 | 14 | def loads(ndjson_string, **kwargs) -> list: 15 | kwargs.setdefault("cls", NdjsonDecoder) 16 | return json.loads(ndjson_string, **kwargs) 17 | 18 | 19 | def dumps(obj, **kwargs): 20 | lines = map(lambda obj: json.dumps(obj, **kwargs), obj) 21 | return "\n".join(lines) 22 | 23 | 24 | def dump(obj, io, **kwargs): 25 | lines = dumps(obj, **kwargs) 26 | io.write(lines) 27 | 28 | 29 | def reader(io_handle, **kwargs): 30 | for line in io_handle: 31 | yield json.loads(line, **kwargs) 32 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/__init__.py: -------------------------------------------------------------------------------- 1 | import labelbox.schema.asset_attachment 2 | import labelbox.schema.annotation_import 3 | import labelbox.schema.benchmark 4 | import labelbox.schema.data_row 5 | import labelbox.schema.dataset 6 | import labelbox.schema.invite 7 | import labelbox.schema.label 8 | import labelbox.schema.labeling_frontend 9 | import labelbox.schema.labeling_service 10 | import labelbox.schema.model 11 | import labelbox.schema.model_run 12 | import labelbox.schema.ontology 13 | import labelbox.schema.organization 14 | import labelbox.schema.project 15 | import labelbox.schema.review 16 | import labelbox.schema.role 17 | import labelbox.schema.task 18 | import labelbox.schema.user 19 | import labelbox.schema.webhook 20 | import labelbox.schema.data_row_metadata 21 | import labelbox.schema.batch 22 | import labelbox.schema.iam_integration 23 | import labelbox.schema.media_type 24 | import labelbox.schema.identifiables 25 | import labelbox.schema.identifiable 26 | import labelbox.schema.catalog 27 | import labelbox.schema.ontology_kind 28 | import labelbox.schema.project_overview 29 | import labelbox.schema.taskstatus 30 | import labelbox.schema.api_key 31 | import labelbox.schema.timeunit 32 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/benchmark.py: -------------------------------------------------------------------------------- 1 | from labelbox.orm.db_object import DbObject 2 | from labelbox.orm.model import Field, Relationship 3 | 4 | 5 | class Benchmark(DbObject): 6 | """Represents a benchmark label. 7 | 8 | The Benchmarks tool works by interspersing data to be labeled, for 9 | which there is a benchmark label, to each person labeling. These 10 | labeled data are compared against their respective benchmark and an 11 | accuracy score between 0 and 100 percent is calculated. 12 | 13 | Attributes: 14 | created_at (datetime) 15 | last_activity (datetime) 16 | average_agreement (float) 17 | completed_count (int) 18 | 19 | created_by (Relationship): `ToOne` relationship to User 20 | reference_label (Relationship): `ToOne` relationship to Label 21 | """ 22 | 23 | created_at = Field.DateTime("created_at") 24 | created_by = Relationship.ToOne("User", False, "created_by") 25 | last_activity = Field.DateTime("last_activity") 26 | average_agreement = Field.Float("average_agreement") 27 | completed_count = Field.Int("completed_count") 28 | 29 | reference_label = Relationship.ToOne("Label", False, "reference_label") 30 | 31 | def delete(self) -> None: 32 | label_param = "labelId" 33 | query_str = """mutation DeleteBenchmarkPyApi($%s: ID!) { 34 | deleteBenchmark(where: {labelId: $%s}) {id}} """ % ( 35 | label_param, 36 | label_param, 37 | ) 38 | self.client.execute( 39 | query_str, {label_param: self.reference_label().uid} 40 | ) 41 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/confidence_presence_checker.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List, Set 2 | 3 | 4 | class LabelsConfidencePresenceChecker: 5 | """ 6 | Checks if a given list of labels contains at least one confidence score 7 | """ 8 | 9 | @classmethod 10 | def check(cls, raw_labels: List[Dict[str, Any]]): 11 | keys: Set[str] = set([]) 12 | cls._collect_keys_from_list(raw_labels, keys) 13 | return len(keys.intersection(set(["confidence"]))) == 1 14 | 15 | @classmethod 16 | def _collect_keys_from_list( 17 | cls, objects: List[Dict[str, Any]], keys: Set[str] 18 | ): 19 | for obj in objects: 20 | if isinstance(obj, (list, tuple)): 21 | cls._collect_keys_from_list(obj, keys) 22 | elif isinstance(obj, dict): 23 | cls._collect_keys_from_object(obj, keys) 24 | 25 | @classmethod 26 | def _collect_keys_from_object(cls, object: Dict[str, Any], keys: Set[str]): 27 | for key in object: 28 | keys.add(key) 29 | if isinstance(object[key], dict): 30 | cls._collect_keys_from_object(object[key], keys) 31 | if isinstance(object[key], (list, tuple)): 32 | cls._collect_keys_from_list(object[key], keys) 33 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/conflict_resolution_strategy.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class ConflictResolutionStrategy(str, Enum): 5 | KeepExisting = "KEEP_EXISTING" 6 | OverrideWithAnnotations = "OVERRIDE_WITH_ANNOTATIONS" 7 | OverrideWithPredictions = "OVERRIDE_WITH_PREDICTIONS" 8 | 9 | @staticmethod 10 | def from_str(label: str) -> "ConflictResolutionStrategy": 11 | return ConflictResolutionStrategy[label] 12 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/consensus_settings.py: -------------------------------------------------------------------------------- 1 | from labelbox.utils import _CamelCaseMixin 2 | 3 | 4 | class ConsensusSettings(_CamelCaseMixin): 5 | """Container for holding consensus quality settings 6 | 7 | >>> ConsensusSettings( 8 | >>> number_of_labels = 2, 9 | >>> coverage_percentage = 0.2 10 | >>> ) 11 | 12 | Args: 13 | number_of_labels: Number of labels for consensus 14 | coverage_percentage: Percentage of data rows to be labeled more than once 15 | """ 16 | 17 | number_of_labels: int 18 | coverage_percentage: float 19 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/data_row_payload_templates.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List, Optional 2 | 3 | from pydantic import BaseModel, Field 4 | 5 | from labelbox.schema.data_row import DataRowMetadataField 6 | 7 | 8 | class ModelEvalutationTemplateRowData(BaseModel): 9 | type: str = Field( 10 | default="application/vnd.labelbox.conversational.model-chat-evaluation", 11 | frozen=True, 12 | ) 13 | draft: bool = Field(default=True, frozen=True) 14 | rootMessageIds: List[str] = Field(default=[]) 15 | actors: Dict = Field(default={}) 16 | version: int = Field(default=2, frozen=True) 17 | messages: Dict = Field(default={}) 18 | 19 | 20 | class ModelEvaluationTemplate(BaseModel): 21 | """ 22 | Use this class to create a model evaluation data row. 23 | 24 | Examples: 25 | >>> data = ModelEvaluationTemplate() 26 | >>> data.row_data.rootMessageIds = ["root1"] 27 | >>> vector = [random.uniform(1.0, 2.0) for _ in range(embedding.dims)] 28 | >>> data.embeddings = [...] 29 | >>> data.metadata_fields = [...] 30 | >>> data.attachments = [...] 31 | >>> content = data.model_dump() 32 | >>> task = dataset.create_data_rows([content]) 33 | """ 34 | 35 | row_data: ModelEvalutationTemplateRowData = Field( 36 | default=ModelEvalutationTemplateRowData() 37 | ) 38 | global_key: Optional[str] = None 39 | attachments: List[Dict] = Field(default=[]) 40 | embeddings: List[Dict] = Field(default=[]) 41 | metadata_fields: List[DataRowMetadataField] = Field(default=[]) 42 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/enums.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class AnnotationImportState(Enum): 5 | """State of the import job when importing annotations (RUNNING, FAILED, or FINISHED). 6 | 7 | .. list-table:: 8 | :widths: 15 150 9 | :header-rows: 1 10 | 11 | * - State 12 | - Description 13 | * - RUNNING 14 | - Indicates that the import job is not done yet. 15 | * - FAILED 16 | - Indicates the import job failed. Check `AnnotationImport.errors` for more information 17 | * - FINISHED 18 | - Indicates the import job is no longer running. Check `AnnotationImport.statuses` for more information 19 | """ 20 | 21 | RUNNING = "RUNNING" 22 | FAILED = "FAILED" 23 | FINISHED = "FINISHED" 24 | 25 | 26 | class CollectionJobStatus(Enum): 27 | """Status of an asynchronous job over a collection. 28 | 29 | * - State 30 | - Description 31 | * - SUCCESS 32 | - Indicates job has successfully processed entire collection of data 33 | * - PARTIAL SUCCESS 34 | - Indicates some data in the collection has succeeded and other data have failed 35 | * - FAILURE 36 | - Indicates job has failed to process entire collection of data 37 | """ 38 | 39 | SUCCESS = "SUCCESS" 40 | PARTIAL_SUCCESS = "PARTIAL SUCCESS" 41 | FAILURE = "FAILURE" 42 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/export_params.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional, TypedDict 2 | 3 | from labelbox.schema.media_type import MediaType 4 | 5 | EXPORT_LIMIT = 30 6 | 7 | 8 | class DataRowParams(TypedDict): 9 | data_row_details: Optional[bool] 10 | metadata_fields: Optional[bool] 11 | attachments: Optional[bool] 12 | embeddings: Optional[bool] 13 | media_type_override: Optional[MediaType] 14 | 15 | 16 | class ProjectExportParams(DataRowParams): 17 | project_details: Optional[bool] 18 | label_details: Optional[bool] 19 | performance_details: Optional[bool] 20 | interpolated_frames: Optional[bool] 21 | 22 | 23 | class CatalogExportParams(DataRowParams): 24 | project_details: Optional[bool] 25 | label_details: Optional[bool] 26 | performance_details: Optional[bool] 27 | model_run_ids: Optional[List[str]] 28 | project_ids: Optional[List[str]] 29 | interpolated_frames: Optional[bool] 30 | all_projects: Optional[bool] 31 | all_model_runs: Optional[bool] 32 | predictions: Optional[bool] 33 | 34 | 35 | class ModelRunExportParams(DataRowParams): 36 | predictions: Optional[bool] 37 | model_run_details: Optional[bool] 38 | 39 | 40 | def _validate_array_length(array, max_length, array_name): 41 | if len(array) > max_length: 42 | raise ValueError(f"{array_name} cannot exceed {max_length} items") 43 | 44 | 45 | def validate_catalog_export_params(params: CatalogExportParams): 46 | if "model_run_ids" in params and params["model_run_ids"] is not None: 47 | _validate_array_length( 48 | params["model_run_ids"], EXPORT_LIMIT, "model_run_ids" 49 | ) 50 | 51 | if "project_ids" in params and params["project_ids"] is not None: 52 | _validate_array_length( 53 | params["project_ids"], EXPORT_LIMIT, "project_ids" 54 | ) 55 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/foundry/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/src/labelbox/schema/foundry/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/foundry/app.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Optional 2 | from pydantic import ConfigDict 3 | from labelbox.utils import _CamelCaseMixin 4 | 5 | 6 | class App(_CamelCaseMixin): 7 | id: Optional[str] = None 8 | model_id: str 9 | name: str 10 | description: Optional[str] = None 11 | inference_params: Dict[str, Any] 12 | class_to_schema_id: Dict[str, str] 13 | ontology_id: str 14 | created_by: Optional[str] = None 15 | 16 | model_config = ConfigDict(protected_namespaces=()) 17 | 18 | @classmethod 19 | def type_name(cls): 20 | return "App" 21 | 22 | 23 | APP_FIELD_NAMES = list(App.model_json_schema()["properties"].keys()) 24 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/foundry/model.py: -------------------------------------------------------------------------------- 1 | from labelbox.utils import _CamelCaseMixin 2 | 3 | 4 | from datetime import datetime 5 | from typing import Dict 6 | from pydantic import BaseModel 7 | 8 | 9 | class Model(_CamelCaseMixin, BaseModel): 10 | id: str 11 | description: str 12 | inference_params_json_schema: Dict 13 | name: str 14 | ontology_id: str 15 | created_at: datetime 16 | 17 | 18 | MODEL_FIELD_NAMES = list(Model.model_json_schema()["properties"].keys()) 19 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/id_type.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | if sys.version_info >= (3, 9): 4 | from strenum import StrEnum 5 | 6 | class BaseStrEnum(StrEnum): 7 | pass 8 | else: 9 | from enum import Enum 10 | 11 | class BaseStrEnum(str, Enum): 12 | pass 13 | 14 | 15 | class IdType(BaseStrEnum): 16 | """ 17 | The type of id used to identify a data row. 18 | 19 | Currently supported types are: 20 | - DataRowId: The id assigned to a data row by Labelbox. 21 | - GlobalKey: The id assigned to a data row by the user. 22 | """ 23 | 24 | DataRowId = "ID" 25 | GlobalKey = "GKEY" 26 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/identifiable.py: -------------------------------------------------------------------------------- 1 | from abc import ABC 2 | from typing import Union 3 | 4 | from labelbox.schema.id_type import IdType 5 | 6 | 7 | class Identifiable(ABC): 8 | """ 9 | Base class for any object representing a unique identifier. 10 | """ 11 | 12 | def __init__(self, key: str, id_type: str): 13 | self._key = key 14 | self._id_type = id_type 15 | 16 | @property 17 | def key(self): 18 | return self._key 19 | 20 | @property 21 | def id_type(self): 22 | return self._id_type 23 | 24 | def __eq__(self, other): 25 | return other.key == self.key and other.id_type == self.id_type 26 | 27 | def __hash__(self): 28 | return hash((self.key, self.id_type)) 29 | 30 | def __str__(self): 31 | return f"{self.id_type}:{self.key}" 32 | 33 | 34 | class UniqueId(Identifiable): 35 | """ 36 | Represents a unique, internally generated id. 37 | """ 38 | 39 | def __init__(self, key: str): 40 | super().__init__(key, IdType.DataRowId) 41 | 42 | 43 | class GlobalKey(Identifiable): 44 | """ 45 | Represents a user generated id. 46 | """ 47 | 48 | def __init__(self, key: str): 49 | super().__init__(key, IdType.GlobalKey) 50 | 51 | 52 | DataRowIdentifier = Union[UniqueId, GlobalKey] 53 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/identifiables.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union 2 | 3 | from labelbox.schema.id_type import IdType 4 | 5 | 6 | class Identifiables: 7 | def __init__(self, iterable, id_type: str): 8 | """ 9 | Args: 10 | iterable: Iterable of ids (unique or global keys) 11 | id_type: The type of id used to identify a data row. 12 | """ 13 | self._iterable = iterable 14 | self._id_type = id_type 15 | 16 | @property 17 | def id_type(self): 18 | return self._id_type 19 | 20 | def __iter__(self): 21 | return iter(self._iterable) 22 | 23 | def __getitem__(self, index): 24 | if isinstance(index, slice): 25 | ids = self._iterable[index] 26 | return self.__class__(ids) # type: ignore 27 | return self._iterable[index] 28 | 29 | def __len__(self): 30 | return len(self._iterable) 31 | 32 | def __repr__(self) -> str: 33 | return f"{self.__class__.__name__}({self._iterable})" 34 | 35 | def __eq__(self, other: object) -> bool: 36 | if not isinstance(other, Identifiables): 37 | return False 38 | return ( 39 | self._iterable == other._iterable 40 | and self._id_type == other._id_type 41 | ) 42 | 43 | 44 | class UniqueIds(Identifiables): 45 | """ 46 | Represents a collection of unique, internally generated ids. 47 | """ 48 | 49 | def __init__(self, iterable: List[str]): 50 | super().__init__(iterable, IdType.DataRowId) 51 | 52 | 53 | class GlobalKeys(Identifiables): 54 | """ 55 | Represents a collection of user generated ids. 56 | """ 57 | 58 | def __init__(self, iterable: List[str]): 59 | super().__init__(iterable, IdType.GlobalKey) 60 | 61 | 62 | DataRowIds = UniqueIds 63 | 64 | DataRowIdentifiers = Union[UniqueIds, GlobalKeys] 65 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/internal/data_row_uploader.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from labelbox.schema.internal.data_row_upsert_item import ( 4 | DataRowItemBase, 5 | ) 6 | from labelbox.schema.internal.descriptor_file_creator import ( 7 | DescriptorFileCreator, 8 | ) 9 | from pydantic import BaseModel 10 | 11 | 12 | class UploadManifest(BaseModel): 13 | source: str 14 | item_count: int 15 | chunk_uris: List[str] 16 | 17 | 18 | SOURCE_SDK = "SDK" 19 | 20 | 21 | def upload_in_chunks( 22 | client, 23 | specs: List[DataRowItemBase], 24 | file_upload_thread_count: int, 25 | max_chunk_size_bytes: int, 26 | ) -> UploadManifest: 27 | empty_specs = list(filter(lambda spec: spec.is_empty(), specs)) 28 | if empty_specs: 29 | ids = list(map(lambda spec: spec.id.get("value"), empty_specs)) 30 | ids = list(filter(lambda x: x is not None and len(x) > 0, ids)) 31 | if len(ids) > 0: 32 | raise ValueError( 33 | f"The following items have an empty payload: {ids}" 34 | ) 35 | else: # case of create items 36 | raise ValueError("Some items have an empty payload") 37 | 38 | chunk_uris = DescriptorFileCreator(client).create( 39 | specs, max_chunk_size_bytes=max_chunk_size_bytes 40 | ) 41 | 42 | return UploadManifest( 43 | source=SOURCE_SDK, item_count=len(specs), chunk_uris=chunk_uris 44 | ) 45 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/internal/datarow_upload_constants.py: -------------------------------------------------------------------------------- 1 | FILE_UPLOAD_THREAD_COUNT = 20 2 | UPSERT_CHUNK_SIZE_BYTES = 10_000_000 3 | DOWNLOAD_RESULT_PAGE_SIZE = 5_000 4 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/label_score.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class LabelScore(BaseModel): 5 | """ 6 | A label score. 7 | 8 | Attributes: 9 | name (str) 10 | score (float) 11 | 12 | """ 13 | 14 | name: str 15 | score: float 16 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/labeling_frontend.py: -------------------------------------------------------------------------------- 1 | from labelbox.orm.db_object import DbObject 2 | from labelbox.orm.model import Field, Relationship 3 | 4 | 5 | class LabelingFrontend(DbObject): 6 | """Private db object representing a projects label editor""" 7 | 8 | name = Field.String("name") 9 | description = Field.String("description") 10 | iframe_url_path = Field.String("iframe_url_path") 11 | 12 | 13 | class LabelingFrontendOptions(DbObject): 14 | """Label interface options. 15 | 16 | Attributes: 17 | customization_options (str) 18 | 19 | project (Relationship): `ToOne` relationship to Project 20 | labeling_frontend (Relationship): `ToOne` relationship to LabelingFrontend 21 | organization (Relationship): `ToOne` relationship to Organization 22 | """ 23 | 24 | customization_options = Field.String("customization_options") 25 | 26 | project = Relationship.ToOne("Project") 27 | labeling_frontend = Relationship.ToOne("LabelingFrontend") 28 | organization = Relationship.ToOne("Organization") 29 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/labeling_service_status.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class LabelingServiceStatus(Enum): 5 | Accepted = "ACCEPTED" 6 | Calibration = "CALIBRATION" 7 | Complete = "COMPLETE" 8 | Production = "PRODUCTION" 9 | Requested = "REQUESTED" 10 | SetUp = "SET_UP" 11 | Missing = None 12 | 13 | @classmethod 14 | def is_supported(cls, value): 15 | return isinstance(value, cls) 16 | 17 | @classmethod 18 | def _missing_(cls, value) -> "LabelingServiceStatus": 19 | """Handle missing null new task types 20 | Handle upper case names for compatibility with 21 | the GraphQL""" 22 | 23 | if value is None: 24 | return cls.Missing 25 | 26 | for name, member in cls.__members__.items(): 27 | if value == name.upper(): 28 | return member 29 | 30 | return cls.Missing 31 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/model_config.py: -------------------------------------------------------------------------------- 1 | from labelbox.orm.db_object import DbObject 2 | from labelbox.orm.model import Field 3 | 4 | 5 | class ModelConfig(DbObject): 6 | """A ModelConfig represents a set of inference params configured for a model 7 | 8 | Attributes: 9 | inference_params (JSON): Dict of inference params 10 | model_id (str): ID of the model to configure 11 | name (str): Name of config 12 | """ 13 | 14 | inference_params = Field.Json("inference_params", "inferenceParams") 15 | model_id = Field.String("model_id", "modelId") 16 | name = Field.String("name", "name") 17 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/project_model_config.py: -------------------------------------------------------------------------------- 1 | from lbox.exceptions import ( 2 | LabelboxError, 3 | error_message_for_unparsed_graphql_error, 4 | ) 5 | 6 | from labelbox.orm.db_object import DbObject 7 | from labelbox.orm.model import Field, Relationship 8 | 9 | 10 | class ProjectModelConfig(DbObject): 11 | """A ProjectModelConfig represents an association between a project and a single model config. 12 | 13 | Attributes: 14 | project_id (str): ID of project to associate 15 | model_config_id (str): ID of the model configuration 16 | model_config (ModelConfig): Configuration for model 17 | """ 18 | 19 | project_id = Field.String("project_id", "projectId") 20 | model_config_id = Field.String("model_config_id", "modelConfigId") 21 | model_config = Relationship.ToOne("ModelConfig", False, "model_config") 22 | 23 | def delete(self) -> bool: 24 | """Deletes this association between a model config and this project. 25 | 26 | Returns: 27 | bool, indicates if the operation was a success. 28 | """ 29 | query = """mutation DeleteProjectModelConfigPyApi($id: ID!) { 30 | deleteProjectModelConfig(input: {id: $id}) { 31 | success 32 | } 33 | }""" 34 | 35 | params = { 36 | "id": self.uid, 37 | } 38 | 39 | try: 40 | result = self.client.execute(query, params) 41 | except LabelboxError as e: 42 | if e.message.startswith( 43 | "Unknown error: " 44 | ): # unfortunate hack to handle unparsed graphql errors 45 | error_content = error_message_for_unparsed_graphql_error( 46 | e.message 47 | ) 48 | else: 49 | error_content = e.message 50 | raise LabelboxError(message=error_content) from e 51 | 52 | return result["deleteProjectModelConfig"]["success"] 53 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/project_resource_tag.py: -------------------------------------------------------------------------------- 1 | from labelbox.orm.db_object import DbObject, Updateable 2 | from labelbox.orm.model import Field 3 | 4 | 5 | class ProjectResourceTag(DbObject, Updateable): 6 | """Project resource tag to associate ProjectResourceTag to Project. 7 | 8 | Attributes: 9 | resourceTagId (str) 10 | projectId (str) 11 | 12 | resource_tag (Relationship): `ToOne` relationship to ResourceTag 13 | """ 14 | 15 | resource_tag_id = Field.ID("resource_tag_id") 16 | project_id = Field.ID("project_id") 17 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/quality_mode.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class QualityMode(str, Enum): 5 | Benchmark = "BENCHMARK" 6 | Consensus = "CONSENSUS" 7 | 8 | 9 | BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS = 1 10 | BENCHMARK_AUTO_AUDIT_PERCENTAGE = 1 11 | CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS = 3 12 | CONSENSUS_AUTO_AUDIT_PERCENTAGE = 0 13 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/resource_tag.py: -------------------------------------------------------------------------------- 1 | from labelbox.orm.db_object import DbObject, Updateable 2 | from labelbox.orm.model import Field 3 | 4 | 5 | class ResourceTag(DbObject, Updateable): 6 | """Resource tag to label and identify your labelbox resources easier. 7 | 8 | Attributes: 9 | text (str) 10 | color (str) 11 | 12 | project_resource_tag (Relationship): `ToMany` relationship to ProjectResourceTag 13 | """ 14 | 15 | text = Field.String("text") 16 | color = Field.String("color") 17 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/review.py: -------------------------------------------------------------------------------- 1 | from enum import Enum, auto 2 | 3 | from labelbox.orm.db_object import DbObject, Updateable, Deletable 4 | from labelbox.orm.model import Field, Relationship 5 | 6 | 7 | class Review(DbObject, Deletable, Updateable): 8 | """Reviewing labeled data is a collaborative quality assurance technique. 9 | 10 | A Review object indicates the quality of the assigned Label. The aggregated 11 | review numbers can be obtained on a Project object. 12 | 13 | Attributes: 14 | created_at (datetime) 15 | updated_at (datetime) 16 | score (float) 17 | 18 | created_by (Relationship): `ToOne` relationship to User 19 | organization (Relationship): `ToOne` relationship to Organization 20 | project (Relationship): `ToOne` relationship to Project 21 | label (Relationship): `ToOne` relationship to Label 22 | """ 23 | 24 | class NetScore(Enum): 25 | """Negative, Zero, or Positive.""" 26 | 27 | Negative = auto() 28 | Zero = auto() 29 | Positive = auto() 30 | 31 | updated_at = Field.DateTime("updated_at") 32 | created_at = Field.DateTime("created_at") 33 | score = Field.Float("score") 34 | 35 | created_by = Relationship.ToOne("User", False, "created_by") 36 | organization = Relationship.ToOne("Organization", False) 37 | project = Relationship.ToOne("Project", False) 38 | label = Relationship.ToOne("Label", False) 39 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/role.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Dict, Optional, TYPE_CHECKING 3 | 4 | from labelbox.orm.model import Field 5 | from labelbox.orm.db_object import DbObject 6 | 7 | if TYPE_CHECKING: 8 | from labelbox import Client, Project 9 | 10 | _ROLES: Optional[Dict[str, "Role"]] = None 11 | 12 | 13 | def get_roles(client: "Client") -> Dict[str, "Role"]: 14 | global _ROLES 15 | if _ROLES is None: 16 | query_str = """query GetAvailableUserRolesPyApi { roles { id name } }""" 17 | res = client.execute(query_str) 18 | _ROLES = {} 19 | for role in res["roles"]: 20 | role["name"] = format_role(role["name"]) 21 | _ROLES[role["name"]] = Role(client, role) 22 | return _ROLES 23 | 24 | 25 | def format_role(name: str): 26 | return name.upper().replace(" ", "_") 27 | 28 | 29 | class Role(DbObject): 30 | name = Field.String("name") 31 | 32 | 33 | class OrgRole(Role): ... 34 | 35 | 36 | class UserRole(Role): ... 37 | 38 | 39 | @dataclass 40 | class ProjectRole: 41 | project: "Project" 42 | role: Role 43 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/serialization.py: -------------------------------------------------------------------------------- 1 | from typing import cast, Any, Dict, List, TYPE_CHECKING, Union 2 | 3 | if TYPE_CHECKING: 4 | from labelbox.types import Label 5 | 6 | 7 | def serialize_labels( 8 | objects: Union[List[Dict[str, Any]], List["Label"]], 9 | ) -> List[Dict[str, Any]]: 10 | """ 11 | Checks if objects are of type Labels and serializes labels for annotation import. Serialization depends the labelbox[data] package, therefore NDJsonConverter is only loaded if using `Label` objects instead of `dict` objects. 12 | """ 13 | if len(objects) == 0: 14 | return [] 15 | 16 | is_label_type = not isinstance(objects[0], Dict) 17 | if is_label_type: 18 | # If a Label object exists, labelbox[data] is already installed, so no error checking is needed. 19 | from labelbox.data.serialization import NDJsonConverter 20 | 21 | labels = cast(List["Label"], objects) 22 | return list(NDJsonConverter.serialize(labels)) 23 | 24 | return cast(List[Dict[str, Any]], objects) 25 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/task_queue.py: -------------------------------------------------------------------------------- 1 | from labelbox.orm.db_object import DbObject 2 | from labelbox.orm.model import Field 3 | 4 | 5 | class TaskQueue(DbObject): 6 | """ 7 | a task queue 8 | 9 | Attributes 10 | name 11 | description 12 | queue_type 13 | data_row_count 14 | 15 | Relationships 16 | project 17 | organization 18 | pass_queue 19 | fail_queue 20 | """ 21 | 22 | name = Field.String("name") 23 | description = Field.String("description") 24 | queue_type = Field.String("queue_type") 25 | data_row_count = Field.Int("data_row_count") 26 | 27 | def __init__(self, client, *args, **kwargs): 28 | super().__init__(client, *args, **kwargs) 29 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/taskstatus.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class TaskStatus(str, Enum): 5 | In_Progress = "IN_PROGRESS" 6 | Complete = "COMPLETE" 7 | Canceling = "CANCELLING" 8 | Canceled = "CANCELED" 9 | Failed = "FAILED" 10 | Unknown = "UNKNOWN" 11 | 12 | @classmethod 13 | def _missing_(cls, value): 14 | """Handle missing or unknown task status values. 15 | 16 | If a task status value is not found in the enum, this method returns 17 | the Unknown status instead of raising an error. 18 | 19 | Args: 20 | value: The status value that doesn't match any enum member 21 | 22 | Returns: 23 | TaskStatus.Unknown: The default status for unrecognized values 24 | """ 25 | return cls.Unknown 26 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/timeunit.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class TimeUnit(Enum): 5 | """Enum representing time units with their values in seconds. 6 | 7 | Used for time-based operations such as setting validity periods for API keys. 8 | 9 | Attributes: 10 | SECOND (int): 1 second 11 | MINUTE (int): 60 seconds 12 | HOUR (int): 3600 seconds (60 minutes) 13 | DAY (int): 86400 seconds (24 hours) 14 | WEEK (int): 604800 seconds (7 days) 15 | """ 16 | 17 | SECOND = 1 18 | MINUTE = 60 19 | HOUR = 60 * 60 20 | DAY = 24 * 60 * 60 21 | WEEK = 7 * 24 * 60 * 60 22 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/tool_building/__init__.py: -------------------------------------------------------------------------------- 1 | import labelbox.schema.tool_building.tool_type 2 | import labelbox.schema.tool_building.step_reasoning_tool 3 | import labelbox.schema.tool_building.fact_checking_tool 4 | import labelbox.schema.tool_building.prompt_issue_tool 5 | import labelbox.schema.tool_building.tool_type_mapping 6 | import labelbox.schema.tool_building.types 7 | import labelbox.schema.tool_building.classification 8 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | from enum import Enum 3 | 4 | from labelbox.schema.tool_building.base_step_reasoning_tool import ( 5 | _BaseStepReasoningTool, 6 | _Definition, 7 | _Variant, 8 | ) 9 | from labelbox.schema.tool_building.tool_type import ToolType 10 | 11 | 12 | class IncorrectStepActions(Enum): 13 | REGENERATE_STEPS = "regenerateSteps" 14 | GENERATE_AND_RATE_ALTERNATIVE_STEPS = "generateAndRateAlternativeSteps" 15 | REWRITE_STEP = "rewriteStep" 16 | JUSTIFICATION = "justification" 17 | 18 | 19 | def build_step_reasoning_definition(): 20 | correct_step = _Variant(id=0, name="Correct", actions=[]) 21 | neutral_step = _Variant(id=1, name="Neutral", actions=[]) 22 | incorrect_step = _Variant( 23 | id=2, 24 | name="Incorrect", 25 | _available_actions={action.value for action in IncorrectStepActions}, 26 | actions=[action.value for action in IncorrectStepActions], 27 | ) 28 | variants = [correct_step, neutral_step, incorrect_step] 29 | return _Definition(variants=variants) 30 | 31 | 32 | @dataclass 33 | class StepReasoningTool(_BaseStepReasoningTool): 34 | """ 35 | Use this class in OntologyBuilder to create a tool for step reasoning 36 | The definition field lists the possible options to evaulate a step 37 | 38 | NOTE: color attribute is for backward compatibility only and should not be set directly 39 | """ 40 | 41 | type: ToolType = field(default=ToolType.STEP_REASONING, init=False) 42 | definition: _Definition = field( 43 | default_factory=build_step_reasoning_definition 44 | ) 45 | 46 | def __post_init__(self): 47 | super().__post_init__() 48 | # Set available actions for variants 0, 1, 2 'out of band' since they are not passed in the definition 49 | self._set_variant_available_actions() 50 | 51 | def _set_variant_available_actions(self): 52 | for variant in self.definition.variants: 53 | if variant.id == 2: 54 | for action in IncorrectStepActions: 55 | variant._available_actions.add(action.value) 56 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/tool_building/tool_type.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class ToolType(Enum): 5 | STEP_REASONING = "step-reasoning" 6 | FACT_CHECKING = "fact-checking" 7 | PROMPT_ISSUE = "prompt-issue" 8 | 9 | @classmethod 10 | def valid(cls, tool_type: str) -> bool: 11 | try: 12 | ToolType(tool_type.lower()) 13 | return True 14 | except ValueError: 15 | return False 16 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py: -------------------------------------------------------------------------------- 1 | from labelbox.schema.tool_building.fact_checking_tool import ( 2 | FactCheckingTool, 3 | ) 4 | from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool 5 | from labelbox.schema.tool_building.step_reasoning_tool import ( 6 | StepReasoningTool, 7 | ) 8 | from labelbox.schema.tool_building.tool_type import ToolType 9 | 10 | 11 | def map_tool_type_to_tool_cls(tool_type_str: str): 12 | if not ToolType.valid(tool_type_str): 13 | return None 14 | 15 | tool_type = ToolType(tool_type_str.lower()) 16 | if tool_type == ToolType.STEP_REASONING: 17 | return StepReasoningTool 18 | elif tool_type == ToolType.FACT_CHECKING: 19 | return FactCheckingTool 20 | elif tool_type == ToolType.PROMPT_ISSUE: 21 | return PromptIssueTool 22 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/schema/tool_building/types.py: -------------------------------------------------------------------------------- 1 | from typing import Annotated, List 2 | from pydantic import Field 3 | 4 | 5 | from dataclasses import dataclass 6 | 7 | from typing import Any, Dict, List 8 | 9 | 10 | @dataclass 11 | class FeatureSchemaAttribute: 12 | attributeName: str 13 | attributeValue: str 14 | 15 | def asdict(self): 16 | return { 17 | "attributeName": self.attributeName, 18 | "attributeValue": self.attributeValue, 19 | } 20 | 21 | @classmethod 22 | def from_dict(cls, dictionary: Dict[str, Any]) -> "FeatureSchemaAttribute": 23 | return cls( 24 | attributeName=dictionary["attributeName"], 25 | attributeValue=dictionary["attributeValue"], 26 | ) 27 | 28 | 29 | FeatureSchemaId = Annotated[str, Field(min_length=25, max_length=25)] 30 | SchemaId = Annotated[str, Field(min_length=25, max_length=25)] 31 | FeatureSchemaAttributes = Annotated[ 32 | List[FeatureSchemaAttribute], Field(default_factory=list) 33 | ] 34 | -------------------------------------------------------------------------------- /libs/labelbox/src/labelbox/types.py: -------------------------------------------------------------------------------- 1 | try: 2 | from labelbox.data.annotation_types import * 3 | except ImportError: 4 | raise ImportError( 5 | "There are missing dependencies for `labelbox.types`, use `pip install labelbox[data] --upgrade` to install missing dependencies." 6 | ) 7 | -------------------------------------------------------------------------------- /libs/labelbox/tests/assets/arxiv-pdf_data_99-word-token-pdfs_0801.3483.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/assets/arxiv-pdf_data_99-word-token-pdfs_0801.3483.pdf -------------------------------------------------------------------------------- /libs/labelbox/tests/assets/loremipsum.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/assets/loremipsum.pdf -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_import/test_annotation_import_limit.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | from labelbox.schema.annotation_import import ( 3 | AnnotationImport, 4 | ) 5 | from labelbox.schema.media_type import MediaType 6 | import pytest 7 | from unittest.mock import patch 8 | 9 | 10 | @patch("labelbox.schema.annotation_import.ANNOTATION_PER_LABEL_LIMIT", 1) 11 | def test_above_annotation_limit_on_single_import_on_single_data_row( 12 | annotations_by_media_type, 13 | ): 14 | annotations_ndjson = list( 15 | itertools.chain.from_iterable( 16 | annotations_by_media_type[MediaType.Image] 17 | ) 18 | ) 19 | data_row_id = annotations_ndjson[0]["dataRow"]["id"] 20 | 21 | data_row_annotations = [ 22 | annotation 23 | for annotation in annotations_ndjson 24 | if annotation["dataRow"]["id"] == data_row_id and "bbox" in annotation 25 | ] 26 | 27 | with pytest.raises(ValueError): 28 | AnnotationImport._validate_data_rows([data_row_annotations[0]] * 2) 29 | 30 | 31 | @patch("labelbox.schema.annotation_import.ANNOTATION_PER_LABEL_LIMIT", 1) 32 | def test_above_annotation_limit_divided_among_different_rows( 33 | annotations_by_media_type, 34 | ): 35 | annotations_ndjson = list( 36 | itertools.chain.from_iterable( 37 | annotations_by_media_type[MediaType.Image] 38 | ) 39 | ) 40 | data_row_id = annotations_ndjson[0]["dataRow"]["id"] 41 | 42 | first_data_row_annotation = [ 43 | annotation 44 | for annotation in annotations_ndjson 45 | if annotation["dataRow"]["id"] == data_row_id and "bbox" in annotation 46 | ][0] 47 | 48 | second_data_row_annotation = first_data_row_annotation.copy() 49 | second_data_row_annotation["dataRow"]["id"] == "data_row_id_2" 50 | 51 | with pytest.raises(ValueError): 52 | AnnotationImport._validate_data_rows( 53 | [first_data_row_annotation, second_data_row_annotation] * 2 54 | ) 55 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_import/test_model.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from lbox.exceptions import ResourceNotFoundError 3 | 4 | from labelbox import Model 5 | 6 | 7 | def test_model(client, configured_project, rand_gen): 8 | # Get all 9 | models = list(client.get_models()) 10 | for m in models: 11 | assert isinstance(m, Model) 12 | 13 | # Create 14 | ontology = configured_project.ontology() 15 | data = {"name": rand_gen(str), "ontology_id": ontology.uid} 16 | model = client.create_model(data["name"], data["ontology_id"]) 17 | assert model.name == data["name"] 18 | 19 | # Get one 20 | model = client.get_model(model.uid) 21 | assert model.name == data["name"] 22 | 23 | # Delete 24 | model.delete() 25 | with pytest.raises(ResourceNotFoundError): 26 | client.get_model(model.uid) 27 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/annotation_types/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/classification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/annotation_types/classification/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/annotation_types/data/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/data/test_raster.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | 3 | import numpy as np 4 | import pytest 5 | from PIL import Image 6 | 7 | from labelbox.data.annotation_types.data import GenericDataRowData, MaskData 8 | from pydantic import ValidationError 9 | 10 | 11 | def test_validate_schema(): 12 | with pytest.raises(ValidationError): 13 | MaskData() 14 | 15 | 16 | def test_im_bytes(): 17 | data = (np.random.random((32, 32, 3)) * 255).astype(np.uint8) 18 | im_bytes = BytesIO() 19 | Image.fromarray(data).save(im_bytes, format="PNG") 20 | raster_data = MaskData(im_bytes=im_bytes.getvalue()) 21 | data_ = raster_data.value 22 | assert np.all(data == data_) 23 | 24 | 25 | def test_im_url(): 26 | raster_data = MaskData( 27 | uid="test", url="https://picsum.photos/id/829/200/300" 28 | ) 29 | data_ = raster_data.value 30 | assert data_.shape == (300, 200, 3) 31 | 32 | 33 | def test_ref(): 34 | external_id = "external_id" 35 | uid = "uid" 36 | metadata = [] 37 | media_attributes = {} 38 | data = GenericDataRowData( 39 | uid=uid, 40 | metadata=metadata, 41 | media_attributes=media_attributes, 42 | ) 43 | assert data.uid == uid 44 | assert data.media_attributes == media_attributes 45 | assert data.metadata == metadata 46 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/geometry/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/annotation_types/geometry/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/geometry/test_line.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import cv2 3 | 4 | from labelbox.data.annotation_types.geometry import Point, Line 5 | from pydantic import ValidationError 6 | 7 | 8 | def test_line(): 9 | with pytest.raises(ValidationError): 10 | line = Line() 11 | 12 | with pytest.raises(ValidationError): 13 | line = Line(points=[[0, 1], [2, 3]]) 14 | 15 | points = [[0, 1], [0, 2], [2, 2]] 16 | expected = {"coordinates": [points], "type": "MultiLineString"} 17 | line = Line(points=[Point(x=x, y=y) for x, y in points]) 18 | assert line.geometry == expected 19 | expected["coordinates"] = tuple([tuple([tuple(x) for x in points])]) 20 | assert line.shapely.__geo_interface__ == expected 21 | 22 | raster = line.draw(height=32, width=32, thickness=1) 23 | assert (cv2.imread("tests/data/assets/line.png") == raster).all() 24 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/geometry/test_point.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import cv2 3 | 4 | from labelbox.data.annotation_types import Point 5 | from pydantic import ValidationError 6 | 7 | 8 | def test_point(): 9 | with pytest.raises(ValidationError): 10 | line = Point() 11 | 12 | with pytest.raises(TypeError): 13 | line = Point([0, 1]) 14 | 15 | point = Point(x=0, y=1) 16 | expected = {"coordinates": [0, 1], "type": "Point"} 17 | assert point.geometry == expected 18 | expected["coordinates"] = tuple(expected["coordinates"]) 19 | assert point.shapely.__geo_interface__ == expected 20 | 21 | raster = point.draw(height=32, width=32, thickness=1) 22 | assert (cv2.imread("tests/data/assets/point.png") == raster).all() 23 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/geometry/test_polygon.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import cv2 3 | 4 | from labelbox.data.annotation_types import Polygon, Point 5 | from pydantic import ValidationError 6 | 7 | 8 | def test_polygon(): 9 | with pytest.raises(ValidationError): 10 | polygon = Polygon() 11 | 12 | with pytest.raises(ValidationError): 13 | polygon = Polygon(points=[[0, 1], [2, 3]]) 14 | 15 | with pytest.raises(ValidationError): 16 | polygon = Polygon(points=[Point(x=0, y=1), Point(x=0, y=1)]) 17 | 18 | points = [[0.0, 1.0], [0.0, 2.0], [2.0, 2.0], [2.0, 0.0]] 19 | expected = {"coordinates": [points + [points[0]]], "type": "Polygon"} 20 | polygon = Polygon(points=[Point(x=x, y=y) for x, y in points]) 21 | assert polygon.geometry == expected 22 | expected["coordinates"] = tuple( 23 | [tuple([tuple(x) for x in points + [points[0]]])] 24 | ) 25 | assert polygon.shapely.__geo_interface__ == expected 26 | 27 | raster = polygon.draw(10, 10) 28 | assert (cv2.imread("tests/data/assets/polygon.png") == raster).all() 29 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/geometry/test_rectangle.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import pytest 3 | 4 | from labelbox.data.annotation_types import Point, Rectangle 5 | from pydantic import ValidationError 6 | 7 | 8 | def test_rectangle(): 9 | with pytest.raises(ValidationError): 10 | rectangle = Rectangle() 11 | 12 | rectangle = Rectangle(start=Point(x=0, y=1), end=Point(x=10, y=10)) 13 | points = [[[0.0, 1.0], [0.0, 10.0], [10.0, 10.0], [10.0, 1.0], [0.0, 1.0]]] 14 | expected = {"coordinates": points, "type": "Polygon"} 15 | assert rectangle.geometry == expected 16 | expected["coordinates"] = tuple([tuple([tuple(x) for x in points[0]])]) 17 | assert rectangle.shapely.__geo_interface__ == expected 18 | 19 | raster = rectangle.draw(height=32, width=32) 20 | assert (cv2.imread("tests/data/assets/rectangle.png") == raster).all() 21 | 22 | xyhw = Rectangle.from_xyhw(0.0, 0, 10, 10) 23 | assert xyhw.start == Point(x=0, y=0.0) 24 | assert xyhw.end == Point(x=10, y=10.0) 25 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/test_label.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | import labelbox.types as lb_types 4 | from labelbox.data.annotation_types import ( 5 | PromptText, 6 | ObjectAnnotation, 7 | Point, 8 | Line, 9 | MaskData, 10 | Label, 11 | ) 12 | import pytest 13 | 14 | 15 | def test_schema_assignment_confidence(): 16 | name = "line_feature" 17 | label = Label( 18 | data=MaskData(arr=np.ones((32, 32, 3), dtype=np.uint8), uid="test"), 19 | annotations=[ 20 | ObjectAnnotation( 21 | value=Line( 22 | points=[Point(x=1, y=2), Point(x=2, y=2)], 23 | ), 24 | name=name, 25 | confidence=0.914, 26 | ) 27 | ], 28 | ) 29 | 30 | assert label.annotations[0].confidence == 0.914 31 | 32 | 33 | def test_initialize_label_no_coercion(): 34 | global_key = "global-key" 35 | ner_annotation = lb_types.ObjectAnnotation( 36 | name="ner", 37 | value=lb_types.ConversationEntity(start=0, end=8, message_id="4"), 38 | ) 39 | label = Label( 40 | data=lb_types.GenericDataRowData(global_key=global_key), 41 | annotations=[ner_annotation], 42 | ) 43 | assert isinstance(label.data, lb_types.GenericDataRowData) 44 | assert label.data.global_key == global_key 45 | 46 | 47 | def test_prompt_classification_validation(): 48 | global_key = "global-key" 49 | prompt_text = lb_types.PromptClassificationAnnotation( 50 | name="prompt text", value=PromptText(answer="test") 51 | ) 52 | prompt_text_2 = lb_types.PromptClassificationAnnotation( 53 | name="prompt text", value=PromptText(answer="test") 54 | ) 55 | with pytest.raises(TypeError) as e_info: 56 | label = Label( 57 | data={"global_key": global_key}, 58 | annotations=[prompt_text, prompt_text_2], 59 | ) 60 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/test_ner.py: -------------------------------------------------------------------------------- 1 | from labelbox.data.annotation_types import ( 2 | TextEntity, 3 | DocumentEntity, 4 | DocumentTextSelection, 5 | ) 6 | from labelbox.data.annotation_types.ner.conversation_entity import ( 7 | ConversationEntity, 8 | ) 9 | 10 | 11 | def test_ner(): 12 | start = 10 13 | end = 12 14 | text_entity = TextEntity(start=start, end=end) 15 | assert text_entity.start == start 16 | assert text_entity.end == end 17 | 18 | 19 | def test_document_entity(): 20 | document_entity = DocumentEntity( 21 | text_selections=[ 22 | DocumentTextSelection(token_ids=["1", "2"], group_id="1", page=1) 23 | ] 24 | ) 25 | 26 | assert document_entity.text_selections[0].token_ids == ["1", "2"] 27 | assert document_entity.text_selections[0].group_id == "1" 28 | assert document_entity.text_selections[0].page == 1 29 | 30 | 31 | def test_conversation_entity(): 32 | conversation_entity = ConversationEntity(message_id="1", start=0, end=1) 33 | 34 | assert conversation_entity.message_id == "1" 35 | assert conversation_entity.start == 0 36 | assert conversation_entity.end == 1 37 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/test_text.py: -------------------------------------------------------------------------------- 1 | from labelbox.data.annotation_types.classification.classification import Text 2 | 3 | 4 | def test_text(): 5 | text_entity = Text(answer="good job") 6 | assert text_entity.answer == "good job" 7 | 8 | 9 | def test_text_confidence(): 10 | text_entity = Text(answer="good job", confidence=0.5) 11 | assert text_entity.answer == "good job" 12 | assert text_entity.confidence == 0.5 13 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/annotation_types/test_video.py: -------------------------------------------------------------------------------- 1 | import labelbox.types as lb_types 2 | 3 | 4 | def test_mask_frame(): 5 | mask_frame = lb_types.MaskFrame( 6 | index=1, instance_uri="http://path/to/frame.png" 7 | ) 8 | assert mask_frame.model_dump(by_alias=True) == { 9 | "index": 1, 10 | "imBytes": None, 11 | "instanceURI": "http://path/to/frame.png", 12 | } 13 | 14 | 15 | def test_mask_instance(): 16 | mask_instance = lb_types.MaskInstance(color_rgb=(0, 0, 255), name="mask1") 17 | assert mask_instance.model_dump(by_alias=True, exclude_none=True) == { 18 | "colorRGB": (0, 0, 255), 19 | "name": "mask1", 20 | } 21 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000129054.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000129054.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000141328.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000141328.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000147498.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000147498.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000184324.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000184324.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000188465.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000188465.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000245764.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000245764.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000311303.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000311303.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000343706.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000343706.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000356387.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000356387.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000370711.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000370711.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000385029.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000385029.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000393226.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000393226.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000434230.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000434230.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000458755.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000458755.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000458768.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000458768.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000462904.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000462904.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000475150.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000475150.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000532481.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000532481.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000532493.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000532493.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/images/000000546823.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/images/000000546823.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000129054.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000129054.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000141328.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000141328.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000147498.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000147498.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000184324.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000184324.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000188465.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000188465.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000245764.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000245764.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000311303.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000311303.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000343706.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000343706.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000356387.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000356387.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000370711.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000370711.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000385029.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000385029.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000393226.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000393226.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000434230.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000434230.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000458755.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000458755.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000458768.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000458768.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000462904.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000462904.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000475150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000475150.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000532481.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000532481.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000532493.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000532493.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/coco/masks/000000546823.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/coco/masks/000000546823.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/line.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/mask.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/classification_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "answer": { 4 | "schemaId": "ckrb1sfl8099g0y91cxbd5ftb", 5 | "confidence": 0.8, 6 | "customMetrics": [ 7 | { 8 | "name": "customMetric1", 9 | "value": 0.5 10 | }, 11 | { 12 | "name": "customMetric2", 13 | "value": 0.3 14 | } 15 | ] 16 | }, 17 | "schemaId": "ckrb1sfjx099a0y914hl319ie", 18 | "dataRow": { 19 | "id": "ckrb1sf1i1g7i0ybcdc6oc8ct" 20 | }, 21 | "uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673" 22 | }, 23 | { 24 | "answer": [ 25 | { 26 | "schemaId": "ckrb1sfl8099e0y919v260awv", 27 | "confidence": 0.82, 28 | "customMetrics": [ 29 | { 30 | "name": "customMetric1", 31 | "value": 0.5 32 | }, 33 | { 34 | "name": "customMetric2", 35 | "value": 0.3 36 | } 37 | ] 38 | } 39 | ], 40 | "schemaId": "ckrb1sfkn099c0y910wbo0p1a", 41 | "dataRow": { 42 | "id": "ckrb1sf1i1g7i0ybcdc6oc8ct" 43 | }, 44 | "uuid": "d009925d-91a3-4f67-abd9-753453f5a584" 45 | }, 46 | { 47 | "answer": "a value", 48 | "schemaId": "ckrb1sfkn099c0y910wbo0p1a", 49 | "dataRow": { 50 | "id": "ckrb1sf1i1g7i0ybcdc6oc8ct" 51 | }, 52 | "uuid": "78ff6a23-bebe-475c-8f67-4c456909648f" 53 | } 54 | ] 55 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/classification_import_name_only.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "answer": { 4 | "name": "choice 1", 5 | "confidence": 0.99, 6 | "customMetrics": [ 7 | { 8 | "name": "customMetric1", 9 | "value": 0.5 10 | }, 11 | { 12 | "name": "customMetric2", 13 | "value": 0.3 14 | } 15 | ] 16 | }, 17 | "name": "classification a", 18 | "dataRow": { 19 | "id": "ckrb1sf1i1g7i0ybcdc6oc8ct" 20 | }, 21 | "uuid": "f6879f59-d2b5-49c2-aceb-d9e8dc478673" 22 | }, 23 | { 24 | "answer": [ 25 | { 26 | "name": "choice 2", 27 | "confidence": 0.945, 28 | "customMetrics": [ 29 | { 30 | "name": "customMetric1", 31 | "value": 0.5 32 | }, 33 | { 34 | "name": "customMetric2", 35 | "value": 0.3 36 | } 37 | ] 38 | } 39 | ], 40 | "name": "classification b", 41 | "dataRow": { 42 | "id": "ckrb1sf1i1g7i0ybcdc6oc8ct" 43 | }, 44 | "uuid": "d009925d-91a3-4f67-abd9-753453f5a584" 45 | }, 46 | { 47 | "answer": "a value", 48 | "name": "classification c", 49 | "dataRow": { 50 | "id": "ckrb1sf1i1g7i0ybcdc6oc8ct" 51 | }, 52 | "uuid": "150d60de-30af-44e4-be20-55201c533312" 53 | } 54 | ] 55 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/conversation_entity_import.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "location": { 3 | "start": 67, 4 | "end": 128 5 | }, 6 | "messageId": "some-message-id", 7 | "uuid": "5ad9c52f-058d-49c8-a749-3f20b84f8cd4", 8 | "dataRow": { 9 | "id": "cl6xnv9h61fv0085yhtoq06ht" 10 | }, 11 | "name": "some-text-entity", 12 | "schemaId": "cl6xnuwt95lqq07330tbb3mfd", 13 | "classifications": [], 14 | "confidence": 0.53, 15 | "customMetrics": [ 16 | { 17 | "name": "customMetric1", 18 | "value": 0.5 19 | }, 20 | { 21 | "name": "customMetric2", 22 | "value": 0.3 23 | } 24 | ] 25 | }] 26 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/conversation_entity_without_confidence_import.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "location": { 3 | "start": 67, 4 | "end": 128 5 | }, 6 | "messageId": "some-message-id", 7 | "uuid": "5ad9c52f-058d-49c8-a749-3f20b84f8cd4", 8 | "dataRow": { 9 | "id": "cl6xnv9h61fv0085yhtoq06ht" 10 | }, 11 | "name": "some-text-entity", 12 | "schemaId": "cl6xnuwt95lqq07330tbb3mfd", 13 | "classifications": [] 14 | }] 15 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/custom_confusion_matrix_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7672", 4 | "dataRow": { 5 | "id": "ckrmdnqj4000007msh9p2a27r" 6 | }, 7 | "metricValue": [ 8 | 1, 9 | 1, 10 | 2, 11 | 3 12 | ], 13 | "metricName": "50%_iou", 14 | "featureName": "sample_class", 15 | "subclassName": "sample_subclass", 16 | "aggregation": "CONFUSION_MATRIX" 17 | }, 18 | { 19 | "uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7673", 20 | "dataRow": { 21 | "id": "ckrmdnqj4000007msh9p2a27r" 22 | }, 23 | "metricValue": [ 24 | 0, 25 | 1, 26 | 2, 27 | 5 28 | ], 29 | "metricName": "50%_iou", 30 | "featureName": "sample_class", 31 | "aggregation": "CONFUSION_MATRIX" 32 | }, 33 | { 34 | "uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7674", 35 | "dataRow": { 36 | "id": "ckrmdnqj4000007msh9p2a27r" 37 | }, 38 | "metricValue": { 39 | "0.1": [ 40 | 0, 41 | 1, 42 | 2, 43 | 3 44 | ], 45 | "0.2": [ 46 | 5, 47 | 3, 48 | 4, 49 | 3 50 | ] 51 | }, 52 | "metricName": "50%_iou", 53 | "aggregation": "CONFUSION_MATRIX" 54 | } 55 | ] 56 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/custom_scalar_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7672", 4 | "dataRow": { 5 | "id": "ckrmdnqj4000007msh9p2a27r" 6 | }, 7 | "metricValue": 0.1, 8 | "metricName": "custom_iou", 9 | "featureName": "sample_class", 10 | "subclassName": "sample_subclass", 11 | "aggregation": "SUM" 12 | }, 13 | { 14 | "uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7673", 15 | "dataRow": { 16 | "id": "ckrmdnqj4000007msh9p2a27r" 17 | }, 18 | "metricValue": 0.1, 19 | "metricName": "custom_iou", 20 | "featureName": "sample_class", 21 | "aggregation": "SUM" 22 | }, 23 | { 24 | "uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7674", 25 | "dataRow": { 26 | "id": "ckrmdnqj4000007msh9p2a27r" 27 | }, 28 | "metricValue": { 29 | "0.1": 0.1, 30 | "0.2": 0.5 31 | }, 32 | "metricName": "custom_iou", 33 | "aggregation": "SUM" 34 | } 35 | ] 36 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/image_import_name_only.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "uuid": "b862c586-8614-483c-b5e6-82810f70cac0", 4 | "name": "ckrazcueb16og0z6609jj7y3y", 5 | "dataRow": { 6 | "id": "ckrazctum0z8a0ybc0b0o0g0v" 7 | }, 8 | "classifications": [], 9 | "confidence": 0.851, 10 | "customMetrics": [ 11 | { 12 | "name": "customMetric1", 13 | "value": 0.4 14 | } 15 | ], 16 | "bbox": { 17 | "top": 1352.0, 18 | "left": 2275.0, 19 | "height": 350.0, 20 | "width": 139.0 21 | } 22 | }, 23 | { 24 | "uuid": "751fc725-f7b6-48ed-89b0-dd7d94d08af6", 25 | "name": "ckrazcuec16ok0z66f956apb7", 26 | "dataRow": { 27 | "id": "ckrazctum0z8a0ybc0b0o0g0v" 28 | }, 29 | "classifications": [], 30 | "confidence": 0.834, 31 | "customMetrics": [ 32 | { 33 | "name": "customMetric1", 34 | "value": 0.3 35 | } 36 | ], 37 | "mask": { 38 | "instanceURI": "https://storage.labelbox.com/ckqcx1czn06830y61gh9v02cs%2F3e729327-f038-f66c-186e-45e921ef9717-1?Expires=1626806874672&KeyName=labelbox-assets-key-3&Signature=YsUOGKrsqmAZ68vT9BlPJOaRyLY" 39 | } 40 | }, 41 | { 42 | "classifications": [], 43 | "uuid": "43d719ac-5d7f-4aea-be00-2ebfca0900fd", 44 | "name": "ckrazcuec16oi0z66dzrd8pfl", 45 | "dataRow": { 46 | "id": "ckrazctum0z8a0ybc0b0o0g0v" 47 | }, 48 | "confidence": 0.986, 49 | "customMetrics": [ 50 | { 51 | "name": "customMetric1", 52 | "value": 0.9 53 | } 54 | ], 55 | "polygon": [ 56 | { 57 | "x": 10.0, 58 | "y": 20.0 59 | }, 60 | { 61 | "x": 15.0, 62 | "y": 20.0 63 | }, 64 | { 65 | "x": 20.0, 66 | "y": 25.0 67 | }, 68 | { 69 | "x": 10.0, 70 | "y": 20.0 71 | } 72 | ] 73 | }, 74 | { 75 | "classifications": [], 76 | "uuid": "b98f3a45-3328-41a0-9077-373a8177ebf2", 77 | "name": "ckrazcuec16om0z66bhhh4tp7", 78 | "dataRow": { 79 | "id": "ckrazctum0z8a0ybc0b0o0g0v" 80 | }, 81 | "point": { 82 | "x": 2122.0, 83 | "y": 1457.0 84 | } 85 | } 86 | ] -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/metric_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "uuid": "a22bbf6e-b2da-4abe-9a11-df84759f7672", 4 | "dataRow": { 5 | "id": "ckrmdnqj4000007msh9p2a27r" 6 | }, 7 | "metricValue": 0.1, 8 | "aggregation": "ARITHMETIC_MEAN" 9 | } 10 | ] -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/mmc_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "dataRow": { 4 | "id": "cnjencjencjfencvj" 5 | }, 6 | "uuid": "c1be3a57-597e-48cb-8d8d-a852665f9e72", 7 | "name": "single-selection", 8 | "messageEvaluationTask": { 9 | "format": "message-single-selection", 10 | "data": { 11 | "messageId": "clxfzocbm00083b6v8vczsept", 12 | "parentMessageId": "clxfznjb800073b6v43ppx9ca", 13 | "modelConfigName": "GPT 5" 14 | } 15 | } 16 | }, 17 | { 18 | "dataRow": { 19 | "id": "cfcerfvergerfefj" 20 | }, 21 | "uuid": "gferf3a57-597e-48cb-8d8d-a8526fefe72", 22 | "name": "multi-selection", 23 | "messageEvaluationTask": { 24 | "format": "message-multi-selection", 25 | "data": { 26 | "parentMessageId": "clxfznjb800073b6v43ppx9ca", 27 | "selectedMessages": [ 28 | { 29 | "messageId": "clxfzocbm00083b6v8vczsept", 30 | "modelConfigName": "GPT 5" 31 | } 32 | ] 33 | } 34 | } 35 | }, 36 | { 37 | "dataRow": { 38 | "id": "cwefgtrgrthveferfferffr" 39 | }, 40 | "uuid": "hybe3a57-5gt7e-48tgrb-8d8d-a852dswqde72", 41 | "name": "ranking", 42 | "messageEvaluationTask": { 43 | "format": "message-ranking", 44 | "data": { 45 | "parentMessageId": "clxfznjb800073b6v43ppx9ca", 46 | "rankedMessages": [ 47 | { 48 | "messageId": "clxfzocbm00083b6v8vczsept", 49 | "modelConfigName": "GPT 4 with temperature 0.7", 50 | "order": 1 51 | }, 52 | { 53 | "messageId": "clxfzocbm00093b6vx4ndisub", 54 | "modelConfigName": "GPT 5", 55 | "order": 2 56 | } 57 | ] 58 | } 59 | } 60 | } 61 | ] -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/polyline_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "line": [ 4 | { 5 | "x": 2534.353, 6 | "y": 249.471 7 | }, 8 | { 9 | "x": 2429.492, 10 | "y": 182.092 11 | }, 12 | { 13 | "x": 2294.322, 14 | "y": 221.962 15 | } 16 | ], 17 | "uuid": "5ad9c52f-058d-49c8-a749-3f20b84f8cd4", 18 | "dataRow": { 19 | "id": "cl6xnv9h61fv0085yhtoq06ht" 20 | }, 21 | "name": "some-line", 22 | "schemaId": "cl6xnuwt95lqq07330tbb3mfd", 23 | "classifications": [], 24 | "confidence": 0.58, 25 | "customMetrics": [ 26 | { 27 | "name": "customMetric1", 28 | "value": 0.5 29 | }, 30 | { 31 | "name": "customMetric2", 32 | "value": 0.3 33 | } 34 | ] 35 | } 36 | ] -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/polyline_without_confidence_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "line": [ 4 | { 5 | "x": 2534.353, 6 | "y": 249.471 7 | }, 8 | { 9 | "x": 2429.492, 10 | "y": 182.092 11 | }, 12 | { 13 | "x": 2294.322, 14 | "y": 221.962 15 | } 16 | ], 17 | "uuid": "5ad9c52f-058d-49c8-a749-3f20b84f8cd4", 18 | "dataRow": { 19 | "id": "cl6xnv9h61fv0085yhtoq06ht" 20 | }, 21 | "name": "some-line", 22 | "schemaId": "cl6xnuwt95lqq07330tbb3mfd", 23 | "classifications": [] 24 | } 25 | ] -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/rectangle_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "dataRow": { 4 | "id": "ckrb1sf1i1g7i0ybcdc6oc8ct" 5 | }, 6 | "name": "bbox", 7 | "uuid": "c1be3a57-597e-48cb-8d8d-a852665f9e72", 8 | "classifications": [], 9 | "bbox": { 10 | "top": 28.0, 11 | "left": 38.0, 12 | "height": 41.0, 13 | "width": 43.0 14 | } 15 | } 16 | ] -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/text_entity_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "location": { 4 | "start": 67, 5 | "end": 128 6 | }, 7 | "uuid": "5ad9c52f-058d-49c8-a749-3f20b84f8cd4", 8 | "dataRow": { 9 | "id": "cl6xnv9h61fv0085yhtoq06ht" 10 | }, 11 | "name": "some-text-entity", 12 | "schemaId": "cl6xnuwt95lqq07330tbb3mfd", 13 | "classifications": [], 14 | "confidence": 0.53, 15 | "customMetrics": [ 16 | { 17 | "name": "customMetric1", 18 | "value": 0.5 19 | }, 20 | { 21 | "name": "customMetric2", 22 | "value": 0.3 23 | } 24 | ] 25 | } 26 | ] -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/ndjson/text_entity_without_confidence_import.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "location": { 4 | "start": 67, 5 | "end": 128 6 | }, 7 | "uuid": "5ad9c52f-058d-49c8-a749-3f20b84f8cd4", 8 | "dataRow": { 9 | "id": "cl6xnv9h61fv0085yhtoq06ht" 10 | }, 11 | "name": "some-text-entity", 12 | "schemaId": "cl6xnuwt95lqq07330tbb3mfd", 13 | "classifications": [] 14 | } 15 | ] -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/point.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/point.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/polygon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/polygon.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/assets/rectangle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/assets/rectangle.png -------------------------------------------------------------------------------- /libs/labelbox/tests/data/export/streamable/test_export_model_run_streamable.py: -------------------------------------------------------------------------------- 1 | from labelbox import ExportTask, StreamType 2 | 3 | 4 | class TestExportModelRun: 5 | def test_export(self, model_run_with_data_rows): 6 | model_run, labels = model_run_with_data_rows 7 | label_ids = [label.uid for label in labels] 8 | expected_data_rows = list(model_run.model_run_data_rows()) 9 | 10 | task_name = "TestExportModelRun:test_export" 11 | params = {"media_attributes": True, "predictions": True} 12 | export_task = model_run.export(task_name, params=params) 13 | assert export_task.name == task_name 14 | export_task.wait_till_done() 15 | 16 | assert export_task.status == "COMPLETE" 17 | assert isinstance(export_task, ExportTask) 18 | assert export_task.has_result() 19 | assert export_task.has_errors() is False 20 | assert ( 21 | export_task.get_total_file_size(stream_type=StreamType.RESULT) > 0 22 | ) 23 | assert export_task.get_total_lines( 24 | stream_type=StreamType.RESULT 25 | ) == len(expected_data_rows) 26 | 27 | for data in export_task.get_buffered_stream(): 28 | obj = data.json 29 | assert ( 30 | "media_attributes" in obj 31 | and obj["media_attributes"] is not None 32 | ) 33 | exported_model_run = obj["experiments"][model_run.model_id]["runs"][ 34 | model_run.uid 35 | ] 36 | task_label_ids_set = set( 37 | map(lambda label: label["id"], exported_model_run["labels"]) 38 | ) 39 | task_prediction_ids_set = set( 40 | map( 41 | lambda prediction: prediction["id"], 42 | exported_model_run["predictions"], 43 | ) 44 | ) 45 | for label_id in task_label_ids_set: 46 | assert label_id in label_ids 47 | for prediction_id in task_prediction_ids_set: 48 | assert prediction_id in label_ids 49 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/metrics/confusion_matrix/test_confusion_matrix_feature.py: -------------------------------------------------------------------------------- 1 | from pytest_cases import fixture_ref 2 | from pytest_cases import parametrize, fixture_ref 3 | 4 | from labelbox.data.metrics.confusion_matrix.confusion_matrix import ( 5 | feature_confusion_matrix_metric, 6 | ) 7 | 8 | 9 | @parametrize( 10 | "tool_examples", 11 | [ 12 | fixture_ref("polygon_pairs"), 13 | fixture_ref("rectangle_pairs"), 14 | fixture_ref("mask_pairs"), 15 | fixture_ref("line_pairs"), 16 | fixture_ref("point_pairs"), 17 | fixture_ref("ner_pairs"), 18 | ], 19 | ) 20 | def test_overlapping_objects(tool_examples): 21 | for example in tool_examples: 22 | for include_subclasses, expected_attr_name in [ 23 | [True, "expected"], 24 | [False, "expected_without_subclasses"], 25 | ]: 26 | metrics = feature_confusion_matrix_metric( 27 | example.ground_truths, 28 | example.predictions, 29 | include_subclasses=include_subclasses, 30 | ) 31 | 32 | metrics = {r.feature_name: list(r.value) for r in metrics} 33 | if len(getattr(example, expected_attr_name)) == 0: 34 | assert len(metrics) == 0 35 | else: 36 | assert metrics == getattr( 37 | example, expected_attr_name 38 | ), f"{example.predictions},{example.ground_truths}" 39 | 40 | 41 | @parametrize( 42 | "tool_examples", 43 | [fixture_ref("checklist_pairs"), fixture_ref("radio_pairs")], 44 | ) 45 | def test_overlapping_classifications(tool_examples): 46 | for example in tool_examples: 47 | metrics = feature_confusion_matrix_metric( 48 | example.ground_truths, example.predictions 49 | ) 50 | 51 | metrics = {r.feature_name: list(r.value) for r in metrics} 52 | if len(example.expected) == 0: 53 | assert len(metrics) == 0 54 | else: 55 | assert ( 56 | metrics == example.expected 57 | ), f"{example.predictions},{example.ground_truths}" 58 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/metrics/iou/feature/test_feature_iou.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | from labelbox.data.metrics.iou.iou import miou_metric, feature_miou_metric 4 | 5 | 6 | def check_iou(pair): 7 | one_metrics = miou_metric(pair.predictions, pair.ground_truths) 8 | metrics = feature_miou_metric(pair.predictions, pair.ground_truths) 9 | result = {metric.feature_name: metric.value for metric in metrics} 10 | assert len(set(pair.expected.keys()).difference(set(result.keys()))) == 0 11 | 12 | for key in result: 13 | assert math.isclose(result[key], pair.expected[key]) 14 | 15 | for metric in metrics: 16 | assert metric.metric_name == "custom_iou" 17 | 18 | if len(pair.expected): 19 | assert len(one_metrics) 20 | one_metric = one_metrics[0] 21 | assert one_metric.value == sum(list(pair.expected.values())) / len( 22 | pair.expected 23 | ) 24 | 25 | 26 | def test_different_classes(different_classes): 27 | for pair in different_classes: 28 | check_iou(pair) 29 | 30 | 31 | def test_empty_annotations(empty_annotations): 32 | for pair in empty_annotations: 33 | check_iou(pair) 34 | 35 | 36 | def test_one_overlap_classes(one_overlap_class): 37 | for pair in one_overlap_class: 38 | check_iou(pair) 39 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/serialization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/serialization/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/tests/data/serialization/ndjson/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/data/serialization/ndjson/__init__.py -------------------------------------------------------------------------------- /libs/labelbox/tests/data/serialization/ndjson/test_data_gen.py: -------------------------------------------------------------------------------- 1 | import labelbox.types as lb_types 2 | from labelbox.data.serialization import NDJsonConverter 3 | 4 | 5 | def test_serialize_label(): 6 | prompt_text_annotation = lb_types.PromptClassificationAnnotation( 7 | feature_schema_id="ckrb1sfkn099c0y910wbo0p1a", 8 | name="test", 9 | extra={"uuid": "test"}, 10 | value=lb_types.PromptText( 11 | answer="the answer to the text questions right here" 12 | ), 13 | ) 14 | 15 | prompt_text_ndjson = { 16 | "answer": "the answer to the text questions right here", 17 | "name": "test", 18 | "schemaId": "ckrb1sfkn099c0y910wbo0p1a", 19 | "dataRow": {"id": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, 20 | "uuid": "test", 21 | } 22 | 23 | data_gen_label = lb_types.Label( 24 | data={"uid": "ckrb1sf1i1g7i0ybcdc6oc8ct"}, 25 | annotations=[prompt_text_annotation], 26 | ) 27 | serialized_label = next(NDJsonConverter().serialize([data_gen_label])) 28 | 29 | assert serialized_label == prompt_text_ndjson 30 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/serialization/ndjson/test_text.py: -------------------------------------------------------------------------------- 1 | from labelbox.data.annotation_types.annotation import ClassificationAnnotation 2 | from labelbox.data.annotation_types.classification.classification import ( 3 | Text, 4 | ) 5 | from labelbox.data.annotation_types.data import GenericDataRowData 6 | from labelbox.data.annotation_types.label import Label 7 | 8 | from labelbox.data.serialization.ndjson.converter import NDJsonConverter 9 | 10 | 11 | def test_serialization(): 12 | label = Label( 13 | uid="ckj7z2q0b0000jx6x0q2q7q0d", 14 | data=GenericDataRowData( 15 | uid="bkj7z2q0b0000jx6x0q2q7q0d", 16 | ), 17 | annotations=[ 18 | ClassificationAnnotation( 19 | name="radio_question_geo", 20 | confidence=0.5, 21 | value=Text(answer="first_radio_answer"), 22 | ) 23 | ], 24 | ) 25 | 26 | serialized = NDJsonConverter.serialize([label]) 27 | res = next(serialized) 28 | assert ( 29 | "confidence" not in res 30 | ) # because confidence needs to be set on the annotation itself 31 | assert res["name"] == "radio_question_geo" 32 | assert res["answer"] == "first_radio_answer" 33 | assert res["dataRow"]["id"] == "bkj7z2q0b0000jx6x0q2q7q0d" 34 | -------------------------------------------------------------------------------- /libs/labelbox/tests/data/test_prefetch_generator.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from labelbox.data.generator import PrefetchGenerator 3 | from random import random 4 | 5 | 6 | class ChildClassGenerator(PrefetchGenerator): 7 | def __init__(self, examples, num_executors=1): 8 | super().__init__(data=examples, num_executors=num_executors) 9 | 10 | def _process(self, value): 11 | num = random() 12 | if num < 0.2: 13 | raise ValueError("Randomized value error") 14 | return value 15 | 16 | 17 | amount = (i for i in range(50)) 18 | 19 | 20 | def test_single_thread_generator(): 21 | generator = ChildClassGenerator(amount, num_executors=1) 22 | 23 | with pytest.raises(ValueError): 24 | for _ in range(51): 25 | next(generator) 26 | 27 | 28 | def test_multi_thread_generator(): 29 | generator = ChildClassGenerator(amount, num_executors=4) 30 | 31 | with pytest.raises(ValueError): 32 | for _ in range(51): 33 | next(generator) 34 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/media/cat.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/integration/media/cat.mp4 -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/media/sample_image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/integration/media/sample_image.jpg -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/media/sample_pdf.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Labelbox/labelbox-python/1784891816475a2c033af1c42501a2707e5c88a7/libs/labelbox/tests/integration/media/sample_pdf.pdf -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/snapshots/test_data_rows.test_data_row_bulk_creation_with_same_global_keys.failed_data_rows.json: -------------------------------------------------------------------------------- 1 | [{"message": "Data rows contain duplicate global keys", "failedDataRows": [{"externalId": "tests/integration/media/sample_image.jpg", "globalKey": "", "rowData": "", "attachmentInputs": []}, {"externalId": "tests/integration/media/sample_image.jpg", "globalKey": "", "rowData": "", "attachmentInputs": []}]}] -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/snapshots/test_task.test_task_errors.errors.json: -------------------------------------------------------------------------------- 1 | "Failed to validate the metadata '[{\"schemaId\":\"cko8s9r5v0001h2dk9elqdidh\",\"value\":\"some msg\"},{\"schemaId\":\"cko8s9r5v0001h2dk9elqdidh\",\"value\":\"some msg 2\"}]': A schemaId can only be specified once per DataRow for CustomMetadataString" -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/snapshots/test_task.test_task_errors.failed_data_rows.json: -------------------------------------------------------------------------------- 1 | [{"message": "Failed to validate the metadata '[{\"schemaId\":\"cko8s9r5v0001h2dk9elqdidh\",\"value\":\"some msg\"},{\"schemaId\":\"cko8s9r5v0001h2dk9elqdidh\",\"value\":\"some msg 2\"}]': A schemaId can only be specified once per DataRow for CustomMetadataString", "failedDataRows": [{"rowData": "", "attachmentInputs": [], "metadataFields": [{"schemaId": "cko8s9r5v0001h2dk9elqdidh", "value": "some msg"}, {"schemaId": "cko8s9r5v0001h2dk9elqdidh", "value": "some msg 2"}]}]}] -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/snapshots/test_task.test_task_success_json.json: -------------------------------------------------------------------------------- 1 | {"id": "DUMMY_ID", "external_id": null, "row_data": "https://dummy.url", "global_key": null} -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_batches.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | 4 | from labelbox import Project, Dataset 5 | 6 | 7 | def test_create_batches(project: Project, big_dataset_data_row_ids: List[str]): 8 | task = project.create_batches( 9 | "test-batch", big_dataset_data_row_ids, priority=3 10 | ) 11 | 12 | task.wait_till_done() 13 | assert task.errors() is None 14 | batches = task.result() 15 | 16 | assert len(batches) == 1 17 | assert batches[0].name == "test-batch0000" 18 | assert batches[0].size == len(big_dataset_data_row_ids) 19 | 20 | 21 | def test_create_batches_from_dataset(project: Project, big_dataset: Dataset): 22 | export_task = big_dataset.export() 23 | export_task.wait_till_done() 24 | stream = export_task.get_buffered_stream() 25 | data_rows = [dr.json["data_row"]["id"] for dr in stream] 26 | project._wait_until_data_rows_are_processed(data_rows, [], 300) 27 | 28 | task = project.create_batches_from_dataset( 29 | "test-batch", big_dataset.uid, priority=3 30 | ) 31 | 32 | task.wait_till_done() 33 | assert task.errors() is None 34 | batches = task.result() 35 | 36 | assert len(batches) == 1 37 | assert batches[0].name == "test-batch0000" 38 | assert batches[0].size == len(data_rows) 39 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_benchmark.py: -------------------------------------------------------------------------------- 1 | def test_benchmark(configured_project_with_label): 2 | project, _, data_row, label = configured_project_with_label 3 | assert set(project.benchmarks()) == set() 4 | assert label.is_benchmark_reference is False 5 | 6 | benchmark = label.create_benchmark() 7 | assert set(project.benchmarks()) == {benchmark} 8 | assert benchmark.reference_label() == label 9 | # Refresh label data to check it's benchmark reference 10 | label = list(data_row.labels())[0] 11 | assert label.is_benchmark_reference is True 12 | 13 | benchmark.delete() 14 | assert set(project.benchmarks()) == set() 15 | # Refresh label data to check it's benchmark reference 16 | label = list(data_row.labels())[0] 17 | assert label.is_benchmark_reference is False 18 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_data_upload.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | def test_file_upload(client, rand_gen, dataset): 5 | data = rand_gen(str) 6 | uri = client.upload_data(data.encode()) 7 | data_row = dataset.create_data_row(row_data=uri) 8 | assert requests.get(data_row.row_data).text == data 9 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_dates.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta, timezone 2 | 3 | 4 | def test_dates(project): 5 | assert isinstance(project.created_at, datetime) 6 | assert isinstance(project.updated_at, datetime) 7 | 8 | project.update(setup_complete=datetime.now()) 9 | assert isinstance(project.setup_complete, datetime) 10 | 11 | 12 | def test_utc_conversion(project): 13 | assert isinstance(project.created_at, datetime) 14 | assert project.created_at.tzinfo == timezone.utc 15 | 16 | # Update with a datetime without TZ info. 17 | project.update(setup_complete=datetime.now()) 18 | # Check that the server-side, UTC date is the same as local date 19 | # converted locally to UTC. 20 | diff = project.setup_complete - datetime.now().astimezone(timezone.utc) 21 | assert abs(diff) < timedelta(minutes=1) 22 | 23 | # Update with a datetime with TZ info 24 | tz = timezone(timedelta(hours=6)) # +6 timezone 25 | project.update(setup_complete=datetime.now(timezone.utc).replace(tzinfo=tz)) 26 | diff = datetime.now(timezone.utc) - project.setup_complete.replace( 27 | tzinfo=timezone.utc 28 | ) 29 | assert diff > timedelta(hours=5, minutes=58) 30 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_embedding.py: -------------------------------------------------------------------------------- 1 | import json 2 | import random 3 | import threading 4 | from tempfile import NamedTemporaryFile 5 | from typing import Any, Dict, List 6 | 7 | import lbox.exceptions 8 | import pytest 9 | 10 | from labelbox import Client, DataRow, Dataset 11 | from labelbox.schema.embedding import Embedding 12 | 13 | 14 | def test_get_embedding_by_id(client: Client, embedding: Embedding): 15 | e = client.get_embedding_by_id(embedding.id) 16 | assert e.id == embedding.id 17 | 18 | e = client.get_embedding_by_name(embedding.name) 19 | assert e.name == embedding.name 20 | 21 | embeddings = client.get_embeddings() 22 | assert len(embeddings) > 0 23 | 24 | 25 | def test_get_embedding_by_name_not_found(client: Client): 26 | with pytest.raises(lbox.exceptions.ResourceNotFoundError): 27 | client.get_embedding_by_name("does-not-exist") 28 | 29 | 30 | @pytest.mark.parametrize("data_rows", [10], indirect=True) 31 | def test_import_vectors_from_file( 32 | data_rows: List[DataRow], embedding: Embedding 33 | ): 34 | vector = [random.uniform(1.0, 2.0) for _ in range(embedding.dims)] 35 | event = threading.Event() 36 | 37 | def callback(_: Dict[str, Any]): 38 | event.set() 39 | 40 | with NamedTemporaryFile(mode="w+") as fp: 41 | lines = [ 42 | json.dumps({"id": dr.uid, "vector": vector}) for dr in data_rows 43 | ] 44 | fp.writelines(lines) 45 | fp.flush() 46 | embedding.import_vectors_from_file(fp.name, callback) 47 | 48 | assert event.wait(10.0) # seconds 49 | 50 | 51 | def test_get_imported_vector_count(dataset: Dataset, embedding: Embedding): 52 | assert embedding.get_imported_vector_count() == 0 53 | 54 | vector = [random.uniform(1.0, 2.0) for _ in range(embedding.dims)] 55 | dataset.create_data_row( 56 | row_data="foo", 57 | embeddings=[{"embedding_id": embedding.id, "vector": vector}], 58 | ) 59 | 60 | assert embedding.get_imported_vector_count() == 1 61 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_ephemeral.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | 4 | 5 | @pytest.mark.skipif( 6 | not os.environ.get("LABELBOX_TEST_ENVIRON") == "ephemeral", 7 | reason="This test only runs in EPHEMERAL environment", 8 | ) 9 | def test_org_and_user_setup(client, ephmeral_client): 10 | assert type(client) is ephmeral_client 11 | assert client.admin_client 12 | assert client.api_key != client.admin_client.api_key 13 | 14 | organization = client.get_organization() 15 | assert organization 16 | user = client.get_user() 17 | assert user 18 | 19 | 20 | @pytest.mark.skipif( 21 | os.environ.get("LABELBOX_TEST_ENVIRON") == "ephemeral", 22 | reason="This test does not run in EPHEMERAL environment", 23 | ) 24 | def test_integration_client(client, integration_client): 25 | assert type(client) is integration_client 26 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_labeling_frontend.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from lbox.exceptions import OperationNotSupportedException 3 | 4 | from labelbox import LabelingFrontend 5 | 6 | 7 | def test_get_labeling_frontends(client): 8 | filtered_frontends = list( 9 | client._get_labeling_frontends(where=LabelingFrontend.name == "Editor") 10 | ) 11 | assert len(filtered_frontends) 12 | 13 | 14 | def test_labeling_frontend_connecting_to_project(project): 15 | client = project.client 16 | default_labeling_frontend = next( 17 | client._get_labeling_frontends(where=LabelingFrontend.name == "Editor") 18 | ) 19 | 20 | assert project.labeling_frontend() is None 21 | 22 | project.labeling_frontend.connect(default_labeling_frontend) 23 | assert project.labeling_frontend() == default_labeling_frontend 24 | 25 | with pytest.raises(OperationNotSupportedException): 26 | project.labeling_frontend.disconnect(default_labeling_frontend) 27 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_legacy_project.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pydantic import ValidationError 3 | 4 | from labelbox.schema.media_type import MediaType 5 | 6 | 7 | def test_project_name_parameter(client, rand_gen): 8 | with pytest.raises( 9 | ValidationError, match="project name must be a valid string" 10 | ): 11 | client.create_project(name=" ", media_type=MediaType.Image) 12 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_model_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from lbox.exceptions import ResourceNotFoundError 3 | 4 | 5 | def test_create_model_config(client, valid_model_id): 6 | model_config = client.create_model_config( 7 | "model_config", valid_model_id, {"param": "value"} 8 | ) 9 | assert model_config.inference_params["param"] == "value" 10 | assert model_config.name == "model_config" 11 | assert model_config.model_id == valid_model_id 12 | 13 | 14 | def test_delete_model_config(client, valid_model_id): 15 | model_config_id = client.create_model_config( 16 | "model_config", valid_model_id, {"param": "value"} 17 | ) 18 | assert client.delete_model_config(model_config_id.uid) 19 | 20 | 21 | def test_delete_nonexistant_model_config(client): 22 | with pytest.raises(ResourceNotFoundError): 23 | client.delete_model_config("invalid_model_id") 24 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_offline_chat_evaluation_project.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def test_create_offline_chat_evaluation_project( 5 | rand_gen, 6 | offline_chat_evaluation_project, 7 | chat_evaluation_ontology, 8 | offline_conversational_data_row, 9 | model_config, 10 | ): 11 | project = offline_chat_evaluation_project 12 | assert project 13 | 14 | ontology = chat_evaluation_ontology 15 | project.connect_ontology(ontology) 16 | 17 | assert project.labeling_frontend().name == "Editor" 18 | assert project.ontology().name == ontology.name 19 | 20 | batch = project.create_batch( 21 | rand_gen(str), 22 | [offline_conversational_data_row.uid], # sample of data row objects 23 | ) 24 | assert batch 25 | 26 | # Can not add a model config to an offline chat evaluation project, since we do not use live models 27 | with pytest.raises(Exception): 28 | project.add_model_config(model_config.uid) 29 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_pagination.py: -------------------------------------------------------------------------------- 1 | from copy import copy 2 | 3 | import pytest 4 | 5 | from labelbox.schema.dataset import Dataset 6 | 7 | 8 | @pytest.fixture 9 | def data_for_dataset_order_test(client, rand_gen): 10 | name = rand_gen(str) 11 | dataset1 = client.create_dataset(name=name) 12 | dataset2 = client.create_dataset(name=name) 13 | 14 | yield name 15 | 16 | dataset1.delete() 17 | dataset2.delete() 18 | 19 | 20 | def test_get_one_and_many_dataset_order(client, data_for_dataset_order_test): 21 | name = data_for_dataset_order_test 22 | 23 | paginator = client.get_datasets(where=Dataset.name == name) 24 | # confirm get_one returns first dataset 25 | all_datasets = list(paginator) 26 | assert len(all_datasets) == 2 27 | get_one_dataset = copy(paginator).get_one() 28 | assert get_one_dataset.uid == all_datasets[0].uid 29 | 30 | # confirm get_many(1) returns first dataset 31 | get_many_datasets = copy(paginator).get_many(1) 32 | assert get_many_datasets[0].uid == all_datasets[0].uid 33 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_project_set_model_setup_complete.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from lbox.exceptions import LabelboxError, OperationNotAllowedException 3 | 4 | 5 | def test_live_chat_evaluation_project( 6 | live_chat_evaluation_project, model_config 7 | ): 8 | project = live_chat_evaluation_project 9 | 10 | project.set_project_model_setup_complete() 11 | assert bool(project.model_setup_complete) is True 12 | 13 | with pytest.raises( 14 | expected_exception=LabelboxError, 15 | match="Cannot create model config for project because model setup is complete", 16 | ): 17 | project.add_model_config(model_config.uid) 18 | 19 | 20 | def test_live_chat_evaluation_project_delete_cofig( 21 | live_chat_evaluation_project, model_config 22 | ): 23 | project = live_chat_evaluation_project 24 | project_model_config_id = project.add_model_config(model_config.uid) 25 | assert project_model_config_id 26 | 27 | project_model_config = None 28 | for pmg in project.project_model_configs(): 29 | if pmg.uid == project_model_config_id: 30 | project_model_config = pmg 31 | break 32 | assert project_model_config 33 | 34 | project.set_project_model_setup_complete() 35 | assert bool(project.model_setup_complete) is True 36 | 37 | with pytest.raises( 38 | expected_exception=LabelboxError, 39 | match="Cannot create model config for project because model setup is complete", 40 | ): 41 | project_model_config.delete() 42 | 43 | 44 | def test_offline_chat_evaluation_project( 45 | offline_chat_evaluation_project, model_config 46 | ): 47 | project = offline_chat_evaluation_project 48 | 49 | with pytest.raises( 50 | expected_exception=OperationNotAllowedException, 51 | match="Only live model chat evaluation projects can complete model setup", 52 | ): 53 | project.set_project_model_setup_complete() 54 | 55 | 56 | def test_any_other_project(project, model_config): 57 | with pytest.raises( 58 | expected_exception=OperationNotAllowedException, 59 | match="Only live model chat evaluation projects can complete model setup", 60 | ): 61 | project.set_project_model_setup_complete() 62 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_response_creation_project.py: -------------------------------------------------------------------------------- 1 | from labelbox.schema.project import Project 2 | import pytest 3 | 4 | from labelbox.schema.ontology_kind import OntologyKind 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "prompt_response_ontology", [OntologyKind.ResponseCreation], indirect=True 9 | ) 10 | def test_create_response_creation_project( 11 | client, 12 | rand_gen, 13 | response_creation_project, 14 | prompt_response_ontology, 15 | response_data_row, 16 | ): 17 | project: Project = response_creation_project 18 | assert project 19 | 20 | ontology = prompt_response_ontology 21 | project.connect_ontology(ontology) 22 | 23 | assert project.labeling_frontend().name == "Editor" 24 | assert project.ontology().name == ontology.name 25 | 26 | batch = project.create_batch( 27 | rand_gen(str), 28 | [response_data_row.uid], # sample of data row objects 29 | ) 30 | assert batch 31 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_sorting.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from labelbox import Project 4 | 5 | 6 | @pytest.mark.xfail(reason="Sorting not supported on top-level fetches") 7 | def test_top_level_sorting(client): 8 | client.get_projects(order_by=Project.name.asc) 9 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_toggle_mal.py: -------------------------------------------------------------------------------- 1 | def test_enable_model_assisted_labeling(project): 2 | response = project.enable_model_assisted_labeling() 3 | assert response is True 4 | 5 | response = project.enable_model_assisted_labeling(True) 6 | assert response is True 7 | 8 | response = project.enable_model_assisted_labeling(False) 9 | assert response is False 10 | -------------------------------------------------------------------------------- /libs/labelbox/tests/integration/test_user_and_org.py: -------------------------------------------------------------------------------- 1 | from labelbox.schema.project import Project 2 | 3 | 4 | def test_user(client): 5 | user = client.get_user() 6 | assert user.uid is not None 7 | assert user.organization() == client.get_organization() 8 | 9 | 10 | def test_organization(client): 11 | organization = client.get_organization() 12 | assert organization.uid is not None 13 | assert client.get_user() in set(organization.users()) 14 | 15 | 16 | def test_user_and_org_projects(client, project): 17 | user = client.get_user() 18 | org = client.get_organization() 19 | user_project = user.projects(where=Project.uid == project.uid) 20 | org_project = org.projects(where=Project.uid == project.uid) 21 | 22 | assert user_project 23 | assert org_project 24 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/schema/test_user_group_v2.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | 3 | import pytest 4 | 5 | from labelbox.schema.user_group_v2 import Member, UserGroupV2 6 | 7 | 8 | @pytest.fixture 9 | def client(): 10 | return MagicMock() 11 | 12 | 13 | def test_parse_members_csv_empty(client): 14 | group = UserGroupV2(client) 15 | assert group._parse_members_csv("") == [] 16 | assert group._parse_members_csv("\n") == [] 17 | 18 | 19 | def test_parse_members_csv_header_only(client): 20 | group = UserGroupV2(client) 21 | assert group._parse_members_csv("email\n") == [] 22 | 23 | 24 | def test_parse_members_csv_single_member(client): 25 | group = UserGroupV2(client) 26 | result = group._parse_members_csv("email\ntest@example.com") 27 | assert len(result) == 1 28 | assert isinstance(result[0], Member) 29 | assert result[0].email == "test@example.com" 30 | 31 | 32 | def test_parse_members_csv_multiple_members(client): 33 | group = UserGroupV2(client) 34 | csv_data = "email\ntest1@example.com\ntest2@example.com\ntest3@example.com" 35 | result = group._parse_members_csv(csv_data) 36 | assert len(result) == 3 37 | assert [m.email for m in result] == [ 38 | "test1@example.com", 39 | "test2@example.com", 40 | "test3@example.com", 41 | ] 42 | 43 | 44 | def test_parse_members_csv_handles_whitespace(client): 45 | group = UserGroupV2(client) 46 | csv_data = "email\n test@example.com \n\nother@example.com\n" 47 | result = group._parse_members_csv(csv_data) 48 | assert len(result) == 2 49 | assert result[0].email == "test@example.com" 50 | assert result[1].email == "other@example.com" 51 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_client.py: -------------------------------------------------------------------------------- 1 | from labelbox.client import Client 2 | 3 | 4 | # @patch.dict(os.environ, {'LABELBOX_API_KEY': 'bar'}) 5 | def test_headers(): 6 | client = Client(api_key="api_key", endpoint="http://localhost:8080/_gql") 7 | assert client.headers 8 | assert client.headers["Authorization"] == "Bearer api_key" 9 | assert client.headers["Content-Type"] == "application/json" 10 | assert client.headers["User-Agent"] 11 | assert client.headers["X-Python-Version"] 12 | 13 | 14 | def test_enable_experimental(): 15 | client = Client(api_key="api_key", enable_experimental=True) 16 | assert client.enable_experimental 17 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_exceptions.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from lbox.exceptions import error_message_for_unparsed_graphql_error 3 | 4 | 5 | @pytest.mark.parametrize( 6 | "exception_message, expected_result", 7 | [ 8 | ( 9 | "Unparsed errors on query execution: [{'message': 'Cannot create model config for project because model setup is complete'}]", 10 | "Cannot create model config for project because model setup is complete", 11 | ), 12 | ("blah blah blah", "Unknown error"), 13 | ], 14 | ) 15 | def test_client_unparsed_exception_messages(exception_message, expected_result): 16 | assert ( 17 | error_message_for_unparsed_graphql_error(exception_message) 18 | == expected_result 19 | ) 20 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_label_data_type.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from labelbox.data.annotation_types.data.generic_data_row_data import ( 3 | GenericDataRowData, 4 | ) 5 | from labelbox.data.annotation_types.label import Label 6 | 7 | 8 | def test_generic_data_type(): 9 | data = { 10 | "global_key": "https://lb-test-data.s3.us-west-1.amazonaws.com/image-samples/sample-image-1.jpg-BEidMVWRmyXjVCnr", 11 | } 12 | label = Label(data=data) 13 | data = label.data 14 | assert isinstance(data, GenericDataRowData) 15 | assert ( 16 | data.global_key 17 | == "https://lb-test-data.s3.us-west-1.amazonaws.com/image-samples/sample-image-1.jpg-BEidMVWRmyXjVCnr" 18 | ) 19 | 20 | 21 | def test_generic_data_type_validations(): 22 | data = { 23 | "row_data": "https://lb-test-data.s3.us-west-1.amazonaws.com/image-samples/sample-image-1.jpg-BEidMVWRmyXjVCnr", 24 | } 25 | with pytest.raises(ValueError, match="Exactly one of"): 26 | Label(data=data) 27 | 28 | data = { 29 | "uid": "abcd", 30 | "global_key": "https://lb-test-data.s3.us-west-1.amazonaws.com/image-samples/sample-image-1.jpg-BEidMVWRmyXjVCnr", 31 | } 32 | with pytest.raises(ValueError, match="Only one of"): 33 | Label(data=data) 34 | 35 | 36 | def test_generic_data_row(): 37 | data = { 38 | "global_key": "https://lb-test-data.s3.us-west-1.amazonaws.com/image-samples/sample-image-1.jpg-BEidMVWRmyXjVCnr", 39 | } 40 | label = Label(data=GenericDataRowData(**data)) 41 | data = label.data 42 | assert isinstance(data, GenericDataRowData) 43 | assert ( 44 | data.global_key 45 | == "https://lb-test-data.s3.us-west-1.amazonaws.com/image-samples/sample-image-1.jpg-BEidMVWRmyXjVCnr" 46 | ) 47 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_ndjson_parsing.py: -------------------------------------------------------------------------------- 1 | import ast 2 | from io import StringIO 3 | 4 | from labelbox import parser 5 | 6 | 7 | def test_loads(ndjson_content): 8 | expected_line, expected_objects = ndjson_content 9 | parsed_line = parser.loads(expected_line) 10 | 11 | assert parsed_line == expected_objects 12 | assert parser.dumps(parsed_line) == expected_line 13 | 14 | 15 | def test_loads_bytes(ndjson_content): 16 | expected_line, expected_objects = ndjson_content 17 | 18 | bytes_line = expected_line.encode("utf-8") 19 | parsed_line = parser.loads(bytes_line) 20 | 21 | assert parsed_line == expected_objects 22 | assert parser.dumps(parsed_line) == expected_line 23 | 24 | 25 | def test_reader_stringio(ndjson_content): 26 | line, ndjson_objects = ndjson_content 27 | 28 | text_io = StringIO(line) 29 | parsed_arr = [] 30 | reader = parser.reader(text_io) 31 | for _, r in enumerate(reader): 32 | parsed_arr.append(r) 33 | assert parsed_arr == ndjson_objects 34 | 35 | 36 | def test_non_ascii_new_line(ndjson_content_with_nonascii_and_line_breaks): 37 | line, expected_objects = ndjson_content_with_nonascii_and_line_breaks 38 | parsed = parser.loads(line) 39 | 40 | assert parsed == expected_objects 41 | 42 | # NOTE: json parser converts unicode chars to unicode literals by default and this is a good practice 43 | # but it is not what we want here since we want to compare the strings with actual unicode chars 44 | assert ast.literal_eval("'" + parser.dumps(parsed) + "'") == line 45 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_case_change.py: -------------------------------------------------------------------------------- 1 | from labelbox import utils 2 | 3 | SNAKE = "this_is_a_string" 4 | TITLE = "ThisIsAString" 5 | CAMEL = "thisIsAString" 6 | MIXED = "this_Is_AString" 7 | 8 | 9 | def test_camel(): 10 | assert utils.camel_case(SNAKE) == CAMEL 11 | assert utils.camel_case(TITLE) == CAMEL 12 | assert utils.camel_case(CAMEL) == CAMEL 13 | assert utils.camel_case(MIXED) == CAMEL 14 | 15 | 16 | def test_snake(): 17 | assert utils.snake_case(SNAKE) == SNAKE 18 | assert utils.snake_case(TITLE) == SNAKE 19 | assert utils.snake_case(CAMEL) == SNAKE 20 | assert utils.snake_case(MIXED) == SNAKE 21 | 22 | 23 | def test_title(): 24 | assert utils.title_case(SNAKE) == TITLE 25 | assert utils.title_case(TITLE) == TITLE 26 | assert utils.title_case(CAMEL) == TITLE 27 | assert utils.title_case(MIXED) == TITLE 28 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_delete_batch_data_row_metadata.py: -------------------------------------------------------------------------------- 1 | from labelbox.schema.data_row_metadata import _DeleteBatchDataRowMetadata 2 | from labelbox.schema.identifiable import GlobalKey, UniqueId 3 | 4 | 5 | def test_dict_delete_data_row_batch(): 6 | obj = _DeleteBatchDataRowMetadata( 7 | data_row_identifier=UniqueId("abcd"), 8 | schema_ids=["clqh77tyk000008l2a9mjesa1", "clqh784br000008jy0yuq04fy"], 9 | ) 10 | assert obj.model_dump() == { 11 | "data_row_identifier": {"id": "abcd", "id_type": "ID"}, 12 | "schema_ids": [ 13 | "clqh77tyk000008l2a9mjesa1", 14 | "clqh784br000008jy0yuq04fy", 15 | ], 16 | } 17 | 18 | obj = _DeleteBatchDataRowMetadata( 19 | data_row_identifier=GlobalKey("fegh"), 20 | schema_ids=["clqh77tyk000008l2a9mjesa1", "clqh784br000008jy0yuq04fy"], 21 | ) 22 | assert obj.model_dump() == { 23 | "data_row_identifier": {"id": "fegh", "id_type": "GKEY"}, 24 | "schema_ids": [ 25 | "clqh77tyk000008l2a9mjesa1", 26 | "clqh784br000008jy0yuq04fy", 27 | ], 28 | } 29 | 30 | 31 | def test_dict_delete_data_row_batch_by_alias(): 32 | obj = _DeleteBatchDataRowMetadata( 33 | data_row_identifier=UniqueId("abcd"), 34 | schema_ids=["clqh77tyk000008l2a9mjesa1", "clqh784br000008jy0yuq04fy"], 35 | ) 36 | assert obj.model_dump(by_alias=True) == { 37 | "dataRowIdentifier": {"id": "abcd", "idType": "ID"}, 38 | "schemaIds": ["clqh77tyk000008l2a9mjesa1", "clqh784br000008jy0yuq04fy"], 39 | } 40 | 41 | obj = _DeleteBatchDataRowMetadata( 42 | data_row_identifier=GlobalKey("fegh"), 43 | schema_ids=["clqh77tyk000008l2a9mjesa1", "clqh784br000008jy0yuq04fy"], 44 | ) 45 | assert obj.model_dump(by_alias=True) == { 46 | "dataRowIdentifier": {"id": "fegh", "idType": "GKEY"}, 47 | "schemaIds": ["clqh77tyk000008l2a9mjesa1", "clqh784br000008jy0yuq04fy"], 48 | } 49 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_descriptor_file_creator.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from unittest.mock import MagicMock 4 | 5 | from labelbox.schema.internal.descriptor_file_creator import ( 6 | DescriptorFileCreator, 7 | ) 8 | 9 | 10 | def test_chunk_down_by_bytes_row_too_large(): 11 | client = MagicMock() 12 | 13 | descriptor_file_creator = DescriptorFileCreator(client) 14 | 15 | chunk = [{"row_data": "a"}] 16 | max_chunk_size_bytes = 1 17 | 18 | res = descriptor_file_creator._chunk_down_by_bytes( 19 | chunk, max_chunk_size_bytes 20 | ) 21 | assert [x for x in res] == [json.dumps([{"row_data": "a"}])] 22 | 23 | 24 | def test_chunk_down_by_bytes_more_chunks(): 25 | client = MagicMock() 26 | 27 | descriptor_file_creator = DescriptorFileCreator(client) 28 | 29 | chunk = [{"row_data": "a"}, {"row_data": "b"}] 30 | max_chunk_size_bytes = len(json.dumps(chunk).encode("utf-8")) - 1 31 | 32 | res = descriptor_file_creator._chunk_down_by_bytes( 33 | chunk, max_chunk_size_bytes 34 | ) 35 | assert [x for x in res] == [ 36 | json.dumps([{"row_data": "a"}]), 37 | json.dumps([{"row_data": "b"}]), 38 | ] 39 | 40 | 41 | def test_chunk_down_by_bytes_one_chunk(): 42 | client = MagicMock() 43 | 44 | descriptor_file_creator = DescriptorFileCreator(client) 45 | 46 | chunk = [{"row_data": "a"}, {"row_data": "b"}] 47 | max_chunk_size_bytes = len(json.dumps(chunk).encode("utf-8")) 48 | 49 | res = descriptor_file_creator._chunk_down_by_bytes( 50 | chunk, max_chunk_size_bytes 51 | ) 52 | assert [x for x in res] == [ 53 | json.dumps([{"row_data": "a"}, {"row_data": "b"}]) 54 | ] 55 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_entity_meta.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from labelbox.orm.model import Relationship 4 | from labelbox.orm.db_object import DbObject 5 | 6 | 7 | def test_illegal_cache_cond1(): 8 | class TestEntityA(DbObject): 9 | test_entity_b = Relationship.ToOne("TestEntityB", cache=True) 10 | 11 | with pytest.raises(TypeError) as exc_info: 12 | 13 | class TestEntityB(DbObject): 14 | another_entity = Relationship.ToOne("AnotherEntity", cache=True) 15 | 16 | assert ( 17 | "`test_entity_a` caches `test_entity_b` which caches `['another_entity']`" 18 | in str(exc_info.value) 19 | ) 20 | 21 | 22 | def test_illegal_cache_cond2(): 23 | class TestEntityD(DbObject): 24 | another_entity = Relationship.ToOne("AnotherEntity", cache=True) 25 | 26 | with pytest.raises(TypeError) as exc_info: 27 | 28 | class TestEntityC(DbObject): 29 | test_entity_d = Relationship.ToOne("TestEntityD", cache=True) 30 | 31 | assert ( 32 | "`test_entity_c` caches `test_entity_d` which caches `['another_entity']`" 33 | in str(exc_info.value) 34 | ) 35 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_export_filters.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | 3 | import pytest 4 | 5 | from labelbox.schema.export_filters import build_filters 6 | 7 | 8 | def test_ids_filter(): 9 | client = MagicMock() 10 | filters = {"data_row_ids": ["id1", "id2"], "batch_ids": ["b1", "b2"]} 11 | assert build_filters(client, filters) == [ 12 | { 13 | "ids": ["id1", "id2"], 14 | "operator": "is", 15 | "type": "data_row_id", 16 | }, 17 | { 18 | "ids": ["b1", "b2"], 19 | "operator": "is", 20 | "type": "batch", 21 | }, 22 | ] 23 | 24 | 25 | def test_ids_empty_filter(): 26 | client = MagicMock() 27 | filters = {"data_row_ids": [], "batch_ids": ["b1", "b2"]} 28 | with pytest.raises( 29 | ValueError, match="data_row_id filter expects a non-empty list." 30 | ): 31 | build_filters(client, filters) 32 | 33 | 34 | def test_global_keys_filter(): 35 | client = MagicMock() 36 | filters = {"global_keys": ["id1", "id2"]} 37 | assert build_filters(client, filters) == [ 38 | { 39 | "ids": ["id1", "id2"], 40 | "operator": "is", 41 | "type": "global_key", 42 | } 43 | ] 44 | 45 | 46 | def test_validations(): 47 | client = MagicMock() 48 | filters = { 49 | "global_keys": ["id1", "id2"], 50 | "data_row_ids": ["id1", "id2"], 51 | } 52 | with pytest.raises( 53 | ValueError, 54 | match="data_rows and global_keys cannot both be present in export filters", 55 | ): 56 | build_filters(client, filters) 57 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_identifiables.py: -------------------------------------------------------------------------------- 1 | from labelbox.schema.identifiables import GlobalKeys, UniqueIds 2 | 3 | 4 | def test_unique_ids(): 5 | ids = ["a", "b", "c"] 6 | identifiables = UniqueIds(ids) 7 | assert [i for i in identifiables] == ids 8 | assert identifiables.id_type == "ID" 9 | assert len(identifiables) == 3 10 | 11 | 12 | def test_global_keys(): 13 | ids = ["a", "b", "c"] 14 | identifiables = GlobalKeys(ids) 15 | assert [i for i in identifiables] == ids 16 | assert identifiables.id_type == "GKEY" 17 | assert len(identifiables) == 3 18 | 19 | 20 | def test_index_access(): 21 | ids = ["a", "b", "c"] 22 | identifiables = GlobalKeys(ids) 23 | assert identifiables[0] == "a" 24 | assert identifiables[1:3] == GlobalKeys(["b", "c"]) 25 | 26 | 27 | def test_repr(): 28 | ids = ["a", "b", "c"] 29 | identifiables = GlobalKeys(ids) 30 | assert repr(identifiables) == "GlobalKeys(['a', 'b', 'c'])" 31 | 32 | ids = ["a", "b", "c"] 33 | identifiables = UniqueIds(ids) 34 | assert repr(identifiables) == "UniqueIds(['a', 'b', 'c'])" 35 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_label_import.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import pytest 3 | from unittest.mock import MagicMock, patch 4 | 5 | from labelbox.schema.annotation_import import LabelImport, logger 6 | 7 | 8 | def test_should_warn_user_about_unsupported_confidence(): 9 | """this test should check running state only to validate running, not completed""" 10 | id = str(uuid.uuid4()) 11 | 12 | labels = [ 13 | { 14 | "uuid": "b862c586-8614-483c-b5e6-82810f70cac0", 15 | "schemaId": "ckrazcueb16og0z6609jj7y3y", 16 | "dataRow": {"id": "ckrazctum0z8a0ybc0b0o0g0v"}, 17 | "confidence": 0.851, 18 | "bbox": {"top": 1352, "left": 2275, "height": 350, "width": 139}, 19 | }, 20 | ] 21 | with patch.object(LabelImport, "_create_label_import_from_bytes"): 22 | with patch.object(logger, "warning") as warning_mock: 23 | LabelImport.create_from_objects( 24 | client=MagicMock(), project_id=id, name=id, labels=labels 25 | ) 26 | warning_mock.assert_called_once() 27 | "Confidence scores are not supported in Label Import" in warning_mock.call_args_list[ 28 | 0 29 | ].args[0] 30 | 31 | 32 | def test_invalid_labels_format(): 33 | """this test should confirm that labels are required to be in a form of list""" 34 | id = str(uuid.uuid4()) 35 | 36 | label = { 37 | "uuid": "b862c586-8614-483c-b5e6-82810f70cac0", 38 | "schemaId": "ckrazcueb16og0z6609jj7y3y", 39 | "dataRow": {"id": "ckrazctum0z8a0ybc0b0o0g0v"}, 40 | "bbox": {"top": 1352, "left": 2275, "height": 350, "width": 139}, 41 | } 42 | with patch.object(LabelImport, "_create_label_import_from_bytes"): 43 | with pytest.raises(TypeError): 44 | LabelImport.create_from_objects( 45 | client=MagicMock(), project_id=id, name=id, labels=label 46 | ) 47 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_ontology_kind.py: -------------------------------------------------------------------------------- 1 | from labelbox.schema.ontology_kind import ( 2 | OntologyKind, 3 | EditorTaskType, 4 | EditorTaskTypeMapper, 5 | ) 6 | from labelbox.schema.media_type import MediaType 7 | 8 | 9 | def test_ontology_kind_conversions_from_editor_task_type(): 10 | ontology_kind = OntologyKind.ModelEvaluation 11 | media_type = MediaType.Conversational 12 | editor_task_type = EditorTaskTypeMapper.to_editor_task_type( 13 | ontology_kind, media_type 14 | ) 15 | assert editor_task_type == EditorTaskType.ModelChatEvaluation 16 | 17 | ontology_kind = OntologyKind.Missing 18 | media_type = MediaType.Image 19 | editor_task_type = EditorTaskTypeMapper.to_editor_task_type( 20 | ontology_kind, media_type 21 | ) 22 | assert editor_task_type == EditorTaskType.Missing 23 | 24 | ontology_kind = OntologyKind.ModelEvaluation 25 | media_type = MediaType.Video 26 | editor_task_type = EditorTaskTypeMapper.to_editor_task_type( 27 | ontology_kind, media_type 28 | ) 29 | assert editor_task_type == EditorTaskType.Missing 30 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_project_validate_labeling_parameter_overrides.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from unittest.mock import MagicMock 3 | 4 | from labelbox.schema.data_row import DataRow 5 | from labelbox.schema.identifiable import GlobalKey, UniqueId 6 | from labelbox.schema.project import validate_labeling_parameter_overrides 7 | 8 | 9 | def test_validate_labeling_parameter_overrides_invalid_priority(): 10 | mock_data_row = MagicMock(spec=DataRow) 11 | mock_data_row.uid = "abc" 12 | data = [ 13 | (mock_data_row, "invalid"), 14 | (UniqueId("efg"), 2), 15 | (GlobalKey("hij"), 3), 16 | ] 17 | with pytest.raises(TypeError): 18 | validate_labeling_parameter_overrides(data) 19 | 20 | 21 | def test_validate_labeling_parameter_overrides_invalid_tuple_length(): 22 | mock_data_row = MagicMock(spec=DataRow) 23 | mock_data_row.uid = "abc" 24 | data = [ 25 | (mock_data_row, "invalid"), 26 | (UniqueId("efg"), 2), 27 | (GlobalKey("hij")), 28 | ] 29 | with pytest.raises(TypeError): 30 | validate_labeling_parameter_overrides(data) 31 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_rand_gen.py: -------------------------------------------------------------------------------- 1 | def test_gen_str(rand_gen): 2 | assert len({rand_gen(str) for _ in range(100)}) == 100 3 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_step_ontology_variants.py: -------------------------------------------------------------------------------- 1 | from labelbox.schema.tool_building.base_step_reasoning_tool import _Variant 2 | 3 | 4 | def test_variant(): 5 | variant = _Variant( 6 | id=0, name="Correct", _available_actions={"regenerateSteps"} 7 | ) 8 | variant.set_actions(["regenerateSteps"]) 9 | assert variant.asdict() == { 10 | "id": 0, 11 | "name": "Correct", 12 | "actions": ["regenerateSteps"], 13 | } 14 | 15 | assert variant._available_actions == {"regenerateSteps"} 16 | variant.reset_actions() 17 | assert variant.asdict() == { 18 | "id": 0, 19 | "name": "Correct", 20 | "actions": [], 21 | } 22 | 23 | 24 | def test_variant_actions(): 25 | variant = _Variant( 26 | id=0, name="Correct", _available_actions={"regenerateSteps"} 27 | ) 28 | variant.set_actions(["regenerateSteps"]) 29 | assert variant.actions == ["regenerateSteps"] 30 | variant.set_actions(["invalidAction"]) 31 | assert variant.actions == [] 32 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_unit_webhook.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | import pytest 3 | 4 | from labelbox import Webhook 5 | 6 | 7 | def test_webhook_create_with_no_secret(rand_gen): 8 | client = MagicMock() 9 | project = MagicMock() 10 | secret = "" 11 | url = "https:/" + rand_gen(str) 12 | topics = [] 13 | 14 | with pytest.raises(ValueError) as exc_info: 15 | Webhook.create(client, topics, url, secret, project) 16 | assert str(exc_info.value) == "Secret must be a non-empty string." 17 | 18 | 19 | def test_webhook_create_with_no_topics(rand_gen): 20 | client = MagicMock() 21 | project = MagicMock() 22 | secret = rand_gen(str) 23 | url = "https:/" + rand_gen(str) 24 | topics = [] 25 | 26 | with pytest.raises(ValueError) as exc_info: 27 | Webhook.create(client, topics, url, secret, project) 28 | assert str(exc_info.value) == "Topics must be a non-empty list." 29 | 30 | 31 | def test_webhook_create_with_no_url(rand_gen): 32 | client = MagicMock() 33 | project = MagicMock() 34 | secret = rand_gen(str) 35 | url = "" 36 | topics = [Webhook.Topic.LABEL_CREATED, Webhook.Topic.LABEL_DELETED] 37 | 38 | with pytest.raises(ValueError) as exc_info: 39 | Webhook.create(client, topics, url, secret, project) 40 | assert str(exc_info.value) == "URL must be a non-empty string." 41 | -------------------------------------------------------------------------------- /libs/labelbox/tests/unit/test_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from labelbox.utils import ( 3 | format_iso_datetime, 4 | format_iso_from_string, 5 | sentence_case, 6 | ) 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "datetime_str, expected_datetime_str", 11 | [ 12 | ("2011-11-04T00:05:23Z", "2011-11-04T00:05:23Z"), 13 | ("2011-11-04T00:05:23+00:00", "2011-11-04T00:05:23Z"), 14 | ("2011-11-04T00:05:23+05:00", "2011-11-03T19:05:23Z"), 15 | ("2011-11-04T00:05:23", "2011-11-04T00:05:23Z"), 16 | ], 17 | ) 18 | def test_datetime_parsing(datetime_str, expected_datetime_str): 19 | # NOTE I would normally not take 'expected' using another function from sdk code, but in this case this is exactly the usage in _validate_parse_datetime 20 | assert ( 21 | format_iso_datetime(format_iso_from_string(datetime_str)) 22 | == expected_datetime_str 23 | ) 24 | 25 | 26 | @pytest.mark.parametrize( 27 | "str, expected_str", 28 | [ 29 | ("AUDIO", "Audio"), 30 | ("LLM_PROMPT_RESPONSE_CREATION", "Llm prompt response creation"), 31 | ], 32 | ) 33 | def test_sentence_case(str, expected_str): 34 | assert sentence_case(str) == expected_str 35 | -------------------------------------------------------------------------------- /libs/labelbox/tests/utils.py: -------------------------------------------------------------------------------- 1 | def remove_keys_recursive(d, keys): 2 | for k in keys: 3 | if k in d: 4 | del d[k] 5 | for k, v in d.items(): 6 | if isinstance(v, dict): 7 | remove_keys_recursive(v, keys) 8 | elif isinstance(v, list): 9 | for i in v: 10 | if isinstance(i, dict): 11 | remove_keys_recursive(i, keys) 12 | 13 | 14 | # NOTE this uses quite a primitive check for cuids but I do not think it is worth coming up with a better one 15 | # Also this function is NOT written with performance in mind, good for small to mid size dicts like we have in our test 16 | def rename_cuid_key_recursive(d): 17 | new_key = "" 18 | for k in list(d.keys()): 19 | if len(k) == 25 and not k.isalpha(): # primitive check for cuid 20 | d[new_key] = d.pop(k) 21 | for k, v in d.items(): 22 | if isinstance(v, dict): 23 | rename_cuid_key_recursive(v) 24 | elif isinstance(v, list): 25 | for i in v: 26 | if isinstance(i, dict): 27 | rename_cuid_key_recursive(i) 28 | 29 | 30 | INTEGRATION_SNAPSHOT_DIRECTORY = "tests/integration/snapshots" 31 | -------------------------------------------------------------------------------- /libs/lbox-clients/.gitignore: -------------------------------------------------------------------------------- 1 | # python generated files 2 | __pycache__/ 3 | *.py[oc] 4 | build/ 5 | dist/ 6 | wheels/ 7 | *.egg-info 8 | 9 | # venv 10 | .venv 11 | -------------------------------------------------------------------------------- /libs/lbox-clients/.python-version: -------------------------------------------------------------------------------- 1 | 3.9.18 2 | -------------------------------------------------------------------------------- /libs/lbox-clients/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/ucyo/python-package-template/blob/master/Dockerfile 2 | FROM python:3.9-slim as rye 3 | 4 | ENV LANG="C.UTF-8" \ 5 | LC_ALL="C.UTF-8" \ 6 | PATH="/home/python/.local/bin:/home/python/.rye/shims:$PATH" \ 7 | PIP_NO_CACHE_DIR="false" \ 8 | RYE_VERSION="0.43.0" \ 9 | RYE_INSTALL_OPTION="--yes" \ 10 | LABELBOX_TEST_ENVIRON="prod" 11 | 12 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 13 | ca-certificates \ 14 | curl \ 15 | inotify-tools \ 16 | make \ 17 | # cv2 18 | libsm6 \ 19 | libxext6 \ 20 | ffmpeg \ 21 | libfontconfig1 \ 22 | libxrender1 \ 23 | libgl1-mesa-glx \ 24 | libgeos-dev \ 25 | gcc \ 26 | && rm -rf /var/lib/apt/lists/* 27 | 28 | RUN groupadd --gid 1000 python && \ 29 | useradd --uid 1000 --gid python --shell /bin/bash --create-home python 30 | 31 | USER 1000 32 | WORKDIR /home/python/ 33 | 34 | RUN curl -sSf https://rye.astral.sh/get | bash - 35 | 36 | COPY --chown=python:python . /home/python/labelbox-python/ 37 | WORKDIR /home/python/labelbox-python 38 | 39 | RUN rye config --set-bool behavior.global-python=true && \ 40 | rye config --set-bool behavior.use-uv=true && \ 41 | rye pin 3.9 && \ 42 | rye sync 43 | 44 | CMD rye run unit && rye integration -------------------------------------------------------------------------------- /libs/lbox-clients/README.md: -------------------------------------------------------------------------------- 1 | # lbox-clients 2 | 3 | This is an example module which can be cloned and reused to develop modules under the `lbox` namespace. 4 | 5 | ## Module Status: Experimental 6 | 7 | **TLDR: This module may be removed or altered at any given time and there is no offical support.** 8 | 9 | Please see [here](https://docs.labelbox.com/docs/product-release-phases) for the formal definition of `Experimental`. -------------------------------------------------------------------------------- /libs/lbox-clients/src/lbox/call_info.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import re 3 | import sys 4 | from typing import TypedDict 5 | 6 | 7 | def python_version_info(): 8 | version_info = sys.version_info 9 | 10 | return f"{version_info.major}.{version_info.minor}.{version_info.micro}-{version_info.releaselevel}" 11 | 12 | 13 | LABELBOX_CALL_PATTERN = re.compile(r"/labelbox/") 14 | TEST_FILE_PATTERN = re.compile(r".*test.*\.py$") 15 | 16 | 17 | class _RequestInfo(TypedDict): 18 | prefix: str 19 | class_name: str 20 | method_name: str 21 | 22 | 23 | def call_info(): 24 | method_name: str = "Unknown" 25 | prefix = "" 26 | class_name = "" 27 | skip_methods = ["wrapper", "__init__", "execute"] 28 | skip_classes = ["PaginatedCollection", "_CursorPagination", "_OffsetPagination"] 29 | 30 | try: 31 | call_info = None 32 | for stack in reversed(inspect.stack()): 33 | if LABELBOX_CALL_PATTERN.search(stack.filename): 34 | call_info = stack 35 | method_name: str = call_info.function 36 | class_name = call_info.frame.f_locals.get( 37 | "self", None 38 | ).__class__.__name__ 39 | if method_name not in skip_methods: 40 | if class_name not in skip_classes: 41 | if TEST_FILE_PATTERN.search(call_info.filename): 42 | prefix = "test:" 43 | else: 44 | if class_name == "NoneType": 45 | class_name = "" 46 | break 47 | 48 | except Exception: 49 | pass 50 | return _RequestInfo(prefix=prefix, class_name=class_name, method_name=method_name) 51 | 52 | 53 | def call_info_as_str(): 54 | info: _RequestInfo = call_info() 55 | return f"{info['prefix']}{info['class_name']}:{info['method_name']}" 56 | -------------------------------------------------------------------------------- /libs/lbox-clients/tests/unit/lbox/test_client.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock 2 | 3 | from lbox.request_client import RequestClient 4 | 5 | 6 | # @patch.dict(os.environ, {'LABELBOX_API_KEY': 'bar'}) 7 | def test_headers(): 8 | client = RequestClient( 9 | sdk_version="foo", api_key="api_key", endpoint="http://localhost:8080/_gql" 10 | ) 11 | assert client.headers 12 | assert client.headers["Authorization"] == "Bearer api_key" 13 | assert client.headers["Content-Type"] == "application/json" 14 | assert client.headers["User-Agent"] 15 | assert client.headers["X-Python-Version"] 16 | 17 | 18 | def test_custom_error_handling(): 19 | mock_raise_error = MagicMock() 20 | 21 | response_dict = { 22 | "errors": [ 23 | { 24 | "message": "Internal server error", 25 | "extensions": {"code": "INTERNAL_SERVER_ERROR"}, 26 | } 27 | ], 28 | } 29 | response = MagicMock() 30 | response.json.return_value = response_dict 31 | response.status_code = 200 32 | 33 | client = RequestClient( 34 | sdk_version="foo", api_key="api_key", endpoint="http://localhost:8080/_gql" 35 | ) 36 | connection_mock = MagicMock() 37 | connection_mock.send.return_value = response 38 | client._connection = connection_mock 39 | 40 | client.execute( 41 | "query_str", 42 | {"projectId": "project_id"}, 43 | raise_return_resource_not_found=True, 44 | error_handlers={"INTERNAL_SERVER_ERROR": mock_raise_error}, 45 | ) 46 | mock_raise_error.assert_called_once_with(response) 47 | -------------------------------------------------------------------------------- /libs/lbox-example/.gitignore: -------------------------------------------------------------------------------- 1 | # python generated files 2 | __pycache__/ 3 | *.py[oc] 4 | build/ 5 | dist/ 6 | wheels/ 7 | *.egg-info 8 | 9 | # venv 10 | .venv 11 | -------------------------------------------------------------------------------- /libs/lbox-example/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/ucyo/python-package-template/blob/master/Dockerfile 2 | FROM python:3.9-slim as rye 3 | 4 | ENV LANG="C.UTF-8" \ 5 | LC_ALL="C.UTF-8" \ 6 | PATH="/home/python/.local/bin:/home/python/.rye/shims:$PATH" \ 7 | PIP_NO_CACHE_DIR="false" \ 8 | RYE_VERSION="0.43.0" \ 9 | RYE_INSTALL_OPTION="--yes" \ 10 | LABELBOX_TEST_ENVIRON="prod" 11 | 12 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 13 | ca-certificates \ 14 | curl \ 15 | inotify-tools \ 16 | make \ 17 | # cv2 18 | libsm6 \ 19 | libxext6 \ 20 | ffmpeg \ 21 | libfontconfig1 \ 22 | libxrender1 \ 23 | libgl1-mesa-glx \ 24 | libgeos-dev \ 25 | gcc \ 26 | && rm -rf /var/lib/apt/lists/* 27 | 28 | RUN groupadd --gid 1000 python && \ 29 | useradd --uid 1000 --gid python --shell /bin/bash --create-home python 30 | 31 | USER 1000 32 | WORKDIR /home/python/ 33 | 34 | RUN curl -sSf https://rye.astral.sh/get | bash - 35 | 36 | COPY --chown=python:python . /home/python/labelbox-python/ 37 | WORKDIR /home/python/labelbox-python 38 | 39 | RUN rye config --set-bool behavior.global-python=true && \ 40 | rye config --set-bool behavior.use-uv=true && \ 41 | rye pin 3.9 && \ 42 | rye sync 43 | 44 | CMD rye run unit && rye integration -------------------------------------------------------------------------------- /libs/lbox-example/README.md: -------------------------------------------------------------------------------- 1 | # lbox-example 2 | 3 | This is an example module which can be cloned and reused to develop modules under the `lbox` namespace. 4 | 5 | ## Module Status: Experimental 6 | 7 | **TLDR: This module may be removed or altered at any given time and there is no offical support.** 8 | 9 | Please see [here](https://docs.labelbox.com/docs/product-release-phases) for the formal definition of `Experimental`. -------------------------------------------------------------------------------- /libs/lbox-example/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "lbox-example" 3 | version = "0.1.1" 4 | description = "An example module which can be used to develop other modules under the lbox namespace" 5 | authors = [ 6 | { name = "Labelbox", email = "engineering@labelbox.com" } 7 | ] 8 | dependencies = [ 9 | "art>=6.2", 10 | ] 11 | readme = "README.md" 12 | requires-python = ">= 3.9" 13 | 14 | classifiers=[ 15 | # How mature is this project? 16 | "Development Status :: 2 - Pre-Alpha", 17 | # Indicate who your project is intended for 18 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 19 | "Topic :: Software Development :: Libraries", 20 | "Intended Audience :: Developers", 21 | "Intended Audience :: Science/Research", 22 | "Intended Audience :: Education", 23 | # Pick your license as you wish 24 | "License :: OSI Approved :: Apache Software License", 25 | # Specify the Python versions you support here. 26 | "Programming Language :: Python :: 3", 27 | "Programming Language :: Python :: 3.9", 28 | "Programming Language :: Python :: 3.10", 29 | "Programming Language :: Python :: 3.11", 30 | "Programming Language :: Python :: 3.12", 31 | ] 32 | keywords = ["ml", "ai", "labelbox", "labeling", "llm", "machinelearning", "edu"] 33 | 34 | [project.urls] 35 | Homepage = "https://labelbox.com/" 36 | Documentation = "https://labelbox-python.readthedocs.io/en/latest/" 37 | Repository = "https://github.com/Labelbox/labelbox-python" 38 | Issues = "https://github.com/Labelbox/labelbox-python/issues" 39 | Changelog = "https://github.com/Labelbox/labelbox-python/blob/develop/libs/labelbox/CHANGELOG.md" 40 | 41 | [build-system] 42 | requires = ["hatchling"] 43 | build-backend = "hatchling.build" 44 | 45 | [tool.rye] 46 | managed = true 47 | dev-dependencies = [] 48 | 49 | [tool.rye.scripts] 50 | unit = "pytest tests/unit" 51 | integration = "python -c \"import sys; sys.exit(0)\"" 52 | 53 | [tool.hatch.metadata] 54 | allow-direct-references = true 55 | 56 | [tool.hatch.build.targets.wheel] 57 | packages = ["src/lbox"] 58 | 59 | [tool.pytest.ini_options] 60 | addopts = "-rP -vvv --durations=20 --cov=lbox.example --import-mode=importlib" -------------------------------------------------------------------------------- /libs/lbox-example/src/lbox/example/art.py: -------------------------------------------------------------------------------- 1 | from art import art 2 | 3 | 4 | def coffee(amount: int = 1) -> str: 5 | """ 6 | Generate some coffee. 7 | 8 | Parameters: 9 | amount (int): The number of coffee cups to generate. Defaults to 1. 10 | 11 | Returns: 12 | str: The ASCII art representation of the coffee cups. 13 | """ 14 | ascii_art = art("coffee", number=amount) 15 | return ascii_art 16 | -------------------------------------------------------------------------------- /libs/lbox-example/tests/unit/lbox/example/test_art.py: -------------------------------------------------------------------------------- 1 | from lbox.example.art import coffee 2 | 3 | 4 | class TestArt: 5 | def test_logo(self): 6 | result = coffee() 7 | assert result == "c[_]" 8 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "labelbox-python" 3 | version = "0.1.0" 4 | description = "Labelbox Python API" 5 | authors = [{ name = "Labelbox", email = "engineering@labelbox.com" }] 6 | dependencies = [ 7 | "sphinx-multiproject>=1.0.0rc1", 8 | "sphinx>=7.1.2", 9 | "sphinx-rtd-theme>=2.0.0", 10 | ] 11 | readme = "README.md" 12 | requires-python = ">= 3.9" 13 | 14 | [tool.rye] 15 | managed = true 16 | virtual = true 17 | dev-dependencies = [ 18 | "pytest>=8.1.1", 19 | "pytest-cases>=3.8.4", 20 | "pytest-rerunfailures>=14.0", 21 | "pytest-snapshot>=0.9.0", 22 | "nbconvert>=7.16.2", 23 | "nbformat>=5.10.3", 24 | "pytest-cov>=4.1.0", 25 | "pytest-xdist>=3.5.0", 26 | "toml-cli>=0.6.0", 27 | "faker>=25.5.0", 28 | "pytest-timestamper>=0.0.10", 29 | "pytest-timeout>=2.3.1", 30 | "pytest-order>=1.2.1", 31 | "pyjwt>=2.9.0", 32 | ] 33 | 34 | [tool.rye.workspace] 35 | members = ["libs/*", "examples"] 36 | 37 | [tool.pytest.ini_options] 38 | # https://github.com/pytest-dev/pytest-rerunfailures/issues/99 39 | addopts = "-rP -vvv --reruns 1 --reruns-delay 5 --durations=20 -n auto --cov=labelbox --import-mode=importlib --order-group-scope=module" 40 | markers = """ 41 | slow: marks tests as slow (deselect with '-m "not slow"') 42 | """ 43 | 44 | [tool.rye.scripts] 45 | docs = "sphinx-build ./docs ./dist" 46 | --------------------------------------------------------------------------------