├── .gitignore ├── data-labelling-cycle-revA.png ├── .env-sample ├── modzy-labelstudio-project-header.png ├── images ├── gilly-8vzFINl6zV8-unsplash.jpg ├── haidan-IOBIgKmjm1Y-unsplash.jpg ├── eva-dang-EXdXLrZXS9Q-unsplash.jpg ├── afif-kusuma-XYQPyn4KkiY-unsplash.jpg ├── alex-knight-wfwUpfVqrKU-unsplash.jpg ├── arthur-yeti-Nn5515mz6ac-unsplash.jpg ├── ashley-byrd-7oyg0_Ry934-unsplash.jpg ├── jace-afsoon-iNBowrFZh0U-unsplash.jpg ├── john-towner-Hf4Ap1-ef40-unsplash.jpg ├── komal-brar-qlqwoY8Bc0E-unsplash.jpg ├── laura-cros-KtJy7cZV5OQ-unsplash.jpg ├── rohan-reddy-NlhwC45YdGg-unsplash.jpg ├── aquib-akhter-oBCc0Hw6LrQ-unsplash.jpg ├── dillon-shook-u7i7rGNfuNA-unsplash.jpg ├── diogo-palhais-tnzzr8HpLhs-unsplash.jpg ├── eric-francis-gCsTc_5JMVM-unsplash.jpg ├── florian-wehde--y3sidWvDxg-unsplash.jpg ├── florian-wehde-DpgujuZ92zE-unsplash.jpg ├── markus-winkler-IvCKp7SHhxI-unsplash.jpg ├── rezaul-karim-102abqkKhbY-unsplash.jpg ├── alexander-kunze-uLh71gTmZ4g-unsplash.jpg ├── nikolay-vorobyev-kuMQrO_C-to-unsplash.jpg ├── oleksandr-zhabin-9UHWO_IDaNs-unsplash.jpg ├── victor-malyushev-9zrfFC6lOEk-unsplash.jpg ├── abdullah-elhariry-wi1hwvGlPfA-unsplash.jpg ├── ljubomir-zarkovic-7jW0_pYGqSA-unsplash.jpg └── devon-janse-van-rensburg-RAAXmcYdoIg-unsplash.jpg ├── images-test ├── eva-dang-EXdXLrZXS9Q-unsplash.jpg ├── ashley-byrd-7oyg0_Ry934-unsplash.jpg ├── jace-afsoon-iNBowrFZh0U-unsplash.jpg ├── markus-winkler-IvCKp7SHhxI-unsplash.jpg └── abdullah-elhariry-wi1hwvGlPfA-unsplash.jpg ├── requirements.txt ├── .github └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── label-studio-setup ├── import-data-example.json ├── class-list.txt ├── label-classes.html └── ground-truth.json ├── inference.py ├── import-annotations.py ├── readme.md └── LICENSE.MD /.gitignore: -------------------------------------------------------------------------------- 1 | env 2 | .env 3 | .DS_Store -------------------------------------------------------------------------------- /data-labelling-cycle-revA.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/data-labelling-cycle-revA.png -------------------------------------------------------------------------------- /.env-sample: -------------------------------------------------------------------------------- 1 | export MODZY_BASE_URL= 2 | export MODZY_API_KEY= 3 | export DROPBOX_ACCESS_TOKEN= 4 | export LABEL_STUDIO_ACCESS_TOKEN= -------------------------------------------------------------------------------- /modzy-labelstudio-project-header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/modzy-labelstudio-project-header.png -------------------------------------------------------------------------------- /images/gilly-8vzFINl6zV8-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/gilly-8vzFINl6zV8-unsplash.jpg -------------------------------------------------------------------------------- /images/haidan-IOBIgKmjm1Y-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/haidan-IOBIgKmjm1Y-unsplash.jpg -------------------------------------------------------------------------------- /images/eva-dang-EXdXLrZXS9Q-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/eva-dang-EXdXLrZXS9Q-unsplash.jpg -------------------------------------------------------------------------------- /images/afif-kusuma-XYQPyn4KkiY-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/afif-kusuma-XYQPyn4KkiY-unsplash.jpg -------------------------------------------------------------------------------- /images/alex-knight-wfwUpfVqrKU-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/alex-knight-wfwUpfVqrKU-unsplash.jpg -------------------------------------------------------------------------------- /images/arthur-yeti-Nn5515mz6ac-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/arthur-yeti-Nn5515mz6ac-unsplash.jpg -------------------------------------------------------------------------------- /images/ashley-byrd-7oyg0_Ry934-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/ashley-byrd-7oyg0_Ry934-unsplash.jpg -------------------------------------------------------------------------------- /images/jace-afsoon-iNBowrFZh0U-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/jace-afsoon-iNBowrFZh0U-unsplash.jpg -------------------------------------------------------------------------------- /images/john-towner-Hf4Ap1-ef40-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/john-towner-Hf4Ap1-ef40-unsplash.jpg -------------------------------------------------------------------------------- /images/komal-brar-qlqwoY8Bc0E-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/komal-brar-qlqwoY8Bc0E-unsplash.jpg -------------------------------------------------------------------------------- /images/laura-cros-KtJy7cZV5OQ-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/laura-cros-KtJy7cZV5OQ-unsplash.jpg -------------------------------------------------------------------------------- /images/rohan-reddy-NlhwC45YdGg-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/rohan-reddy-NlhwC45YdGg-unsplash.jpg -------------------------------------------------------------------------------- /images-test/eva-dang-EXdXLrZXS9Q-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images-test/eva-dang-EXdXLrZXS9Q-unsplash.jpg -------------------------------------------------------------------------------- /images/aquib-akhter-oBCc0Hw6LrQ-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/aquib-akhter-oBCc0Hw6LrQ-unsplash.jpg -------------------------------------------------------------------------------- /images/dillon-shook-u7i7rGNfuNA-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/dillon-shook-u7i7rGNfuNA-unsplash.jpg -------------------------------------------------------------------------------- /images/diogo-palhais-tnzzr8HpLhs-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/diogo-palhais-tnzzr8HpLhs-unsplash.jpg -------------------------------------------------------------------------------- /images/eric-francis-gCsTc_5JMVM-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/eric-francis-gCsTc_5JMVM-unsplash.jpg -------------------------------------------------------------------------------- /images/florian-wehde--y3sidWvDxg-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/florian-wehde--y3sidWvDxg-unsplash.jpg -------------------------------------------------------------------------------- /images/florian-wehde-DpgujuZ92zE-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/florian-wehde-DpgujuZ92zE-unsplash.jpg -------------------------------------------------------------------------------- /images/markus-winkler-IvCKp7SHhxI-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/markus-winkler-IvCKp7SHhxI-unsplash.jpg -------------------------------------------------------------------------------- /images/rezaul-karim-102abqkKhbY-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/rezaul-karim-102abqkKhbY-unsplash.jpg -------------------------------------------------------------------------------- /images-test/ashley-byrd-7oyg0_Ry934-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images-test/ashley-byrd-7oyg0_Ry934-unsplash.jpg -------------------------------------------------------------------------------- /images-test/jace-afsoon-iNBowrFZh0U-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images-test/jace-afsoon-iNBowrFZh0U-unsplash.jpg -------------------------------------------------------------------------------- /images/alexander-kunze-uLh71gTmZ4g-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/alexander-kunze-uLh71gTmZ4g-unsplash.jpg -------------------------------------------------------------------------------- /images/nikolay-vorobyev-kuMQrO_C-to-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/nikolay-vorobyev-kuMQrO_C-to-unsplash.jpg -------------------------------------------------------------------------------- /images/oleksandr-zhabin-9UHWO_IDaNs-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/oleksandr-zhabin-9UHWO_IDaNs-unsplash.jpg -------------------------------------------------------------------------------- /images/victor-malyushev-9zrfFC6lOEk-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/victor-malyushev-9zrfFC6lOEk-unsplash.jpg -------------------------------------------------------------------------------- /images-test/markus-winkler-IvCKp7SHhxI-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images-test/markus-winkler-IvCKp7SHhxI-unsplash.jpg -------------------------------------------------------------------------------- /images/abdullah-elhariry-wi1hwvGlPfA-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/abdullah-elhariry-wi1hwvGlPfA-unsplash.jpg -------------------------------------------------------------------------------- /images/ljubomir-zarkovic-7jW0_pYGqSA-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/ljubomir-zarkovic-7jW0_pYGqSA-unsplash.jpg -------------------------------------------------------------------------------- /images-test/abdullah-elhariry-wi1hwvGlPfA-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images-test/abdullah-elhariry-wi1hwvGlPfA-unsplash.jpg -------------------------------------------------------------------------------- /images/devon-janse-van-rensburg-RAAXmcYdoIg-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/modzy/modzy-labelstudio-sample/HEAD/images/devon-janse-van-rensburg-RAAXmcYdoIg-unsplash.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cachetools==5.2.0 2 | certifi==2022.6.15 3 | charset-normalizer==2.1.1 4 | deprecation==2.1.0 5 | dropbox==11.33.0 6 | google-api-core==2.8.2 7 | google-api-python-client==2.58.0 8 | google-auth==2.11.0 9 | google-auth-httplib2==0.1.0 10 | googleapis-common-protos==1.56.4 11 | grpcio==1.48.1 12 | httplib2==0.20.4 13 | idna==3.3 14 | modzy-sdk==0.9.0.post0 15 | packaging==21.3 16 | ply==3.11 17 | protobuf==3.19.4 18 | pyasn1==0.4.8 19 | pyasn1-modules==0.2.8 20 | pyparsing==3.0.9 21 | python-dotenv==0.20.0 22 | requests==2.28.1 23 | rsa==4.9 24 | six==1.16.0 25 | stone==3.3.1 26 | uritemplate==4.1.1 27 | urllib3==1.26.12 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /label-studio-setup/import-data-example.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "data": { 3 | "image": "https://images.unsplash.com/photo-1622805206221-bbbb21e28ac3?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=1625&q=80", 4 | "predicted_value": "Predicted value: Cairo, Northern Africa (98.6% certainty) 👍: 10 👎: 2", 5 | "downvotes": 2, 6 | "explainable_url": "/operations/explainability/765d88f8-1239-41e8-854a-8b66f7ac0cca/abdullah-elhariry-wi1hwvGlPfA-unsplash.jpg" 7 | }, 8 | "predictions": [{ 9 | "model_version": "1.0.1", 10 | "score": 0.986, 11 | "result": [ 12 | { 13 | "id": "da9b3857-4e1c-4a8e-b023-18abf4cb1223", 14 | "type": "choices", 15 | "from_name": "location", "to_name": "image", 16 | "value": { 17 | "choices": ["Cairo, Northern Africa"] 18 | } 19 | }] 20 | }] 21 | }] -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /label-studio-setup/class-list.txt: -------------------------------------------------------------------------------- 1 | Beijing, Asia 2 | Bangkok, Asia 3 | Delhi, Asia 4 | Hong Kong, Asia 5 | Indonesia, Asia 6 | Tokyo, Asia 7 | Sydney, Australia 8 | Havana, Carribbean 9 | Central America 10 | Mexico City, Central America 11 | Moscow, Eastern-Europe 12 | Saint Petersburg, Eastern-Europe 13 | Amsterdam, Europe 14 | Athens, Europe 15 | Barcelona, Europe 16 | Berlin, Europe 17 | Budapest, Europe 18 | Dublin, Europe 19 | Florence, Europe 20 | Helsinki, Europe 21 | Lisbon, Europe 22 | London, Europe 23 | Madrid, Europe 24 | Paris, Europe 25 | Rome, Europe 26 | Valencia, Europe 27 | Cairo, Northern Africa 28 | Middle East 29 | Dubai, Middle East 30 | Istanbul, Middle East 31 | Chicago, North America 32 | Dallas, North America 33 | Denver, North America 34 | Los Angeles, North America 35 | New York, North America 36 | Quebec City, North America 37 | San Francisco, North America 38 | Toronto, North America 39 | Vancouver, North America 40 | Northwestern Africa 41 | Buenos Aires, South America 42 | Rio, South America 43 | Santiago, South America 44 | Sub-Saharan Africa 45 | Cape Town, Sub-Saharan Africa -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- 1 | import dropbox 2 | import os 3 | from modzy import ApiClient 4 | from modzy._util import file_to_bytes 5 | 6 | #Initializes Modzy and Dropbox clients 7 | mdz = ApiClient(base_url=os.getenv('MODZY_BASE_URL'), api_key=os.getenv("MODZY_API_KEY")) 8 | dbx = dropbox.Dropbox(os.getenv("DROPBOX_ACCESS_TOKEN")) 9 | 10 | #Runs an inference on each image with a specified folder and then saves each to Dropbox 11 | def main(): 12 | 13 | #Defines the image folder being used, this can be changed to "images" 14 | image_folder = "images-test" 15 | 16 | for filename in os.listdir(image_folder): 17 | 18 | #Gets the filepath for each image 19 | f = os.path.join(image_folder,filename) 20 | 21 | #Sends the image to Modzy for inference and retrieves an inference job ID number 22 | job_ID = modzy_inference(f,filename) 23 | 24 | #Uploads the image to Dropbox and names it after the inference job ID number 25 | upload_image(f,job_ID) 26 | 27 | print(f"Inference ID {job_ID} complete") 28 | 29 | #Submits an image to a model that performs Image-based Geolocation 30 | def modzy_inference(image_path,image_ID): 31 | 32 | #This base64 encodes the image and formats it for this model 33 | sources = {} 34 | sources[image_ID] = { 35 | "image": file_to_bytes(image_path), 36 | } 37 | 38 | ##This is Modzy's unique ID for the Image-based Geolocation model 39 | model_id = "aevbu1h3yw" 40 | 41 | ##This is the version number of the model being used 42 | model_version = "1.0.1" 43 | 44 | ##Submits the image for inference 45 | ##Adding "explain=True" enables explainability for each prediction 46 | job = mdz.jobs.submit_file(model_id, model_version, sources, explain=True) 47 | 48 | ##Returns the inference job ID 49 | return job.get("jobIdentifier") 50 | 51 | #Uploads the image to Dropbox 52 | def upload_image(image_path,image_ID): 53 | dbx.files_upload(file_to_bytes(image_path),f"/{image_ID}.jpg") 54 | 55 | if __name__ == '__main__': 56 | main() -------------------------------------------------------------------------------- /label-studio-setup/label-classes.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /import-annotations.py: -------------------------------------------------------------------------------- 1 | import os 2 | import dropbox 3 | import requests 4 | from modzy import ApiClient 5 | 6 | #Initializes Modzy and Dropbox clients 7 | mdz = ApiClient(base_url=os.getenv('MODZY_BASE_URL'), api_key=os.getenv("MODZY_API_KEY")) 8 | dbx = dropbox.Dropbox(os.getenv("DROPBOX_ACCESS_TOKEN")) 9 | 10 | def main(): 11 | 12 | #Loops through all images in a dropbox folder and adds the image names and URLs to a dict 13 | jobs = [] 14 | path_urls = {} 15 | images = dbx.files_list_folder("") 16 | for image in images.entries: 17 | job_ID = image.path_display[1:].split(".")[0] 18 | jobs.append(job_ID) 19 | path_urls[job_ID] = get_dropbox_URL(image.path_display) 20 | 21 | #Creates a dictionary with all of the inference information that will be sent to Label Studio 22 | results_data = {} 23 | for i, job in enumerate(jobs): 24 | job_details = mdz.jobs.get(job) 25 | result = mdz.results.block_until_complete(job) 26 | input_filename = list(result["results"].keys())[0] 27 | results_data[f'job_{i}'] = { 28 | "job_id": job, 29 | "input_name": input_filename, 30 | "model_version": job_details["model"]["version"], 31 | "label": result["results"][input_filename]["results.json"]["data"]["result"]["classPredictions"][0]["class"], 32 | "score": result["results"][input_filename]["results.json"]["data"]["result"]["classPredictions"][0]["score"], 33 | "num_upvotes": result["results"][input_filename]["voting"]["up"], 34 | "num_downvotes": result["results"][input_filename]["voting"]["down"] 35 | } 36 | 37 | # Creates the data object that Label Studio is expecting for our template 38 | data = [{ 39 | "data": { 40 | "image": path_urls[results_data[f'job_{i}']["job_id"]], 41 | "predicted_value": predicted_value_msg(results_data[f'job_{i}']["label"], results_data[f'job_{i}']["score"], results_data[f'job_{i}']["num_upvotes"], results_data[f'job_{i}']["num_downvotes"]), 42 | "downvotes": results_data[f'job_{i}']["num_downvotes"], 43 | "explainable_url": explainable_url(results_data[f'job_{i}']["job_id"], results_data[f'job_{i}']["input_name"]) 44 | }, 45 | "predictions": [{ 46 | "model_version": results_data[f'job_{i}']["model_version"], 47 | "score": results_data[f'job_{i}']["score"], 48 | "result": [ 49 | { 50 | "id": results_data[f'job_{i}']["job_id"], 51 | "type": "choices", 52 | "from_name": "location", 53 | "to_name": "image", 54 | "value": { 55 | "choices": [results_data[f'job_{i}']["label"]] 56 | } 57 | }] 58 | }] 59 | } for i in range(len(results_data))] 60 | 61 | #Posts pre-annotated image links and results to Label Studio 62 | labelStudioURL = 'http://localhost:8080/api/projects/1/import' 63 | authHeader = f"Token {os.getenv('LABEL_STUDIO_ACCESS_TOKEN')}" 64 | r = requests.post(labelStudioURL, json=data, headers={'Authorization': authHeader}) 65 | 66 | #Gets shareable link from dropbox 67 | def get_dropbox_URL(image_path): 68 | sharedURL = dbx.sharing_create_shared_link(image_path).url 69 | cleanURL = sharedURL.replace("dl=0","raw=1") 70 | return cleanURL 71 | 72 | #Formats the message included with each labeling task 73 | def predicted_value_msg(label, score, upvotes, downvotes): 74 | msg = f"Predicted value: {label} ({score:.1%} certainty) 👍: {upvotes} 👎: {downvotes}" 75 | return msg 76 | 77 | #Formats the URL to view the explanation of each prediction 78 | def explainable_url(job_ID, input_name): 79 | url = f"{os.getenv('MODZY_BASE_URL')}/operations/explainability/{job_ID}/{input_name}" 80 | return url 81 | 82 | if __name__ == '__main__': 83 | main() -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Modzy - Label Studio Sample Project 2 | 3 | ![Modzy-LabelStudio project banner](modzy-labelstudio-project-header.png) 4 | 5 |
6 | 7 | ![GitHub contributors](https://img.shields.io/github/contributors/modzy/modzy-labelstudio-sample?logo=github&style=flat-square) 8 | ![GitHub last commit](https://img.shields.io/github/last-commit/modzy/modzy-labelstudio-sample?logo=GitHub&style=flat-square) 9 | ![GitHub issues](https://img.shields.io/github/issues-raw/modzy/modzy-labelstudio-sample?logo=github&style=flat-square) 10 | ![GitHub](https://img.shields.io/github/license/modzy/modzy-labelstudio-sample?logo=apache&style=flat-square) 11 | 12 | ![GitHub repo size](https://img.shields.io/github/repo-size/modzy/modzy-labelstudio-sample?logo=GitHub&style=flat-square) 13 | 14 | :tv: ***Follow along on YouTube: https://youtu.be/ZOxeNHzIbfg*** 15 |
16 | 17 | This sample project demonstrates how to use Modzy and Label studio to dynamically re-label production inferences using a combination of semi-supervised learning, explainable AI, and human-in-the-loop review. This is a comprehensive and low-lift way to create training data sets from production data that can be used to retrain a machine learning model. 18 | 19 |
20 | data labelling flowchart with Modzy, Dropbox, and LabelStudio 21 |
22 | 23 | This sample project uses: 24 | 25 | Modzy logoFor model inference and explainability

26 | 27 | Dropbox logoAs a makeshift data lake for storing images 28 | 29 | LabelStudio logoFor labeling production inference results 30 | 31 | ## Installation & Set-Up 32 | 33 | ### Label Studio Set-up 34 | * First, install Label Studio and start it up. Instructions can be found [here](https://github.com/heartexlabs/label-studio#try-out-label-studio) (installing with PIP was a breeze) 35 | * Once it's running, log in and download your API [access token](https://labelstud.io/guide/api.html#Authenticate-to-the-API) 36 | * Finally, create a project in Label Studio. Then go to "Settings" > "Labeling Interface" and paste in [label-classes.html](label-studio-setup/label-classes.html) as code 37 | 38 | ### Modzy Set-up 39 | In your instance of Modzy, [create a project](https://docs.modzy.com/docs/how-to-create-a-project) and then download your project [api key](https://docs.modzy.com/docs/how-to-use-a-project). 40 | 41 | ### Dropbox Set-up 42 | * If you don't have one, create a free Dropbox account 43 | * Then, you'll need to create a [Dropbox App](https://www.dropbox.com/developers/reference/getting-started) 44 | * Next, give your app the following permissions: `files.metadata.write` `files.content.write` `files.content.read` `sharing.write` 45 | * Finally, generate an [access token](https://dropbox.tech/developers/generate-an-access-token-for-your-own-account) 46 | 47 | ### Environment Set-Up 48 | Before you can start running the sample images as inferences, you'll need to do the following: 49 | * Clone this repo 50 | * [Optional, but recommended] Create a virtual environment within your project folder and activate it 51 | * Run `$ pip install -r requirements.txt` to install all necessary dependencies 52 | * Create a copy of [.env-sample](.env-sample) and rename it to `.env`. Then update it to include your API access tokens for Modzy, Dropbox, and Label Studio 53 | * Run `$ source .env` to load your environment variables 54 | * Update the `labelStudioURL` variable in [import-annotations.py](import-annotations.py) to the URL of your instance of Label Studio 55 | 56 | ## Usage 57 | 58 | ### Send sample images to Modzy and Dropbox 59 | The next step will send the images in the [/images-test](/images-test) folder to a model that executes "Image-based Geolocation" which means that it will try to predict where in the world the image was taken. Then the image will be uploaded to Dropbox. 60 | 61 | In your terminal run: 62 | 63 | `$ python3 inference.py` 64 | 65 | ### Run import-annotations.py 66 | Finally, send all predictions generated from Modzy over to Label Studio for review and labeling. 67 | 68 | In your terminal run: 69 | 70 | `$ python3 import-annotations.py` 71 | -------------------------------------------------------------------------------- /label-studio-setup/ground-truth.json: -------------------------------------------------------------------------------- 1 | { 2 | "abdullah-elhariry-wi1hwvGlPfA-unsplash": { 3 | "image-name": "abdullah-elhariry-wi1hwvGlPfA-unsplash.jpg", 4 | "location": "Cairo, Northern Africa", 5 | "credit": "Photo by Abdullah Elhariry on Unsplash" 6 | }, 7 | "afif-kusuma-XYQPyn4KkiY-unsplash": { 8 | "image-name": "afif-kusuma-XYQPyn4KkiY-unsplash.jpg", 9 | "location": "Indonesia, Asia", 10 | "credit": "Photo by Afif Kusuma on Unsplash" 11 | }, 12 | "alex-knight-wfwUpfVqrKU-unsplash": { 13 | "image-name": "alex-knight-wfwUpfVqrKU-unsplash.jpg", 14 | "location": "Tokyo, Asia", 15 | "credit": "Photo by Alex Knight on Unsplash" 16 | }, 17 | "alexander-kunze-uLh71gTmZ4g-unsplash": { 18 | "image-name": "alexander-kunze-uLh71gTmZ4g-unsplash.jpg", 19 | "location": "Havana, Carribbean", 20 | "credit": "Photo by Alexander Kunze on Unsplash" 21 | }, 22 | "aquib-akhter-oBCc0Hw6LrQ-unsplash": { 23 | "image-name": "aquib-akhter-oBCc0Hw6LrQ-unsplash.jpg", 24 | "location": "Delhi, Asia", 25 | "credit": "Photo by Aquib-akhter on Unsplash" 26 | }, 27 | "arthur-yeti-Nn5515mz6ac-unsplash": { 28 | "image-name": "arthur-yeti-Nn5515mz6ac-unsplash.jpg", 29 | "location": "Rome, Europe", 30 | "credit": "Photo by Arthur Yeti on Unsplash" 31 | }, 32 | "ashley-byrd-7oyg0_Ry934-unsplash": { 33 | "image-name": "ashley-byrd-7oyg0_Ry934-unsplash.jpg", 34 | "location": "Dallas, North America", 35 | "credit": "Photo by Ashley Byrd on Unsplash" 36 | }, 37 | "devon-janse-van-rensburg-RAAXmcYdoIg-unsplash": { 38 | "image-name": "devon-janse-van-rensburg-RAAXmcYdoIg-unsplash.jpg", 39 | "location": "Cape Town, Sub-Saharan Africa", 40 | "credit": "Photo by Devon Janse van Rensburg on Unsplash" 41 | }, 42 | "dillon-shook-u7i7rGNfuNA-unsplash": { 43 | "image-name": "dillon-shook-u7i7rGNfuNA-unsplash.jpg", 44 | "location": "Los Angeles, North America", 45 | "credit": "Photo by Dillon Shood on Unsplash" 46 | }, 47 | "diogo-palhais-tnzzr8HpLhs-unsplash": { 48 | "image-name": "diogo-palhais-tnzzr8HpLhs-unsplash.jpg", 49 | "location": "Dublin, Europe", 50 | "credit": "Photo by Diogo Palhais on Unsplash" 51 | }, 52 | "eric-francis-gCsTc_5JMVM-unsplash": { 53 | "image-name": "eric-francis-gCsTc_5JMVM-unsplash.jpg", 54 | "location": "Denver, North America", 55 | "credit": "Photo by Eric Francis on Unsplash" 56 | }, 57 | "eva-dang-EXdXLrZXS9Q-unsplash": { 58 | "image-name": "eva-dang-EXdXLrZXS9Q-unsplash.jpg", 59 | "location": "London, Europe", 60 | "credit": "Photo by Eva Dang on Unsplash" 61 | }, 62 | "florian-wehde--y3sidWvDxg-unsplash": { 63 | "image-name": "florian-wehde--y3sidWvDxg-unsplash.jpg", 64 | "location": "Bangkok, Asia", 65 | "credit": "Photo by Florian Wehde on Unsplash" 66 | }, 67 | "florian-wehde-DpgujuZ92zE-unsplash": { 68 | "image-name": "florian-wehde-DpgujuZ92zE-unsplash.jpg", 69 | "location": "Hong Kong, Asia", 70 | "credit": "Photo by Florian Wehde on Unsplash" 71 | }, 72 | "gilly-8vzFINl6zV8-unsplash": { 73 | "image-name": "gilly-8vzFINl6zV8-unsplash.jpg", 74 | "location": "Berlin, Europe", 75 | "credit": "Photo by Gilly on Unsplash" 76 | }, 77 | "haidan-IOBIgKmjm1Y-unsplash": { 78 | "image-name": "haidan-IOBIgKmjm1Y-unsplash.jpg", 79 | "location": "Middle East", 80 | "credit": "Photo by Haidan on Unsplash" 81 | }, 82 | "jace-afsoon-iNBowrFZh0U-unsplash": { 83 | "image-name": "jace-afsoon-iNBowrFZh0U-unsplash.jpg", 84 | "location": "Amsterdam, Europe", 85 | "credit": "Photo by Jace & Afsoon on Unsplash" 86 | }, 87 | "john-towner-Hf4Ap1-ef40-unsplash": { 88 | "image-name": "john-towner-Hf4Ap1-ef40-unsplash.jpg", 89 | "location": "Paris, Europe", 90 | "credit": "Photo by John Towner on Unsplash" 91 | }, 92 | "komal-brar-qlqwoY8Bc0E-unsplash": { 93 | "image-name": "komal-brar-qlqwoY8Bc0E-unsplash.jpg", 94 | "location": "Vancouver, North America", 95 | "credit": "Photo by Komal Brar on Unsplash" 96 | }, 97 | "laura-cros-KtJy7cZV5OQ-unsplash": { 98 | "image-name": "laura-cros-KtJy7cZV5OQ-unsplash.jpg", 99 | "location": "Sydney, Australia", 100 | "credit": "Photo by Laura Cros on Unsplash" 101 | }, 102 | "ljubomir-zarkovic-7jW0_pYGqSA-unsplash": { 103 | "image-name": "ljubomir-zarkovic-7jW0_pYGqSA-unsplash.jpg", 104 | "location": "Budapest, Europe", 105 | "credit": "Photo by Ljubomir Zarkovic on Unsplash" 106 | }, 107 | "markus-winkler-IvCKp7SHhxI-unsplash": { 108 | "image-name": "markus-winkler-IvCKp7SHhxI-unsplash.jpg", 109 | "location": "Athens, Europe", 110 | "credit": "Photo by Markus Winkler on Unsplash" 111 | }, 112 | "nikolay-vorobyev-kuMQrO_C-to-unsplash": { 113 | "image-name": "nikolay-vorobyev-kuMQrO_C-to-unsplash.jpg", 114 | "location": "Moscow, Eastern-Europe", 115 | "credit": "Photo by Nikolay Vorobyev on Unsplash" 116 | }, 117 | "oleksandr-zhabin-9UHWO_IDaNs-unsplash": { 118 | "image-name": "oleksandr-zhabin-9UHWO_IDaNs-unsplash.jpg", 119 | "location": "Florence, Europe", 120 | "credit": "Photo by Oleksandr Zhabin on Unsplash" 121 | }, 122 | "rezaul-karim-102abqkKhbY-unsplash": { 123 | "image-name": "rezaul-karim-102abqkKhbY-unsplash.jpg", 124 | "location": "San Francisco, North America", 125 | "credit": "Photo by Rezaul Karim on Unsplash" 126 | }, 127 | "rohan-reddy-NlhwC45YdGg-unsplash": { 128 | "image-name": "rohan-reddy-NlhwC45YdGg-unsplash.jpg", 129 | "location": "Istanbul, Middle East", 130 | "credit": "Photo by Rohan Reddy on Unsplash" 131 | }, 132 | "victor-malyushev-9zrfFC6lOEk-unsplash": { 133 | "image-name": "victor-malyushev-9zrfFC6lOEk-unsplash.jpg", 134 | "location": "Saint Petersburg, Eastern-Europe", 135 | "credit": "Photo by Victor Malyushev on Unsplash" 136 | } 137 | } -------------------------------------------------------------------------------- /LICENSE.MD: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------