4 |
5 |
2 |
3 |
4 |
5 |
18 | 
19 |
29 | 
30 |
35 | 
36 |
42 |
43 | 
44 |
45 |
70 | 
71 |
76 | 
77 |
99 | 
100 |
Make your own Mosaic here.
26 | 27 | -------------------------------------------------------------------------------- /data_prep/download_images.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pandas as pd 4 | import requests 5 | from tqdm import tqdm 6 | 7 | batch_size = 256 8 | img_width = 225 9 | img_height = 225 10 | model = "resnet" 11 | metadata_fn = "metadata.json" 12 | data_dir = "images" 13 | 14 | os.makedirs(data_dir, exist_ok=True) 15 | metadata = pd.read_json(metadata_fn, lines=True) 16 | for i, row in tqdm(metadata.iterrows()): 17 | target_file = os.path.join(data_dir, row["id"] + ".jpg") 18 | if not os.path.exists(target_file): 19 | try: 20 | with open(target_file, 'wb') as f: 21 | f.write(requests.get(row["Thumbnail_Url"]).content) 22 | except Exception as e: 23 | print(e) 24 | -------------------------------------------------------------------------------- /data_prep/featurize_and_match.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pandas as pd 4 | import torch 5 | import torch.nn as nn 6 | from PIL import Image 7 | from torch.utils.data import Dataset 8 | from torchvision import models, transforms 9 | from tqdm import tqdm 10 | import numpy as np 11 | import pickle 12 | 13 | data_dir = 'images' 14 | metadata_fn = "metadata.json" 15 | features_dir = "features" 16 | features_file = os.path.join(features_dir, "pytorch_rn50.pkl") 17 | featurize_images = True 18 | device = torch.device("cuda:0") 19 | 20 | os.makedirs(data_dir, exist_ok=True) 21 | os.makedirs(features_dir, exist_ok=True) 22 | 23 | if featurize_images: 24 | class ArtDataset(Dataset): 25 | """Face Landmarks dataset.""" 26 | 27 | def __init__(self, metadata_json, image_dir, transform): 28 | """ 29 | Args: 30 | csv_file (string): Path to the csv file with annotations. 31 | root_dir (string): Directory with all the images. 32 | transform (callable, optional): Optional transform to be applied 33 | on a sample. 34 | """ 35 | self.metadata = pd.read_json(metadata_json, lines=True) 36 | self.image_dir = image_dir 37 | self.transform = transform 38 | 39 | def __len__(self): 40 | return len(self.metadata) 41 | 42 | def __getitem__(self, idx): 43 | if torch.is_tensor(idx): 44 | idx = idx.tolist() 45 | metadata = self.metadata.iloc[idx] 46 | with open(os.path.join(self.image_dir, metadata["id"] + ".jpg"), "rb") as f: 47 | image = Image.open(f).convert("RGB") 48 | return self.transform(image), metadata["id"] 49 | 50 | 51 | data_transform = transforms.Compose([ 52 | transforms.Resize((224, 224)), 53 | transforms.ToTensor(), 54 | transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) 55 | ]) 56 | dataset = ArtDataset(metadata_fn, data_dir, data_transform) 57 | data_loader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=False, num_workers=4) 58 | 59 | dataset_size = len(dataset) 60 | 61 | # Get a batch of training data 62 | model = models.resnet50(pretrained=True) 63 | model.eval() 64 | model.to(device) 65 | cut_model = nn.Sequential(*list(model.children())[:-1]) 66 | 67 | all_outputs = [] 68 | all_ids = [] 69 | for i, (inputs, ids) in enumerate(tqdm(data_loader)): 70 | inputs = inputs.to(device) 71 | outputs = torch.squeeze(cut_model(inputs)).detach().cpu().numpy() 72 | all_outputs.append(outputs) 73 | all_ids.append(list(ids)) 74 | 75 | all_outputs = np.concatenate(all_outputs, axis=0) 76 | all_ids = np.concatenate(all_ids, axis=0) 77 | 78 | with open(features_file, "wb+") as f: 79 | pickle.dump((all_outputs, all_ids), f) 80 | 81 | with open(features_file, "rb") as f: 82 | with torch.no_grad(): 83 | (all_outputs, all_ids) = pickle.load(f) 84 | all_urls = np.array(pd.read_json(metadata_fn, lines=True).loc[:, "Thumbnail_Url"]) 85 | features = torch.from_numpy(all_outputs).float().to("cpu:0") 86 | features = features / torch.sqrt(torch.sum(features ** 2, dim=1, keepdim=True)) 87 | features = features.to(device) 88 | indicies = torch.arange(0, features.shape[0]).to(device) 89 | print("loaded features") 90 | 91 | metadata = pd.read_json(metadata_fn, lines=True) 92 | culture_arr = np.array(metadata["Culture"]) 93 | cultures = metadata.groupby("Culture").count()["id"].sort_values(ascending=False).index.to_list() 94 | media_arr = np.array(metadata["Classification"]) 95 | media = metadata.groupby("Classification").count()["id"].sort_values(ascending=False).index.to_list() 96 | ids = np.array(metadata["id"]) 97 | 98 | 99 | masks = {"culture": {}, "medium": {}} 100 | for culture in cultures: 101 | masks["culture"][culture] = torch.from_numpy(culture_arr == culture).to(device) 102 | for medium in media: 103 | masks["medium"][medium] = torch.from_numpy(media_arr == medium).to(device) 104 | 105 | 106 | all_matches = [] 107 | for i, row in tqdm(metadata.iterrows()): 108 | feature = features[i] 109 | matches = {"culture": {}, "medium": {}} 110 | all_dists = torch.sum(features * feature, dim=1).to(device) 111 | for culture in cultures: 112 | selected_indicies = indicies[masks["culture"][culture]] 113 | k = min(10, selected_indicies.shape[0]) 114 | dists, inds = torch.topk(all_dists[selected_indicies], k, sorted=True) 115 | matches["culture"][culture] = ids[selected_indicies[inds].cpu().numpy()] 116 | for medium in media: 117 | selected_indicies = indicies[masks["medium"][medium]] 118 | k = min(10, selected_indicies.shape[0]) 119 | dists, inds = torch.topk(all_dists[selected_indicies], k, sorted=True) 120 | matches["medium"][medium] = ids[selected_indicies[inds].cpu().numpy()] 121 | all_matches.append(matches) 122 | 123 | metadata["matches"] = all_matches 124 | 125 | metadata.to_json("results/metadata_enriched.json") 126 | print("here") 127 | 128 | 129 | -------------------------------------------------------------------------------- /developer_guide.md: -------------------------------------------------------------------------------- 1 | 2 | # Mosaic Developer Guide 3 | 4 | ## Building from Scratch 5 | 6 | ### Backend 7 | 8 | To deploy your own conditional image retrieval search index please follow these steps 9 | 10 | 1. Download Image Metadata: 11 | ```bash 12 | wget https://mmlsparkdemo.blob.core.windows.net/cknn/metadata.json?sv=2019-02-02&st=2020-07-23T02%3A22%3A30Z&se=2023-07-24T02%3A22%3A00Z&sr=b&sp=r&sig=hDnGw9y%2BO5XlggL6br%2FPzSKmpAdUZ%2F1LJKVkcmbVmCE%3D 13 | ``` 14 | 1. Download Images: 15 | ```bash 16 | cd data_prep 17 | python download_images.py 18 | ``` 19 | 1. Featurize and perform Conditional Image Retrieval on every image 20 | ```bash 21 | cd data_prep 22 | python featurize_and_match.py 23 | ``` 24 | 1. Write enriched information to an Azure Search Index. 25 | 26 | More detailed code coming soon, Follow [the closely related guide]( 27 | https://docs.microsoft.com/en-us/azure/cognitive-services/big-data/recipes/art-explorer) for a similiar example. 28 | 29 | ### Frontend 30 | 31 | To build a copy of the mosaic website locally please use the following. Note that the introductory animations were made and deployed using a different framework. 32 | 33 | #### Development 34 | 35 | 1. Install `npm` if you dont already have it. You can find instructions at [https://nodejs.org/](https://nodejs.org/). 36 | 1. Install dependencies: 37 | ```bash 38 | cd frontend 39 | npm install 40 | ``` 41 | 1. Start the development server: 42 | ```bash 43 | npm start 44 | ``` 45 | 1. Navigate to [http://localhost:3000/art](http://localhost:3000/art) to explore the local website. 46 | 47 | #### Deployment 48 | 49 | 1. run `npm run build` to create a optimized build 50 | 1. Copy `frontend/utils/404.html` file to the `frontend/build` directory 51 | 1. In `frontend/build/index.html`, between the noscript and script tags (right after the root div), copy in the script from `frontend/utils/singe_page_app.txt` 52 | 1. run `npm run deploy` to push the build to github pages -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | .env 3 | 4 | # dependencies 5 | /node_modules 6 | /.pnp 7 | .pnp.js 8 | 9 | # testing 10 | /coverage 11 | 12 | # production 13 | /build 14 | 15 | # misc 16 | .DS_Store 17 | .env.local 18 | .env.development.local 19 | .env.test.local 20 | .env.production.local 21 | 22 | npm-debug.log* 23 | yarn-debug.log* 24 | yarn-error.log* 25 | -------------------------------------------------------------------------------- /frontend/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /frontend/README.md: -------------------------------------------------------------------------------- 1 | # Installing and Running 2 | To run this, you'll need the latest version of Node.js installed. 3 | Open the project root and run `npm install`. 4 | 5 | To access the Rijksmuseum API, create a `.env` file in the project root with the following text: 6 | 7 | ``` 8 | REACT_APP_RIJKSMUSEUM_API_KEY={yourapikey} 9 | ``` 10 | 11 | where `{yourapikey}` is replaced by your personal API key. More details about the API are found [here](https://data.rijksmuseum.nl/object-metadata/api/). 12 | 13 | Then run `npm start`. 14 | 15 | # Design framework 16 | This project uses React Fabric UI. 17 | 18 | https://developer.microsoft.com/en-us/fabric#/controls/web/contextualmenu 19 | 20 | # Prepareing for Build 21 | Since we use Client Side routing, some hacks need to be introduced to get the site working on github pages. 22 | 23 | Step One: Modify Source Code 24 | 25 | 1. Package.json 26 | After the "name" line insert the line 27 | "homepage":"https://microsoft.github.io/art/", 28 | 29 | This enables the website to know what its base URL is. 30 | 31 | 2. ResultArtwork.tsx: exploreArtUrlSuffix(), line 42 32 | change '/' to '/art/' 33 | 34 | 3. NavBar.tsx, line 23 35 | change '/' to '/art/' 36 | 37 | 4. ResultBox.tsx: exploreArtUrlSuffic(), line 36 38 | change '/' to '/art/' 39 | 40 | NOTE: It is not needed to change the URL generation in SubmitControl, since it "pushes" too the end of the homepage URL. 41 | 42 | Step Two: Make the Build 43 | 44 | run 'npm run build' 45 | This makes a build and populates the build folder. 46 | 47 | 48 | 49 | Step Three: Hack the Build 50 | We follow the instrucions on this page: https://github.com/rafrex/spa-github-pages 51 | 52 | 1. In the build folder, create a 404.html file, and copy in the contents from the above link. 53 | On line 26, change segmentCount from 0 to 1. This enables the site to handle the /art/ at the end of the homepage URL. 54 | 55 | 2. In the html, between the noscript and script tags (right after the root div), copy in the script from the above link 56 | 57 | 58 | Step Four: Deploy the Hacked Build 59 | 60 | run 'npm run deploy' 61 | This takes the contents of the build folder, and deploys it to the github pages. It also makes a push to the gh-pages branch of a repo. 62 | 63 | 64 | # Converting From Deploy to Local 65 | 66 | Revert the changes made in the Source Code in Step One of Preparing for Build 67 | 68 | 69 | # Todo 70 | 71 | Add Information at the bottom of the page 72 | - This would be accomplished by adding to the end of the ExplorePage Heirarchy 73 | 74 | Carousel Random Picks 75 | - This involves changing the default queries made in ExplorePage 76 | 77 | Move Matches Functionality 78 | - This would involve moving the call stack found in an ExplorePage Button to wherever else the functionality needs to be 79 | 80 | MultiAcces Queries 81 | - This would probably involve a change of the APi and a change how we handle the data from the Dropdown menus 82 | 83 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "art", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@material-ui/core": "^4.9.0", 7 | "@microsoft/applicationinsights-react-js": "^2.3.1", 8 | "@microsoft/applicationinsights-web": "^2.3.1", 9 | "@reach/router": "^1.2.1", 10 | "@types/reach__router": "^1.2.6", 11 | "@types/react-burger-menu": "^2.6.1", 12 | "@types/react-helmet": "^5.0.15", 13 | "@types/react-lazyload": "^2.6.0", 14 | "@types/react-router": "^5.1.4", 15 | "@types/react-share": "^3.0.3", 16 | "@types/react-slick": "^0.23.4", 17 | "@uifabric/fluent-theme": "^0.16.7", 18 | "@uifabric/react-cards": "^0.109.3", 19 | "@uifabric/react-hooks": "^7.0.1", 20 | "base64url": "^3.0.1", 21 | "gh-pages": "^2.2.0", 22 | "jimp": "^0.9.3", 23 | "jpeg-js": ">=0.4.0", 24 | "node-sass": "^4.14.1", 25 | "office-ui-fabric-react": "^6.174.0", 26 | "pure-react-carousel": "^1.24.1", 27 | "react": "^16.8.6", 28 | "react-burger-menu": "^2.7.0", 29 | "react-dom": "^16.8.6", 30 | "react-helmet": "^5.2.1", 31 | "react-i18next": "^11.2.7", 32 | "react-iframe": "^1.8.0", 33 | "react-lazyload": "^2.6.5", 34 | "react-responsive-carousel": "^3.1.51", 35 | "react-router-dom": "^5.1.2", 36 | "react-scripts": "^3.4.3", 37 | "react-share": "^3.0.1", 38 | "react-slick": "^0.25.2", 39 | "react-transition-group": "^4.3.0", 40 | "react-with-breakpoints": "^4.0.4", 41 | "reactjs-popup": "^1.5.0", 42 | "slick-carousel": "^1.8.1", 43 | "typescript": "3.4.5" 44 | }, 45 | "scripts": { 46 | "deploy": "gh-pages -d build", 47 | "start": "react-scripts start", 48 | "build": "react-scripts build", 49 | "test": "react-scripts test", 50 | "eject": "react-scripts eject" 51 | }, 52 | "eslintConfig": { 53 | "extends": "react-app" 54 | }, 55 | "browserslist": { 56 | "production": [ 57 | ">0.2%", 58 | "not dead", 59 | "not op_mini all" 60 | ], 61 | "development": [ 62 | "last 1 chrome version", 63 | "last 1 firefox version", 64 | "last 1 safari version" 65 | ] 66 | }, 67 | "resolutions": { 68 | "@babel/runtime": "7.0.0-beta.46" 69 | }, 70 | "devDependencies": { 71 | "@types/jest": "^24.0.12", 72 | "@types/node": "^11.13.8", 73 | "@types/react": "^16.8.15", 74 | "@types/react-dom": "^16.9.4", 75 | "@types/react-router-dom": "^5.1.3", 76 | "typescript": "^3.4.5" 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /frontend/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/art/3ea0616a449cc1c03ee49f1777ce6b331c520b57/frontend/public/favicon.ico -------------------------------------------------------------------------------- /frontend/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 22 |