├── .github └── workflows │ └── pylint.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── __init__.py ├── common ├── __init__.py └── models.py ├── constants.py ├── docker ├── Dockerfile └── git-blacklist ├── download.py ├── exif_data_generators ├── __init__.py ├── custom_geojson_to_exif.py ├── exif_generator_interface.py └── metadata_to_exif.py ├── images └── README ├── io_storage ├── __init__.py └── storage.py ├── login_controller.py ├── logo-KartaView-light.png ├── osc_api_config.py ├── osc_api_gateway.py ├── osc_api_models.py ├── osc_discoverer.py ├── osc_models.py ├── osc_tools.py ├── osc_uploader.py ├── osc_utils.py ├── osm_access.py ├── parsers ├── __init__.py ├── base.py ├── custom_data_parsers │ ├── __init__.py │ ├── custom_geojson.py │ ├── custom_mapillary.py │ └── custom_models.py ├── exif │ ├── __init__.py │ ├── exif.py │ └── utils.py ├── geojson.py ├── gpx.py ├── osc_metadata │ ├── __init__.py │ ├── item_factory.py │ ├── legacy_item_factory.py │ └── parser.py └── xmp.py ├── requirements.txt ├── validators.py └── visual_data_discover.py /.github/workflows/pylint.yml: -------------------------------------------------------------------------------- 1 | name: Pylint 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ["3.9", "3.10"] 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Set up Python ${{ matrix.python-version }} 14 | uses: actions/setup-python@v2 15 | with: 16 | python-version: ${{ matrix.python-version }} 17 | - name: Install dependencies 18 | run: | 19 | python -m pip install --upgrade pip 20 | pip install pylint 21 | - name: Analysing the code with pylint 22 | run: | 23 | pylint --disable=F0401,C0115,C0116,R0903,I1101,W1514,E0611 $(find . -name "*.py" | xargs) --max-line-length=120 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # OSC Tools 2 | credentials.json 3 | *.log 4 | .idea/ 5 | 6 | # General 7 | .DS_Store 8 | .AppleDouble 9 | .LSOverride 10 | 11 | # Icon must end with two \r 12 | Icon 13 | 14 | 15 | # Thumbnails 16 | ._* 17 | 18 | # Files that might appear in the root of a volume 19 | .DocumentRevisions-V100 20 | .fseventsd 21 | .Spotlight-V100 22 | .TemporaryItems 23 | .Trashes 24 | .VolumeIcon.icns 25 | .com.apple.timemachine.donotpresent 26 | 27 | # Directories potentially created on remote AFP share 28 | .AppleDB 29 | .AppleDesktop 30 | Network Trash Folder 31 | Temporary Items 32 | .apdisk 33 | 34 | # Byte-compiled / optimized / DLL files 35 | __pycache__/ 36 | *.py[cod] 37 | *$py.class 38 | 39 | # C extensions 40 | *.so 41 | 42 | # Distribution / packaging 43 | .Python 44 | build/ 45 | develop-eggs/ 46 | dist/ 47 | downloads/ 48 | eggs/ 49 | .eggs/ 50 | lib/ 51 | lib64/ 52 | parts/ 53 | sdist/ 54 | var/ 55 | wheels/ 56 | pip-wheel-metadata/ 57 | share/python-wheels/ 58 | *.egg-info/ 59 | .installed.cfg 60 | *.egg 61 | MANIFEST 62 | 63 | # PyInstaller 64 | # Usually these files are written by a python script from a template 65 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 66 | *.manifest 67 | *.spec 68 | 69 | # Installer logs 70 | pip-log.txt 71 | pip-delete-this-directory.txt 72 | 73 | # Unit test / coverage reports 74 | htmlcov/ 75 | .tox/ 76 | .nox/ 77 | .coverage 78 | .coverage.* 79 | .cache 80 | nosetests.xml 81 | coverage.xml 82 | *.cover 83 | .hypothesis/ 84 | .pytest_cache/ 85 | 86 | # Translations 87 | *.mo 88 | *.pot 89 | 90 | # Django stuff: 91 | local_settings.py 92 | db.sqlite3 93 | 94 | # Flask stuff: 95 | instance/ 96 | .webassets-cache 97 | 98 | # Scrapy stuff: 99 | .scrapy 100 | 101 | # Sphinx documentation 102 | docs/_build/ 103 | 104 | # PyBuilder 105 | target/ 106 | 107 | # Jupyter Notebook 108 | .ipynb_checkpoints 109 | 110 | # IPython 111 | profile_default/ 112 | ipython_config.py 113 | 114 | # pyenv 115 | .python-version 116 | 117 | # celery beat schedule file 118 | celerybeat-schedule 119 | 120 | # SageMath parsed files 121 | *.sage.py 122 | 123 | # Environments 124 | .env 125 | .venv 126 | env/ 127 | venv/ 128 | ENV/ 129 | env.bak/ 130 | venv.bak/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | .spyproject 135 | 136 | # Rope project settings 137 | .ropeproject 138 | 139 | # mkdocs documentation 140 | /site 141 | 142 | # mypy 143 | .mypy_cache/ 144 | .dmypy.json 145 | dmypy.json 146 | 147 | # Pyre type checker 148 | .pyre/ 149 | 150 | # ctags 151 | /tags 152 | 153 | # vim swap 154 | .swp 155 | 156 | # requirements.txt copy used for Docker 157 | docker/requirements.txt 158 | 159 | # ignore images Docker helper directory 160 | images/* 161 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Grabtaxi Holdings PTE LTE (GRAB) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: default pycodestyle clean docker help 2 | 3 | default: help 4 | 5 | help: 6 | @echo 'tags: Build ctags' 7 | @echo 'pycodestyle: run pycodestyle (pep8)' 8 | @echo 'docker: build docker containter' 9 | @echo 'clean: remove development and docker debris' 10 | 11 | tags: *.py 12 | ctags ./ 13 | 14 | pycodestyle: 15 | pycodestyle --max-line-length=100 ./ 16 | 17 | clean: 18 | if [ -f tags ] ; then rm tags; fi 19 | if [ -f docker/requirements.txt ]; then rm docker/requirements.txt; fi 20 | 21 | docker/requirements.txt: 22 | cp requirements.txt docker/requirements.txt 23 | 24 | docker: docker/requirements.txt 25 | docker build -t osc-up docker 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![KartaView](https://github.com/kartaview/upload-scripts/blob/master/logo-KartaView-light.png) 2 | 3 | ## KartaView Tools 4 | 5 | ##### Description 6 | Tools developed by [KartaView](https://kartaview.org/) to help contributors. 7 | 8 | ##### Requirements 9 | * Python 3.6+ 10 | * Dependencies from _requirements.txt_ file. 11 | The dependencies can be installed by running: 12 | ``` 13 | pip3 install virtualenv 14 | 15 | virtualenv -p python3 . 16 | 17 | source bin/activate 18 | 19 | pip3 install -r requirements.txt 20 | ``` 21 | 22 | ## 1. Upload photos to KartaView 23 | 24 | ##### Description 25 | This script is used to upload sequences from a local directory. The available formats are: 26 | * Sequences taken with the OSC mobile apps 27 | * Exif images 28 | 29 | ##### Usage 30 | ``` 31 | cd /path_to_scripts/osc_tools 32 | 33 | # help 34 | python osc_tools.py -h 35 | 36 | # help for upload 37 | python osc_tools.py upload -h 38 | 39 | # upload all sequences from ~/OSC_sequences folder 40 | python osc_tools.py upload -p ~/OSC_seqences 41 | 42 | ``` 43 | 44 | ## 2. Generate Exif info 45 | 46 | ##### Description 47 | This script generates GPS Exif for each image. It has two options generating exif info from metadata file or generating exif info from a custom geojson file. 48 | 49 | ``` 50 | cd /path_to_scripts 51 | 52 | # help 53 | python osc_tools.py -h 54 | 55 | # help for Exif generation 56 | python osc_tools.py generate_exif -h 57 | ``` 58 | 59 | #### Option I. From an KV metadata format file 60 | ``` 61 | # Exif generation for mobile recorded sequence having metadata files in ~/OSC_sequences/Sequence1 folder 62 | python osc_tools.py generate_exif -exif_source metadata -p ~/OSC_seqences/Sequence1 63 | 64 | ``` 65 | 66 | #### Option II. From custom geojson file and the images that require the exif data 67 | 68 | ``` 69 | # Exif generation for custom geojson + imagery 70 | python osc_tools.py generate_exif -exif_source custom_geojson -p ~/CustomFolderContainingGeoJsonAndImages 71 | 72 | ``` 73 |
74 | Folder structure 75 | ~/CustomFolderContainingGeoJsonAndImages
76 | ~/CustomFolderContainingGeoJsonAndImages/a_file.geojson
77 | ~/CustomFolderContainingGeoJsonAndImages/folder_with_images
78 | ~/CustomFolderContainingGeoJsonAndImages/folder_with_images/image1.jpg 79 | ~/CustomFolderContainingGeoJsonAndImages/folder_with_images/image2.jpg 80 | ~/CustomFolderContainingGeoJsonAndImages/folder_with_images/image3.jpg 81 |
82 | 83 | 84 |
85 | Expand to see the custom geojson sample: 86 | 87 | ```javascript 88 | 89 | { 90 | "type":"FeatureCollection", 91 | "features":[ 92 | { 93 | "type":"Feature", 94 | "properties":{ 95 | "order":1.0, 96 | "path":"folder_with_images/image1.jpg", 97 | "direction":236.0, 98 | "Lat":1.910309, 99 | "Lon":1.503069, 100 | "Timestamp":"2020-01-20T08:00:01Z" 101 | }, 102 | "geometry":{ 103 | "type":"Point", 104 | "coordinates":[ 1.503069408072847, 1.910308570011793 ] 105 | } 106 | }, 107 | { 108 | "type":"Feature", 109 | "properties":{ 110 | "order":2.0, 111 | "path":"folder_with_images/image2.jpg", 112 | "direction":236.0, 113 | "Lat":1.910199, 114 | "Lon":1.502908, 115 | "Timestamp":"2020-01-20T08:01:21Z" 116 | }, 117 | "geometry":{ 118 | "type":"Point", 119 | "coordinates":[ 1.502907515952158, 1.910198963742701 ] 120 | } 121 | }, 122 | { 123 | "type":"Feature", 124 | "properties":{ 125 | "order":3.0, 126 | "path":"folder_with_images/image3.jpg", 127 | "direction":236.0, 128 | "Lat":1.910096, 129 | "Lon":1.502764, 130 | "Timestamp":"2020-01-20T08:12:10Z" 131 | }, 132 | "geometry":{ 133 | "type":"Point", 134 | "coordinates":[ 1.50276400212099, 1.910095961756973 ] 135 | } 136 | } 137 | ] 138 | 139 | } 140 | 141 | 142 | ``` 143 | 144 |
145 | 146 | 147 | ## 3. Download Imagery based on user 148 | ##### Description 149 | This script will download all your user uploaded data into a local folder that you provide as input. 150 | It will require a login with your OSM account. 151 | Output imagery data will be grouped based on sequences. 152 | ###### *Current limitations: It does not work for Google and Facebook accounts. 153 | ##### Usage 154 | ``` 155 | python3 osc_tools.py download -p "path to a local folder in which you will have all the data downloaded" 156 | ``` 157 | 158 | ### Docker Support 159 | To run the scripts inside a Docker container: 160 | ``` 161 | make docker 162 | docker run -Pit osc-up osc_tools.py 163 | docker run -Pit --mount type=bind,source="$(pwd)",target=/opt/osc osc-up /opt/osc/osc_tools.py 164 | ``` 165 | The 'images' directory in the repo will be available in the container at /opt/osc/images 166 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | # pylint: skip-file 2 | -------------------------------------------------------------------------------- /common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartaview/upload-scripts/28650a631e398ebe8991c1c965248d41ae68dd87/common/__init__.py -------------------------------------------------------------------------------- /common/models.py: -------------------------------------------------------------------------------- 1 | """This file will contain all common models""" 2 | from enum import Enum 3 | from typing import Optional 4 | 5 | 6 | class RecordingType(Enum): 7 | """This enum represents all possible visual data types.""" 8 | UNKNOWN = 0 9 | PHOTO = 1 10 | VIDEO = 2 11 | RAW = 3 12 | 13 | def __eq__(self, other): 14 | if isinstance(other, RecordingType): 15 | return self.value == other.value 16 | return False 17 | 18 | def __hash__(self): 19 | return hash(self.value) 20 | 21 | 22 | class SensorItem: 23 | """This is a model class representing a generic data item""" 24 | 25 | def __init__(self): 26 | # value is always specified in seconds having sub-millisecond precision. 27 | self.timestamp: Optional[float] = None 28 | 29 | def __eq__(self, other): 30 | if isinstance(other, SensorItem): 31 | return self.timestamp == other.timestamp 32 | return False 33 | 34 | def __hash__(self): 35 | return hash(self.timestamp) 36 | 37 | 38 | class PhotoMetadata(SensorItem): 39 | """PhotoMetadata is a SensorItem that represents a photo""" 40 | 41 | def __init__(self): 42 | super().__init__() 43 | self.gps: GPS = GPS() 44 | self.obd: OBD = OBD() 45 | self.compass: Compass = Compass() 46 | # index of the video in witch the PhotoMetadata has the corresponding image data. 47 | self.video_index: Optional[int] = None 48 | # frame index of the PhotoMetadata relative to the entire sequence of photos. 49 | self.frame_index: int = None 50 | 51 | def __eq__(self, other): 52 | if isinstance(other, PhotoMetadata): 53 | return self.timestamp == other.timestamp and \ 54 | self.gps.latitude == other.gps.latitude and \ 55 | self.gps.longitude == other.gps.longitude and \ 56 | self.video_index == other.video_index and \ 57 | self.frame_index == other.frame_index 58 | return False 59 | 60 | def __hash__(self): 61 | return hash((self.timestamp, 62 | self.gps.latitude, 63 | self.gps.longitude, 64 | self.video_index, 65 | self.frame_index)) 66 | 67 | 68 | class GPS(SensorItem): 69 | """GPS is a SensorItem model class that can represent all information found in a GPS item""" 70 | 71 | def __init__(self): 72 | super().__init__() 73 | # in degrees 74 | self.latitude: Optional[float] = None 75 | self.longitude: Optional[float] = None 76 | # in meters 77 | self.altitude: Optional[float] = None 78 | # in meters 79 | self.horizontal_accuracy: Optional[float] = None 80 | # in meters 81 | self.vertical_accuracy: Optional[float] = None 82 | # speed is in m/s 83 | self.speed: Optional[float] = None 84 | 85 | @classmethod 86 | def gps(cls, timestamp: float, latitude: float, longitude: float): 87 | gps = cls() 88 | gps.timestamp = timestamp 89 | gps.latitude = latitude 90 | gps.longitude = longitude 91 | return gps 92 | 93 | def __eq__(self, other): 94 | if isinstance(other, GPS): 95 | return self.timestamp == other.timestamp and \ 96 | self.latitude == other.latitude and \ 97 | self.longitude == other.longitude 98 | return False 99 | 100 | def __hash__(self): 101 | return hash((self.timestamp, self.latitude, self.longitude)) 102 | 103 | def __str__(self): 104 | return f"[{self.timestamp} : ({self.longitude}, {self.latitude})]" 105 | 106 | 107 | class Acceleration(SensorItem): 108 | """Acceleration is a SensorItem model representing an acceleration data""" 109 | 110 | def __init__(self): 111 | super().__init__() 112 | # X-axis acceleration in G's 113 | self.acc_x: Optional[float] = None 114 | # Y-axis acceleration in G's 115 | self.acc_y: Optional[float] = None 116 | # Z-axis acceleration in G's 117 | self.acc_z: Optional[float] = None 118 | 119 | def __eq__(self, other): 120 | if isinstance(other, Acceleration): 121 | return self.timestamp == other.timestamp and \ 122 | self.acc_x == other.acc_x and \ 123 | self.acc_y == other.acc_y and \ 124 | self.acc_z == other.acc_z 125 | return False 126 | 127 | def __hash__(self): 128 | return hash((self.timestamp, 129 | self.acc_x, 130 | self.acc_y, 131 | self.acc_z)) 132 | 133 | 134 | class Compass(SensorItem): 135 | """Compass is a SensorItem model class that can represent a compass data""" 136 | 137 | def __init__(self): 138 | super().__init__() 139 | # The heading (measured in degrees) relative to true north 140 | self.compass: Optional[float] = None 141 | 142 | def __eq__(self, other): 143 | if isinstance(other, Compass): 144 | return self.timestamp == other.timestamp and \ 145 | self.compass == other.compass 146 | return False 147 | 148 | def __hash__(self): 149 | return hash((self.timestamp, 150 | self.compass)) 151 | 152 | 153 | class OBD(SensorItem): 154 | """OBD is a SensorItem model class that can represent an obd data""" 155 | 156 | def __init__(self): 157 | super().__init__() 158 | # value is in km/h 159 | self.speed: Optional[float] = None 160 | 161 | def __eq__(self, other): 162 | if isinstance(other, OBD): 163 | return self.timestamp == other.timestamp and \ 164 | self.speed == other.speed 165 | return False 166 | 167 | def __hash__(self): 168 | return hash((self.timestamp, 169 | self.speed)) 170 | 171 | 172 | class Pressure(SensorItem): 173 | """Pressure is a SensorItem model class that can represent an pressure data""" 174 | 175 | def __init__(self): 176 | super().__init__() 177 | # value is in kPa 178 | self.pressure: Optional[float] = None 179 | 180 | def __eq__(self, other): 181 | if isinstance(other, Pressure): 182 | return self.timestamp == other.timestamp and \ 183 | self.pressure == other.pressure 184 | return False 185 | 186 | def __hash__(self): 187 | return hash((self.timestamp, 188 | self.pressure)) 189 | 190 | 191 | class Attitude(SensorItem): 192 | """Attitude is a SensorItem model class that can represent an attitude data""" 193 | 194 | def __init__(self): 195 | super().__init__() 196 | # Returns the yaw of the device in radians. 197 | self.yaw: Optional[float] = None 198 | # Returns the pitch of the device in radians. 199 | self.pitch: Optional[float] = None 200 | # Returns the roll of the device in radians. 201 | self.roll: Optional[float] = None 202 | 203 | def __eq__(self, other): 204 | if isinstance(other, Attitude): 205 | return self.timestamp == other.timestamp and \ 206 | self.yaw == other.yaw and \ 207 | self.pitch == other.pitch and \ 208 | self.roll == other.roll 209 | return False 210 | 211 | def __hash__(self): 212 | return hash((self.timestamp, 213 | self.yaw, 214 | self.pitch, 215 | self.roll)) 216 | 217 | 218 | class Gravity(Acceleration): 219 | """Gravity is a SensorItem model class that can represent a gravity data""" 220 | 221 | def __eq__(self, other): 222 | if isinstance(other, Gravity): 223 | return self.timestamp == other.timestamp and \ 224 | self.acc_x == other.acc_x and \ 225 | self.acc_y == other.acc_y and \ 226 | self.acc_z == other.acc_z 227 | return False 228 | 229 | def __hash__(self): 230 | return hash((self.timestamp, 231 | self.acc_x, 232 | self.acc_y, 233 | self.acc_z)) 234 | 235 | 236 | class DeviceMotion(SensorItem): 237 | """DeviceMotion is a SensorItem model class that can represent a device motion data""" 238 | 239 | def __init__(self): 240 | super().__init__() 241 | # Returns the attitude of the device. 242 | self.gyroscope: Attitude = Attitude() 243 | # Returns the acceleration that the user is giving to the device. Note 244 | # that the total acceleration of the device is equal to gravity plus 245 | # acceleration. 246 | self.acceleration: Acceleration = Acceleration() 247 | # Returns the gravity vector expressed in the device's reference frame. Note 248 | # that the total acceleration of the device is equal to gravity plus 249 | # acceleration. 250 | self.gravity: Gravity = Gravity() 251 | 252 | def __eq__(self, other): 253 | if isinstance(other, DeviceMotion): 254 | return self.timestamp == other.timestamp and \ 255 | self.gyroscope == other.gyroscope and \ 256 | self.acceleration == other.acceleration and \ 257 | self.gravity == other.gravity 258 | return False 259 | 260 | def __hash__(self): 261 | return hash((self.timestamp, 262 | self.gyroscope, 263 | self.acceleration, 264 | self.gravity)) 265 | 266 | @staticmethod 267 | def type_conversions(device_motion): 268 | device_motion.acceleration.acc_x = float(device_motion.acceleration.acc_x) 269 | device_motion.acceleration.acc_y = float(device_motion.acceleration.acc_y) 270 | device_motion.acceleration.acc_z = float(device_motion.acceleration.acc_z) 271 | device_motion.gravity.acc_x = float(device_motion.gravity.acc_x) 272 | device_motion.gravity.acc_y = float(device_motion.gravity.acc_y) 273 | device_motion.gravity.acc_z = float(device_motion.gravity.acc_z) 274 | device_motion.gyroscope.yaw = float(device_motion.gyroscope.yaw) 275 | device_motion.gyroscope.pitch = float(device_motion.gyroscope.pitch) 276 | device_motion.gyroscope.roll = float(device_motion.gyroscope.roll) 277 | 278 | 279 | class OSCDevice(SensorItem): 280 | """OSCDevice is a SensorItem model class that can represent an device data""" 281 | 282 | def __init__(self): 283 | super().__init__() 284 | # The platform from which the track was recorded: 285 | self.platform_name: Optional[str] = None 286 | # Custom version of operating system. Default OS name (platform) will be used if there is 287 | # no customized version. Ex: iOS, Yun OS, Paranoid Android 288 | self.os_raw_name: Optional[str] = None 289 | # The OS version from the device from which the track was recorded. 290 | self.os_version: Optional[str] = None 291 | # The raw name of the device. Eg: iPhone10,3 for iPhone X. 292 | self.device_raw_name: Optional[str] = None 293 | # App version with X.Y or X.Y.Z format. Eg: 2.4, 2.4.1 294 | self.app_version: Optional[str] = None 295 | # Build number for app version. 296 | self.app_build_number: Optional[str] = None 297 | # The type of recording: video, photo 298 | self.recording_type: Optional[RecordingType] = None 299 | 300 | def __eq__(self, other): 301 | if isinstance(other, OSCDevice): 302 | return self.timestamp == other.timestamp and \ 303 | self.platform_name == other.platform_name and \ 304 | self.os_raw_name == other.os_raw_name and \ 305 | self.os_version == other.os_version and \ 306 | self.device_raw_name == other.device_raw_name and \ 307 | self.app_version == other.app_version and \ 308 | self.app_build_number == other.app_build_number and \ 309 | self.recording_type == other.recording_type 310 | return False 311 | 312 | def __hash__(self): 313 | return hash((self.timestamp, 314 | self.platform_name, 315 | self.os_raw_name, 316 | self.os_version, 317 | self.device_raw_name, 318 | self.app_version, 319 | self.app_build_number, 320 | self.recording_type)) 321 | 322 | 323 | class CameraParameters(SensorItem): 324 | """CameraParameters is a SensorItem model class that can represent camera parameters""" 325 | 326 | def __init__(self): 327 | super().__init__() 328 | # Horizontal field of view in degrees. If field of view is unknown, 0 is returned. 329 | self.h_fov: Optional[float] = None 330 | # Vertical field of view in degrees. If field of view is unknown, 0 is returned. 331 | self.v_fov: Optional[float] = None 332 | # Aperture of the device. 333 | self.aperture: Optional[str] = None 334 | self.projection: CameraProjection = None 335 | 336 | def __eq__(self, other): 337 | if isinstance(other, CameraParameters): 338 | return self.timestamp == other.timestamp and \ 339 | self.v_fov == other.v_fov and \ 340 | self.h_fov == other.h_fov and \ 341 | self.aperture == other.aperture and \ 342 | self.projection == other.projection 343 | return False 344 | 345 | def __hash__(self): 346 | return hash((self.timestamp, 347 | self.h_fov, 348 | self.v_fov, 349 | self.aperture)) 350 | 351 | 352 | class ExifParameters(SensorItem): 353 | """ExifParameters is a SensorItem model class that can represent a focal and f number""" 354 | 355 | def __init__(self): 356 | super().__init__() 357 | self.focal_length: Optional[float] = None 358 | self.width: Optional[int] = None 359 | self.height: Optional[int] = None 360 | 361 | def __eq__(self, other): 362 | if isinstance(other, ExifParameters): 363 | return self.timestamp == other.timestamp and \ 364 | self.focal_length == other.focal_length 365 | return False 366 | 367 | def __hash__(self): 368 | return hash((self.timestamp, 369 | self.focal_length)) 370 | 371 | 372 | class CameraProjection(Enum): 373 | EQUIRECTANGULAR = "equirectangular" 374 | DUAL_FISHEYE = "DUAL_FISHEYE" 375 | FISHEYE_BACK = "FISHEYE_BACK" 376 | FISHEYE_FRONT = "FISHEYE_FRONT" 377 | PLAIN = "plain" 378 | 379 | 380 | def projection_type_from_name(projection_name) -> Optional[CameraProjection]: 381 | projection = None 382 | if CameraProjection.PLAIN.name.lower() in projection_name.lower(): 383 | projection = CameraProjection.PLAIN 384 | elif CameraProjection.EQUIRECTANGULAR.name.lower() in projection_name.lower(): 385 | projection = CameraProjection.EQUIRECTANGULAR 386 | elif CameraProjection.DUAL_FISHEYE.name.lower() in projection_name.lower(): 387 | projection = CameraProjection.DUAL_FISHEYE 388 | elif CameraProjection.FISHEYE_FRONT.name.lower() in projection_name.lower(): 389 | projection = CameraProjection.FISHEYE_FRONT 390 | elif CameraProjection.FISHEYE_BACK.name.lower() in projection_name.lower(): 391 | projection = CameraProjection.FISHEYE_BACK 392 | return projection 393 | -------------------------------------------------------------------------------- /constants.py: -------------------------------------------------------------------------------- 1 | """This module will contain project constants required in multiple modules""" 2 | 3 | PROGRESS_FILE_NAME = "osc_sequence_upload_progress.txt" 4 | UPLOAD_FINISHED = "finished" 5 | METADATA_ZIP_NAME = "track.txt.gz" 6 | METADATA_NAME = "track.txt" 7 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:buster-slim 2 | 3 | COPY requirements.txt /etc/requirements.txt 4 | COPY git-blacklist /etc/apt/preferences.d/git-blacklist 5 | 6 | RUN apt-get update && apt-get -y upgrade && apt-get -y install python3 python3-pip && pip3 install -r /etc/requirements.txt && apt-get -y clean 7 | 8 | -------------------------------------------------------------------------------- /docker/git-blacklist: -------------------------------------------------------------------------------- 1 | Package: git 2 | Pin: release * 3 | Pin-Priority: -1 4 | -------------------------------------------------------------------------------- /download.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module is created in order to support the download of user uploaded data 3 | """ 4 | 5 | import logging 6 | import os 7 | from concurrent.futures import ( 8 | as_completed, 9 | ThreadPoolExecutor, 10 | ) 11 | from typing import List, Tuple 12 | 13 | from tqdm import tqdm 14 | 15 | from common.models import GPS 16 | from io_storage.storage import Local 17 | from login_controller import LoginController 18 | from osc_api_config import OSCAPISubDomain 19 | from osc_api_gateway import OSCApi 20 | from osc_api_models import OSCPhoto 21 | from parsers.exif.exif import ExifParser 22 | 23 | LOGGER = logging.getLogger('osc_tools.osc_utils') 24 | 25 | 26 | def download_user_images(to_path): 27 | login_controller = LoginController(OSCAPISubDomain.PRODUCTION) 28 | # login to get the valid user 29 | user = login_controller.login() 30 | osc_api = login_controller.osc_api 31 | 32 | # get all the sequneces for this user 33 | sequences, error = osc_api.user_sequences(user.name) 34 | if error: 35 | LOGGER.info("Could not get sequences for the current user, try again or report a issue on " 36 | "github") 37 | return 38 | 39 | user_dir_path = os.path.join(to_path, user.name) 40 | os.makedirs(user_dir_path, exist_ok=True) 41 | 42 | # download each sequence 43 | for sequence in tqdm(sequences, desc="Downloading sequences"): 44 | if sequence is None or isinstance(sequence, BaseException): 45 | continue 46 | 47 | sequence_path = os.path.join(user_dir_path, str(sequence.online_id)) 48 | os.makedirs(sequence_path, exist_ok=True) 49 | 50 | download_success, _ = _download_photo_sequence(osc_api, 51 | sequence, 52 | sequence_path, 53 | add_gps_to_exif=True) 54 | if not download_success: 55 | LOGGER.info("There was an error downloading the sequence: %s", str(sequence.online_id)) 56 | continue 57 | 58 | 59 | def _download_photo_sequence(osc_api: OSCApi, 60 | sequence, 61 | sequence_path: str, 62 | add_gps_to_exif: bool = False) -> Tuple[bool, List[OSCPhoto]]: 63 | photos, error = osc_api.get_photos(sequence.online_id) 64 | if error: 65 | return False, [] 66 | # download metadata 67 | if sequence.metadata_url is not None: 68 | metadata_path = os.path.join(sequence_path, sequence.online_id + ".txt") 69 | osc_api.download_resource(sequence.metadata_url, metadata_path, Local()) 70 | sequence.metadata_url = metadata_path 71 | 72 | # download photos 73 | download_success = True 74 | download_bar = tqdm(total=len(photos), 75 | desc="Downloading sequence " + str(sequence.online_id)) 76 | with ThreadPoolExecutor(max_workers=10) as executors: 77 | futures = [executors.submit(_download_photo, 78 | photo, 79 | sequence_path, 80 | osc_api, 81 | add_gps_to_exif) for photo in photos] 82 | for future in as_completed(futures): 83 | photo_success, _ = future.result() 84 | download_success = download_success and photo_success 85 | download_bar.update(1) 86 | if not download_success: 87 | LOGGER.info("Download failed for sequence %s", str(sequence.online_id)) 88 | return False, [] 89 | return True, photos 90 | 91 | 92 | def _download_photo(photo: OSCPhoto, 93 | folder_path: str, 94 | osc_api: OSCApi, 95 | add_gps_to_exif: bool = False): 96 | photo_download_name = os.path.join(folder_path, str(photo.sequence_index) + ".jpg") 97 | local_storage = Local() 98 | photo_success, error = osc_api.download_resource(photo.photo_url(), 99 | photo_download_name, 100 | local_storage) 101 | if error or not photo_success: 102 | LOGGER.debug("Failed to download image: %s", photo.photo_url()) 103 | 104 | if add_gps_to_exif: 105 | parser = ExifParser(photo_download_name, local_storage) 106 | if len(parser.items_with_class(GPS)) == 0: 107 | item = GPS() 108 | item.latitude = photo.latitude 109 | item.longitude = photo.longitude 110 | item.timestamp = photo.timestamp 111 | parser.add_items([item]) 112 | parser.serialize() 113 | return photo_success, error 114 | -------------------------------------------------------------------------------- /exif_data_generators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartaview/upload-scripts/28650a631e398ebe8991c1c965248d41ae68dd87/exif_data_generators/__init__.py -------------------------------------------------------------------------------- /exif_data_generators/custom_geojson_to_exif.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exif generation from custom geojson file. Required a geojson file containing a 3 | FeatureCollection. Each Feature from the FeatureCollection must contain the folowing properties: 4 | { 5 | "Timestamp": "%Y-%m-%dT%H:%M:%SZ" 6 | "Lat": float 7 | "Lng": float 8 | "order": int 9 | "path": image path relative to the geojson file 10 | } 11 | """ 12 | import logging 13 | import os 14 | from typing import List 15 | 16 | from parsers.custom_data_parsers.custom_geojson import FeaturePhotoGeoJsonParser, PhotoGeoJson 17 | from parsers.exif.utils import create_required_gps_tags, add_optional_gps_tags, add_gps_tags 18 | from exif_data_generators.exif_generator_interface import ExifGenerator 19 | from io_storage.storage import Local 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class ExifCustomGeoJson(ExifGenerator): 25 | 26 | @staticmethod 27 | def create_exif(path: str) -> bool: 28 | logger.warning("Creating exif from custom geojson file %s", path) 29 | for folder_path, _, files in os.walk(path): 30 | for file in files: 31 | _, file_extension = os.path.splitext(file) 32 | if 'geojson' in file_extension: 33 | parser = FeaturePhotoGeoJsonParser(os.path.join(folder_path, file), Local()) 34 | parser.start_new_reading() 35 | custom_photos: List[PhotoGeoJson] = parser.items_with_class(PhotoGeoJson) 36 | for photo in custom_photos: 37 | absolute_path = os.path.join(folder_path, photo.relative_image_path) 38 | if photo.gps.latitude and photo.gps.longitude: 39 | tags = create_required_gps_tags(photo.gps.timestamp, 40 | photo.gps.latitude, 41 | photo.gps.longitude) 42 | add_optional_gps_tags(tags, 43 | photo.gps.speed, 44 | photo.gps.altitude, 45 | photo.compass.compass) 46 | add_gps_tags(absolute_path, tags) 47 | return True 48 | 49 | @staticmethod 50 | def has_necessary_data(path) -> bool: 51 | for _, _, files in os.walk(path): 52 | for file in files: 53 | _, file_extension = os.path.splitext(file) 54 | if 'geojson' in file_extension: 55 | return True 56 | return False 57 | -------------------------------------------------------------------------------- /exif_data_generators/exif_generator_interface.py: -------------------------------------------------------------------------------- 1 | """ 2 | Interface for exif generation. This should be implemented in order to add support for exif 3 | generation from any custom data source. 4 | """ 5 | import abc 6 | 7 | 8 | class ExifGenerator(metaclass=abc.ABCMeta): 9 | 10 | @staticmethod 11 | @abc.abstractmethod 12 | def create_exif(path: str) -> bool: 13 | pass 14 | 15 | @staticmethod 16 | @abc.abstractmethod 17 | def has_necessary_data(path: str) -> bool: 18 | pass 19 | -------------------------------------------------------------------------------- /exif_data_generators/metadata_to_exif.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module is used to generate exif data from OSC Metadata file recorded by iOS or Android apps. 3 | """ 4 | import logging 5 | import os 6 | from typing import cast, List 7 | 8 | import constants 9 | from common.models import PhotoMetadata 10 | from exif_data_generators.exif_generator_interface import ExifGenerator 11 | from io_storage.storage import Local 12 | from osc_models import Photo 13 | from parsers.exif.utils import create_required_gps_tags, add_optional_gps_tags, add_gps_tags 14 | from parsers.osc_metadata.parser import metadata_parser 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class ExifMetadataGenerator(ExifGenerator): 20 | 21 | @staticmethod 22 | def create_exif(path: str) -> bool: 23 | """this method will generate exif data from metadata""" 24 | logger.warning("Creating exif from metadata file %s", path) 25 | files = os.listdir(path) 26 | photos = [] 27 | metadata_photos = [] 28 | 29 | for file_path in files: 30 | file_name, file_extension = os.path.splitext(file_path) 31 | if ("jpg" in file_extension or "jpeg" in file_extension) \ 32 | and "thumb" not in file_name.lower(): 33 | photo = Photo(os.path.join(path, file_path)) 34 | if file_name.isdigit(): 35 | photo.index = int(file_name) 36 | photos.append(photo) 37 | elif ".txt" in file_extension and constants.METADATA_NAME in file_path: 38 | metadata_file = file_path 39 | parser = metadata_parser(os.path.join(path, metadata_file), Local()) 40 | parser.start_new_reading() 41 | metadata_photos = cast(List[PhotoMetadata], parser.items_with_class(PhotoMetadata)) 42 | 43 | if metadata_photos: 44 | photos.sort(key=lambda x: int(os.path.splitext(os.path.basename(x.path))[0])) 45 | metadata_photos.sort(key=lambda x: x.frame_index) 46 | else: 47 | logger.warning("WARNING: NO metadata photos found at %s", path) 48 | return False 49 | 50 | for photo in photos: 51 | for tmp_photo in metadata_photos: 52 | metadata_photo: PhotoMetadata = cast(PhotoMetadata, tmp_photo) 53 | if (int(metadata_photo.frame_index) == photo.index and 54 | metadata_photo.gps.latitude and metadata_photo.gps.longitude): 55 | tags = create_required_gps_tags(metadata_photo.gps.timestamp, 56 | metadata_photo.gps.latitude, 57 | metadata_photo.gps.longitude) 58 | add_optional_gps_tags(tags, 59 | metadata_photo.gps.speed, 60 | metadata_photo.gps.altitude, 61 | metadata_photo.compass.compass) 62 | add_gps_tags(photo.path, tags) 63 | break 64 | 65 | return True 66 | 67 | @staticmethod 68 | def has_necessary_data(path) -> bool: 69 | return os.path.isfile(os.path.join(path, constants.METADATA_NAME)) 70 | -------------------------------------------------------------------------------- /images/README: -------------------------------------------------------------------------------- 1 | This is a directory ignored by git that exists to help easily mount an image 2 | directory along with the upload-scripts code to a Docker container. 3 | -------------------------------------------------------------------------------- /io_storage/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartaview/upload-scripts/28650a631e398ebe8991c1c965248d41ae68dd87/io_storage/__init__.py -------------------------------------------------------------------------------- /io_storage/storage.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module enables the use of custom storage implementations e.g. cloud storage solutions without 3 | changing the functionality of the scripts. 4 | If one needs support for other type of storages they can just implement the Storage interface and 5 | use the new storage type. 6 | """ 7 | import abc 8 | import hashlib 9 | import os 10 | import logging 11 | 12 | from typing import List 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class Storage(metaclass=abc.ABCMeta): 18 | 19 | @property 20 | @abc.abstractmethod 21 | def container_name(self): 22 | pass 23 | 24 | @property 25 | @abc.abstractmethod 26 | def storage_url(self): 27 | pass 28 | 29 | @abc.abstractmethod 30 | def listdir(self, path: str) -> List[str]: 31 | pass 32 | 33 | @abc.abstractmethod 34 | def walk(self, path: str): 35 | pass 36 | 37 | @abc.abstractmethod 38 | def exists(self, path: str) -> bool: 39 | pass 40 | 41 | @abc.abstractmethod 42 | def isdir(self, path: str) -> bool: 43 | pass 44 | 45 | @abc.abstractmethod 46 | def isfile(self, path: str) -> bool: 47 | pass 48 | 49 | @abc.abstractmethod 50 | def open(self, path: str, mode='r'): 51 | pass 52 | 53 | def unique_file_identifier(self, file_path: str, block_size: int = 165536) -> str: 54 | md5_hash = hashlib.md5() 55 | with self.open(file_path, "rb") as file: 56 | for block in iter(lambda: file.read(block_size), b""): 57 | md5_hash.update(block) 58 | return str(md5_hash.hexdigest()) 59 | 60 | def unique_path_identifier(self, path): 61 | return hashlib.md5(self.abs_path(path).encode()).hexdigest() 62 | 63 | @abc.abstractmethod 64 | def abs_path(self, path: str) -> str: 65 | pass 66 | 67 | @abc.abstractmethod 68 | def getsize(self, path: str) -> int: 69 | pass 70 | 71 | @abc.abstractmethod 72 | def getctime(self, path: str) -> float: 73 | pass 74 | 75 | @abc.abstractmethod 76 | def getmtime(self, path: str) -> float: 77 | pass 78 | 79 | @abc.abstractmethod 80 | def rename(self, src: str, dst: str): 81 | pass 82 | 83 | @abc.abstractmethod 84 | def remove(self, path: str): 85 | pass 86 | 87 | def put(self, data, destination): 88 | raise NotImplementedError() 89 | 90 | 91 | class Local(Storage): 92 | 93 | @property 94 | def container_name(self): 95 | return "" 96 | 97 | @property 98 | def storage_url(self): 99 | return "" 100 | 101 | def listdir(self, path: str) -> List[str]: 102 | return os.listdir(path) 103 | 104 | def walk(self, path: str): 105 | yield os.walk(path) 106 | 107 | def exists(self, path: str) -> bool: 108 | return os.path.exists(path) 109 | 110 | def isdir(self, path: str) -> bool: 111 | return os.path.isdir(path) 112 | 113 | def isfile(self, path: str) -> bool: 114 | return os.path.isfile(path) 115 | 116 | def open(self, path: str, mode='r'): 117 | return open(path, mode) 118 | 119 | def abs_path(self, path: str) -> str: 120 | return os.path.abspath(path) 121 | 122 | def getsize(self, path: str) -> int: 123 | return os.path.getsize(path) 124 | 125 | def getctime(self, path: str) -> float: 126 | return os.path.getctime(path) 127 | 128 | def getmtime(self, path: str) -> float: 129 | return os.path.getmtime(path) 130 | 131 | def rename(self, src: str, dst: str): 132 | return os.rename(src, dst) 133 | 134 | def remove(self, path: str): 135 | return os.remove(path) 136 | 137 | def put(self, data, destination): 138 | with open(destination, "wb") as out_file: 139 | out_file.write(data) 140 | -------------------------------------------------------------------------------- /login_controller.py: -------------------------------------------------------------------------------- 1 | """This module is responsible with getting access to OSC API""" 2 | import os 3 | import sys 4 | import urllib 5 | import json 6 | import logging 7 | from typing import Optional 8 | 9 | from osm_access import osm_auth 10 | from osc_api_gateway import OSCApi 11 | from osc_api_gateway import OSCUser 12 | from osc_api_gateway import OSCAPISubDomain 13 | 14 | # constants 15 | CREDENTIALS_FILE = "credentials.json" 16 | OSM_KEY = "osm" 17 | OSC_KEY = "osc" 18 | 19 | TOKEN_KEY = "token" 20 | TOKEN_SECRET_KEY = "token_secret" 21 | 22 | OSC_ENV_KEY = "osc_env" 23 | USER_ID_KEY = "user_id" 24 | USER_NAME_KEY = "user_name" 25 | USER_FULL_NAME_KEY = "full_name" 26 | 27 | LOGGER = logging.getLogger('osc_tools.logging_controller') 28 | 29 | 30 | class LoginController: 31 | """This class will enable """ 32 | 33 | def __init__(self, sub_domain: OSCAPISubDomain): 34 | self.osc_api = OSCApi(sub_domain) 35 | self.handle_retry_count = 0 36 | self.user: Optional[OSCUser] = None 37 | self.osm_token = "" 38 | self.osm_token_secret = "" 39 | 40 | osm_token, osm_token_secret, osc_user, env = self.__read_persistent_login() 41 | self.osm_token = osm_token 42 | self.osm_token_secret = osm_token_secret 43 | LOGGER.debug("Current environment: %s Cached environment: %s", str(sub_domain), str(env)) 44 | if env == sub_domain: 45 | LOGGER.debug("Same environment detected") 46 | self.user = osc_user 47 | 48 | def login(self) -> OSCUser: 49 | """This method makes osm authentication and request osc authorization for API usage""" 50 | if self.user is not None: 51 | dir_path = os.path.dirname(os.path.realpath(__file__)) 52 | LOGGER.info("Logged in user: %s. To log out delete %s/%s", 53 | self.user.name, 54 | dir_path, 55 | CREDENTIALS_FILE) 56 | return self.user 57 | 58 | if self.osm_token == "" or self.osm_token_secret == "": 59 | # osm authentication 60 | LOGGER.debug("Will start osm auth") 61 | osm_token, osm_secret = osm_auth(self.__prompt_user_for_login, 62 | self.__handle_osm_auth_error) 63 | LOGGER.debug("OSM auth done") 64 | if osm_token is None or osm_secret is None: 65 | LOGGER.debug("osm auth failed will retry") 66 | # retry login process. this happens just once. 67 | return self.login() 68 | 69 | self.__persist_login(osm_token=osm_token, osm_secret=osm_secret) 70 | self.osm_token = osm_token 71 | self.osm_token_secret = osm_secret 72 | 73 | # osc authorization 74 | osc_user, exception = self.osc_api.authorized_user("osm", 75 | self.osm_token, 76 | self.osm_token_secret) 77 | if exception is not None: 78 | self.__handle_error_on_authorization(exception) 79 | 80 | self.__persist_login(osc_user=osc_user) 81 | self.user = osc_user 82 | LOGGER.info("Greetings, %s!", self.user.name) 83 | return osc_user 84 | 85 | @classmethod 86 | def logout(cls): 87 | """This method will remove the previously cached credentials""" 88 | os.remove(CREDENTIALS_FILE) 89 | 90 | def __handle_osm_auth_error(self, error: Exception): 91 | if isinstance(error, urllib.error.HTTPError): 92 | LOGGER.warning("Can't get osm id") 93 | LOGGER.warning("Please report this issue with the error code on " 94 | "https://github.com/kartaview/upload-scripts") 95 | LOGGER.warning(error.code) 96 | LOGGER.warning(error.read()) 97 | sys.exit() 98 | elif self.handle_retry_count == 0: 99 | self.handle_retry_count = 1 100 | LOGGER.warning("An error occurred.") 101 | LOGGER.debug("Error message: %s", str(error)) 102 | LOGGER.warning("This error can occur when you did not log in your browser. " 103 | "Please make sure to follow the next steps to retry.") 104 | else: 105 | LOGGER.warning("Error message: %s", str(error)) 106 | LOGGER.warning("Please report this issue with the error message" 107 | "https://github.com/kartaview/upload-scripts") 108 | sys.exit() 109 | 110 | @classmethod 111 | def __prompt_user_for_login(cls, osm_url: str): 112 | LOGGER.warning("") 113 | return input(f"1. Login by opening this URL in your browser:\n\t{osm_url}\n" 114 | f"2. Copy, paste the 'Authorization code' from the browser in console\n" 115 | f"3. Press ENTER\n") 116 | 117 | @classmethod 118 | def __handle_error_on_authorization(cls, exception: Exception): 119 | LOGGER.warning("Can't get osc authorization right now please try again " 120 | "or if you already did this.") 121 | LOGGER.warning("Please report this issue with the error message on " 122 | "https://github.com/kartaview/upload-scripts") 123 | LOGGER.warning(exception) 124 | sys.exit() 125 | 126 | def __persist_login(self, osm_token: str = "", osm_secret: str = "", osc_user: OSCUser = None): 127 | LOGGER.debug("will save credentials into file") 128 | # read the cached values from json file 129 | old_osm_token, old_osm_secret, old_user, old_environment = self.__read_persistent_login() 130 | # update the json file with the new values 131 | # if value is missing just use the old value 132 | 133 | credentials_dict = {} 134 | 135 | # validate osm credentials 136 | if osm_token == "" or osm_secret == "" and old_osm_token != "" and old_osm_secret != "": 137 | LOGGER.debug("using cached osm credentials") 138 | osm_token = old_osm_token 139 | osm_secret = old_osm_secret 140 | 141 | if osm_token != "" and osm_secret != "": 142 | LOGGER.debug("prepare to write osm credentials") 143 | credentials_dict[OSM_KEY] = {TOKEN_KEY: osm_token, 144 | TOKEN_SECRET_KEY: osm_secret} 145 | # validate osc credentials 146 | osc_env = self.osc_api.environment 147 | if osc_user is None and \ 148 | old_user is not None and \ 149 | self.osc_api.environment == old_environment: 150 | LOGGER.debug("using cached osc user") 151 | osc_user = old_user 152 | 153 | if osc_user is not None: 154 | LOGGER.debug("prepare to write osc user") 155 | credentials_dict[OSC_KEY] = {USER_ID_KEY: osc_user.user_id, 156 | USER_NAME_KEY: osc_user.name, 157 | USER_FULL_NAME_KEY: osc_user.full_name, 158 | TOKEN_KEY: osc_user.access_token, 159 | OSC_ENV_KEY: osc_env.value} 160 | with open(CREDENTIALS_FILE, 'w') as output: 161 | json.dump(credentials_dict, output) 162 | LOGGER.debug("Did write data to credentials file") 163 | 164 | @classmethod 165 | def __read_persistent_login(cls) -> (str, str, OSCUser, OSCAPISubDomain): 166 | LOGGER.debug("will read credentials file") 167 | try: 168 | with open(CREDENTIALS_FILE) as json_file: 169 | data = json.load(json_file) 170 | osc_user: OSCUser = None 171 | environment: OSCAPISubDomain = None 172 | if OSC_KEY in data: 173 | osc_data = data[OSC_KEY] 174 | if USER_NAME_KEY in osc_data and \ 175 | USER_ID_KEY in osc_data and \ 176 | USER_FULL_NAME_KEY in osc_data and \ 177 | OSC_ENV_KEY in osc_data and \ 178 | TOKEN_KEY in osc_data: 179 | LOGGER.debug("OSC User found in credentials file") 180 | osc_user = OSCUser() 181 | osc_user.user_id = osc_data[USER_ID_KEY] 182 | osc_user.name = osc_data[USER_NAME_KEY] 183 | osc_user.full_name = osc_data[USER_FULL_NAME_KEY] 184 | osc_user.access_token = osc_data[TOKEN_KEY] 185 | environment = OSCAPISubDomain(osc_data[OSC_ENV_KEY]) 186 | 187 | osm_token = "" 188 | osm_token_secret = "" 189 | if OSM_KEY in data: 190 | osm_data = data[OSM_KEY] 191 | if TOKEN_KEY in osm_data and TOKEN_SECRET_KEY in osm_data: 192 | LOGGER.debug("OSM User found in credentials file") 193 | osm_token = osm_data[TOKEN_KEY] 194 | osm_token_secret = osm_data[TOKEN_SECRET_KEY] 195 | 196 | return osm_token, osm_token_secret, osc_user, environment 197 | except FileNotFoundError: 198 | LOGGER.debug("No file credentials file found") 199 | return "", "", None, None 200 | -------------------------------------------------------------------------------- /logo-KartaView-light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartaview/upload-scripts/28650a631e398ebe8991c1c965248d41ae68dd87/logo-KartaView-light.png -------------------------------------------------------------------------------- /osc_api_config.py: -------------------------------------------------------------------------------- 1 | """This file contains API configurations.""" 2 | 3 | from enum import Enum 4 | 5 | PROTOCOL = "https://" 6 | DOMAIN = "openstreetcam.org" 7 | VERSION = "1.0" 8 | 9 | 10 | class OSCAPISubDomain(Enum): 11 | """This is an enumeration of sub domains. 12 | Default sub domain is PRODUCTION.""" 13 | PRODUCTION = 'api.' 14 | TESTING = 'testing-api.' 15 | STAGING = 'staging-api.' 16 | BETA = 'beta-api.' 17 | -------------------------------------------------------------------------------- /osc_api_gateway.py: -------------------------------------------------------------------------------- 1 | """This module is used as a gateway to the OSC api.""" 2 | import asyncio 3 | import concurrent.futures 4 | import datetime 5 | import hashlib 6 | import os.path 7 | import shutil 8 | import logging 9 | from typing import Tuple, Optional, List 10 | 11 | import requests 12 | import constants 13 | import osc_api_config 14 | from io_storage.storage import Storage 15 | from osc_api_config import OSCAPISubDomain 16 | from osc_api_models import OSCSequence, OSCPhoto, OSCUser 17 | 18 | LOGGER = logging.getLogger('osc_tools.osc_api_gateway') 19 | 20 | 21 | def _upload_url(env: OSCAPISubDomain, resource: str) -> str: 22 | return _osc_url(env) + '/' + _version() + '/' + resource + '/' 23 | 24 | 25 | def _osc_url(env: OSCAPISubDomain) -> str: 26 | base_url = __protocol() + env.value + __domain() 27 | return base_url 28 | 29 | 30 | def __protocol() -> str: 31 | return osc_api_config.PROTOCOL 32 | 33 | 34 | def __domain() -> str: 35 | return osc_api_config.DOMAIN 36 | 37 | 38 | def _version() -> str: 39 | return osc_api_config.VERSION 40 | 41 | 42 | def _website(url: str) -> str: 43 | return url.replace("-api", "").replace("api.", "") 44 | 45 | 46 | class OSCAPIResource: 47 | 48 | @classmethod 49 | def base_url(cls, env: OSCAPISubDomain): 50 | return _osc_url(env) 51 | 52 | @classmethod 53 | def video(cls, env: OSCAPISubDomain, video_id=None) -> str: 54 | if video_id is None: 55 | return _osc_url(env) + "/" + "2.0/video/" 56 | return _osc_url(env) + "/" + "2.0/video/" + str(video_id) 57 | 58 | @classmethod 59 | def photo_part(cls, env: OSCAPISubDomain) -> str: 60 | return _osc_url(env) + "/" + "2.0/photo-part/" 61 | 62 | @classmethod 63 | def photo(cls, env: OSCAPISubDomain, photo_id=None) -> str: 64 | if photo_id is None: 65 | return _osc_url(env) + "/" + "2.0/photo/" 66 | return _osc_url(env) + "/" + "2.0/photo/" + str(photo_id) 67 | 68 | @classmethod 69 | def sequence(cls, env: OSCAPISubDomain, sequence_id=None) -> str: 70 | if sequence_id is None: 71 | return _osc_url(env) + "/2.0/sequence/" 72 | return _osc_url(env) + "/2.0/sequence/" + str(sequence_id) 73 | 74 | @classmethod 75 | def sequence_details(cls, env: OSCAPISubDomain, sequence_id=None) -> str: 76 | return _osc_url(env) + "/details/" + str(sequence_id) 77 | 78 | @classmethod 79 | def user(cls, env: OSCAPISubDomain, user_name=None) -> str: 80 | if user_name is None: 81 | return _osc_url(env) + "/" + "2.0/user/" 82 | return _osc_url(env) + "/" + "2.0/user/" + str(user_name) 83 | 84 | 85 | class OSCApiMethods: 86 | """This is a factory class that creates API methods based on environment""" 87 | 88 | @classmethod 89 | def sequence_create(cls, env: OSCAPISubDomain) -> str: 90 | """this method will return the link to sequence create method""" 91 | return _osc_url(env) + "/" + _version() + "/sequence/" 92 | 93 | @classmethod 94 | def sequence_details(cls, env: OSCAPISubDomain) -> str: 95 | """this method will return the link to the sequence details method""" 96 | return _osc_url(env) + "/details" 97 | 98 | @classmethod 99 | def user_sequences(cls, env: OSCAPISubDomain) -> str: 100 | """this method returns the urls to the list of sequences that 101 | belong to a user""" 102 | return _osc_url(env) + "/my-list" 103 | 104 | @classmethod 105 | def resource(cls, env: OSCAPISubDomain, resource_name: str) -> str: 106 | """this method returns the url to a resource""" 107 | return _osc_url(env) + '/' + resource_name 108 | 109 | @classmethod 110 | def photo_list(cls, env: OSCAPISubDomain) -> str: 111 | """this method returns photo list URL""" 112 | return _osc_url(env) + '/' + _version() + '/sequence/photo-list/' 113 | 114 | @classmethod 115 | def video_upload(cls, env: OSCAPISubDomain) -> str: 116 | """this method returns video upload URL""" 117 | return _upload_url(env, 'video') 118 | 119 | @classmethod 120 | def photo_upload(cls, env: OSCAPISubDomain) -> str: 121 | """this method returns photo upload URL""" 122 | return _upload_url(env, 'photo') 123 | 124 | @classmethod 125 | def login(cls, env: OSCAPISubDomain, provider: str) -> Optional[str]: 126 | """this method returns login URL""" 127 | if provider == "google": 128 | return _osc_url(env) + '/auth/google/client_auth' 129 | if provider == "facebook": 130 | return _osc_url(env) + '/auth/facebook/client_auth' 131 | # default to OSM 132 | return _osc_url(env) + '/auth/openstreetmap/client_auth' 133 | 134 | @classmethod 135 | def finish_upload(cls, env: OSCAPISubDomain) -> str: 136 | """this method returns a finish upload url""" 137 | return _osc_url(env) + '/' + _version() + '/sequence/finished-uploading/' 138 | 139 | 140 | class OSCApi: 141 | """This class is a gateway for the API""" 142 | 143 | def __init__(self, env: OSCAPISubDomain): 144 | self.environment = env 145 | 146 | @classmethod 147 | def __upload_response_success(cls, response: requests.Response, 148 | upload_type: str, 149 | index: int, 150 | sequence_id: int) -> bool: 151 | if response is None: 152 | return False 153 | try: 154 | json_response = response.json() 155 | if response.status_code != 200: 156 | if "status" in json_response and \ 157 | "apiMessage" in json_response["status"] and \ 158 | "duplicate entry" in json_response["status"]["apiMessage"]: 159 | LOGGER.debug("Received duplicate %s index: %d, photo_id %s sequence_id %s", 160 | upload_type, 161 | index, 162 | None, 163 | sequence_id) 164 | return True 165 | LOGGER.debug("Failed to upload %s index: %d response:%s sequence_id %s", 166 | upload_type, 167 | index, 168 | json_response, 169 | sequence_id) 170 | return False 171 | 172 | if ("osv" in json_response and 173 | (("photo" in json_response["osv"] and "id" in json_response["osv"]["photo"]) or 174 | ("video" in json_response["osv"] and "id" in json_response["osv"]["video"]))): 175 | return True 176 | except ValueError: 177 | return False 178 | return False 179 | 180 | def _sequence_page(self, user_name, page) -> Tuple[List[OSCSequence], Exception]: 181 | try: 182 | parameters = {'ipp': 100, 183 | 'page': page, 184 | 'username': user_name} 185 | login_url = OSCApiMethods.user_sequences(self.environment) 186 | response = requests.post(url=login_url, data=parameters) 187 | json_response = response.json() 188 | 189 | sequences = [] 190 | if 'currentPageItems' in json_response: 191 | items = json_response['currentPageItems'] 192 | for item in items: 193 | sequence = OSCSequence.sequence_from_json(item) 194 | sequences.append(sequence) 195 | 196 | return sequences, None 197 | except requests.RequestException as ex: 198 | return None, ex 199 | 200 | def authorized_user(self, provider: str, token: str, secret: str) -> Tuple[OSCUser, Exception]: 201 | """This method will get a authorization token for OSC API""" 202 | try: 203 | data_access = {'request_token': token, 204 | 'secret_token': secret 205 | } 206 | login_url = OSCApiMethods.login(self.environment, provider) 207 | response = requests.post(url=login_url, data=data_access) 208 | json_response = response.json() 209 | 210 | if 'osv' in json_response: 211 | osc_data = json_response['osv'] 212 | user = OSCUser() 213 | missing_field = None 214 | if 'access_token' in osc_data: 215 | user.access_token = osc_data['access_token'] 216 | else: 217 | missing_field = "access token" 218 | 219 | if 'id' in osc_data: 220 | user.user_id = osc_data['id'] 221 | else: 222 | missing_field = "id" 223 | 224 | if 'username' in osc_data: 225 | user.name = osc_data['username'] 226 | else: 227 | missing_field = "username" 228 | 229 | if 'full_name' in osc_data: 230 | user.full_name = osc_data['full_name'] 231 | else: 232 | missing_field = "fullname" 233 | 234 | if missing_field is not None: 235 | return None, Exception("OSC API bug. OSCUser missing " + missing_field) 236 | 237 | else: 238 | return None, Exception("OSC API bug. OSCUser missing username") 239 | 240 | except requests.RequestException as ex: 241 | return None, ex 242 | 243 | return user, None 244 | 245 | def get_photos(self, sequence_id, page=None) -> Tuple[List[OSCPhoto], Optional[Exception]]: 246 | try: 247 | photo_url = OSCAPIResource.photo(self.environment) 248 | has_more_data = True 249 | photos = [] 250 | current_page = page 251 | if page is None or page < 1: 252 | current_page = 1 253 | while has_more_data: 254 | response = requests.get(photo_url, 255 | params={"sequenceId": sequence_id, 256 | "page": current_page, 257 | "itemsPerPage": 500}) 258 | json_response = response.json() 259 | result = json_response.get("result", {}) 260 | photo_list = result.get("data", []) 261 | for photo_json in photo_list: 262 | photo = OSCPhoto.photo_from_json(photo_json) 263 | photos.append(photo) 264 | has_more_data = result.get("hasMoreData", False) 265 | if page is not None: 266 | break 267 | if has_more_data: 268 | current_page += 1 269 | return photos, None 270 | except requests.RequestException as ex: 271 | return [], ex 272 | 273 | def download_all_images(self, photo_list: [OSCPhoto], 274 | track_path: str, 275 | override=False, 276 | workers: int = 10): 277 | """This method will download all images to a path overriding or not the files at 278 | that path. By default this method uses 10 parallel workers.""" 279 | with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: 280 | loop = asyncio.new_event_loop() 281 | futures = [ 282 | loop.run_in_executor(executor, 283 | self.get_image, photo, track_path, override) 284 | for photo in photo_list 285 | ] 286 | if not futures: 287 | loop.close() 288 | return 289 | 290 | loop.run_until_complete(asyncio.gather(*futures)) 291 | loop.close() 292 | 293 | def get_image(self, photo: OSCPhoto, path: str, override=False) -> Optional[Exception]: 294 | """downloads the image at the path specified""" 295 | jpg_name = path + '/' + str(photo.sequence_index) + '.jpg' 296 | if not override and os.path.isfile(jpg_name): 297 | return None 298 | 299 | try: 300 | response = requests.get(OSCApiMethods.resource(self.environment, 301 | photo.image_name), 302 | stream=True) 303 | if response.status_code == 200: 304 | with open(jpg_name, 'wb') as file: 305 | response.raw.decode_content = True 306 | shutil.copyfileobj(response.raw, file) 307 | except requests.RequestException as ex: 308 | return ex 309 | return None 310 | 311 | def user_sequences(self, user_name: str) -> Tuple[List[OSCSequence], Exception]: 312 | """get all tracks for a user id """ 313 | LOGGER.debug("getting all sequences for user: %s", user_name) 314 | try: 315 | parameters = {'ipp': 100, 316 | 'page': 1, 317 | 'username': user_name} 318 | json_response = requests.post(url=OSCApiMethods.user_sequences(self.environment), 319 | data=parameters).json() 320 | 321 | if 'totalFilteredItems' not in json_response: 322 | return [], Exception("OSC API bug missing totalFilteredItems from response") 323 | 324 | total_items = int(json_response['totalFilteredItems'][0]) 325 | pages_count = int(total_items / parameters['ipp']) + 1 326 | LOGGER.debug("all sequences count: %s pages count: %s", 327 | str(total_items), str(pages_count)) 328 | sequences = [] 329 | if 'currentPageItems' in json_response: 330 | for item in json_response['currentPageItems']: 331 | sequences.append(OSCSequence.sequence_from_json(item)) 332 | 333 | with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: 334 | loop = asyncio.new_event_loop() 335 | futures = [ 336 | loop.run_in_executor(executor, 337 | self._sequence_page, user_name, page) 338 | for page in range(2, pages_count + 1) 339 | ] 340 | if not futures: 341 | loop.close() 342 | return sequences, None 343 | 344 | done = loop.run_until_complete(asyncio.gather(*futures)) 345 | loop.close() 346 | 347 | for sequence_page_return in done: 348 | # sequence_page method will return a tuple the first element 349 | # is a list of sequences 350 | sequences = sequences + sequence_page_return[0] 351 | 352 | return sequences, None 353 | except requests.RequestException as ex: 354 | return None, ex 355 | 356 | def sequence_link(self, sequence) -> str: 357 | """This method will return a link to OSC website page displaying the sequence 358 | sent as parameter""" 359 | sequence_details_url = OSCApiMethods.sequence_details(self.environment) 360 | return _website(f"{sequence_details_url}/{str(sequence.online_id)}") 361 | 362 | def download_metadata(self, sequence: OSCSequence, path: str, override=False): 363 | """this method will download a metadata file of a sequence to the specified path. 364 | If there is a metadata file at that path by default no override will be made.""" 365 | if sequence.metadata_url is None: 366 | return None 367 | metadata_path = path + "/track.txt" 368 | if not override and os.path.isfile(metadata_path): 369 | return None 370 | 371 | try: 372 | response = requests.get(OSCApiMethods.resource(self.environment, 373 | sequence.metadata_url), 374 | stream=True) 375 | if response.status_code == 200: 376 | with open(metadata_path, 'wb') as file: 377 | response.raw.decode_content = True 378 | shutil.copyfileobj(response.raw, file) 379 | except requests.RequestException as ex: 380 | return ex 381 | 382 | return None 383 | 384 | def create_sequence(self, sequence: OSCSequence, token: str) -> Tuple[int, Exception]: 385 | """this method will create a online sequence from the current sequence and will return its 386 | id as a integer or a exception if fail""" 387 | try: 388 | parameters = {'uploadSource': 'kv_tools-0.1.0', 389 | 'access_token': token, 390 | 'currentCoordinate': sequence.location() 391 | } 392 | if sequence.platform is not None: 393 | parameters['platformName'] = sequence.platform 394 | 395 | if sequence.device is not None: 396 | parameters['deviceName'] = sequence.device 397 | 398 | url = OSCApiMethods.sequence_create(self.environment) 399 | if sequence.metadata_url: 400 | with open(sequence.metadata_url, 'rb') as metadata_file: 401 | load_data = {'metaData': (constants.METADATA_NAME, 402 | metadata_file, 403 | 'text/plain')} 404 | response = requests.post(url, 405 | data=parameters, 406 | files=load_data) 407 | else: 408 | response = requests.post(url, data=parameters) 409 | json_response = response.json() 410 | if 'osv' in json_response: 411 | osc_data = json_response["osv"] 412 | if "sequence" in osc_data: 413 | sequence = OSCSequence.sequence_from_json(osc_data["sequence"]) 414 | return sequence.online_id, None 415 | except requests.RequestException as ex: 416 | return None, ex 417 | 418 | return None, None 419 | 420 | def finish_upload(self, sequence: OSCSequence, token: str) -> Tuple[Optional[bool], 421 | Optional[Exception]]: 422 | """this method must be called in order to signal that a sequence has no more data to be 423 | uploaded.""" 424 | try: 425 | parameters = {'sequenceId': sequence.online_id, 426 | 'access_token': token} 427 | response = requests.post(OSCApiMethods.finish_upload(self.environment), 428 | data=parameters) 429 | json_response = response.json() 430 | if "status" not in json_response: 431 | # we don't have a proper status documentation 432 | return False, None 433 | return True, None 434 | except requests.RequestException as ex: 435 | return None, ex 436 | 437 | def upload_video(self, access_token, 438 | sequence_id, 439 | video_path: str, 440 | video_index) -> Tuple[bool, Optional[Exception]]: 441 | """This method will upload a video to OSC API""" 442 | try: 443 | parameters = {'access_token': access_token, 444 | 'sequenceId': sequence_id, 445 | 'sequenceIndex': video_index 446 | } 447 | with open(video_path, 'rb') as video_file: 448 | load_data = {'video': (os.path.basename(video_path), 449 | video_file, 450 | 'video/mp4')} 451 | video_upload_url = OSCApiMethods.video_upload(self.environment) 452 | response = requests.post(video_upload_url, 453 | data=parameters, 454 | files=load_data, 455 | timeout=100) 456 | return OSCApi.__upload_response_success(response, 457 | "video", 458 | video_index, 459 | sequence_id), None 460 | except requests.RequestException as ex: 461 | LOGGER.debug("Received exception on video upload %s", str(ex)) 462 | return False, ex 463 | 464 | # pylint: disable=R0913,R0914 465 | def upload_photo(self, access_token, 466 | sequence_id, 467 | photo: OSCPhoto, 468 | photo_path: str, 469 | fov=None, 470 | projection=None) -> Tuple[bool, Optional[Exception]]: 471 | """This method will upload a photo to OSC API""" 472 | LOGGER.debug("uploading photo %s, sequence id %s", photo_path, sequence_id) 473 | try: 474 | shot_date = datetime.datetime.utcfromtimestamp(photo.timestamp) 475 | shot_date_string = shot_date.strftime('%Y-%m-%d %H:%M:%S') 476 | parameters = {'access_token': access_token, 477 | 'coordinate': str(photo.latitude) + "," + str(photo.longitude), 478 | 'sequenceId': sequence_id, 479 | 'sequenceIndex': photo.sequence_index, 480 | 'shotDate': shot_date_string 481 | } 482 | if photo.compass: 483 | parameters["headers"] = photo.compass 484 | 485 | if fov is not None and projection is not None: 486 | parameters['projection'] = projection 487 | parameters['fieldOfView'] = fov 488 | 489 | if photo.yaw is not None: 490 | parameters["projectionYaw"] = photo.yaw 491 | 492 | photo_upload_url = OSCApiMethods.photo_upload(self.environment) 493 | name = str(hashlib.md5(os.path.basename(photo.image_name).encode()).hexdigest()) 494 | extension = os.path.split(photo.image_name)[1] 495 | name += extension 496 | with open(photo_path, 'rb') as image_file: 497 | load_data = {'photo': (name, 498 | image_file, 499 | 'image/jpeg')} 500 | response = requests.post(photo_upload_url, 501 | data=parameters, 502 | files=load_data, 503 | timeout=100) 504 | success = self.__upload_response_success(response, 505 | "photo", 506 | photo.sequence_index, 507 | sequence_id) 508 | return success, None 509 | except requests.RequestException as ex: 510 | LOGGER.debug("Received exception on photo upload %s", str(ex)) 511 | return False, ex 512 | # pylint: enable=R0913,R0914 513 | 514 | def get_sequence(self, sequence_id) -> Tuple[Optional[OSCSequence], Optional[Exception]]: 515 | try: 516 | sequence_url = OSCAPIResource.sequence(self.environment, sequence_id) 517 | response = requests.get(sequence_url) 518 | response.raise_for_status() 519 | except requests.RequestException as ex: 520 | return None, ex 521 | 522 | json_response = response.json() 523 | if json_response['status']['apiCode'] != 600: 524 | return None, Exception(json_response['status']['apiMessage']) 525 | 526 | result = json_response.get("result", {}) 527 | sequence_json = result.get("data", {}) 528 | sequence = OSCSequence.from_json(sequence_json) 529 | return sequence, None 530 | 531 | @staticmethod 532 | def download_resource(resource_url: str, 533 | file_path: str, 534 | storage: Storage, 535 | override=False) -> Tuple[bool, Optional[Exception]]: 536 | if not override and storage.isfile(file_path): 537 | return True, None 538 | try: 539 | with requests.get(resource_url) as response: 540 | response.raise_for_status() 541 | storage.put(response.content, file_path + "partial") 542 | storage.rename(file_path + "partial", file_path) 543 | return True, None 544 | except requests.RequestException as ex: 545 | return False, ex 546 | -------------------------------------------------------------------------------- /osc_api_models.py: -------------------------------------------------------------------------------- 1 | """This file contains all the osc api models """ 2 | # pylint: disable=R0902 3 | 4 | import datetime 5 | from decimal import Decimal 6 | from typing import Optional 7 | 8 | 9 | class OSCUser: 10 | """This class is model for OSC User""" 11 | 12 | def __init__(self): 13 | self.name = "" 14 | self.user_id = "" 15 | self.full_name = "" 16 | self.access_token = "" 17 | 18 | def description(self) -> str: 19 | """description method that will return a string representation for this class""" 20 | return "{ name: " + self.name + \ 21 | ", user_id: " + self.user_id + \ 22 | ", full_name: " + self.full_name + \ 23 | ", access_token = " + self.access_token + " }" 24 | 25 | def __eq__(self, other): 26 | if isinstance(other, OSCUser): 27 | return self.user_id == other.user_id 28 | return False 29 | 30 | def __hash__(self): 31 | return hash(self.user_id) 32 | 33 | 34 | class OSCPhoto: 35 | """this is a model class for a photo from OSC API""" 36 | 37 | def __init__(self): 38 | self.timestamp = None 39 | self.latitude = None 40 | self.longitude = None 41 | self.compass = None 42 | self.sequence_index = None 43 | self.photo_id = None 44 | self.image_name = None 45 | self.date_added = None 46 | self.status = None 47 | self.processing_status = None 48 | self._file_url = None 49 | self.yaw = None 50 | self.projection = None 51 | self.field_of_view = None 52 | 53 | @classmethod 54 | def photo_from_json(cls, json): 55 | """factory method to build a photo from a json representation""" 56 | shot_date = json.get('shotDate', None) 57 | if shot_date is None: 58 | return None 59 | photo = OSCPhoto() 60 | photo.photo_id = json.get('id', None) 61 | photo.latitude = json.get('lat', None) 62 | photo.longitude = json.get('lng', None) 63 | photo.timestamp = datetime.datetime.strptime(shot_date, '%Y-%m-%d %H:%M:%S').timestamp() 64 | photo.sequence_index = json.get('sequenceIndex', None) 65 | 66 | if photo.photo_id is None or \ 67 | photo.latitude is None or \ 68 | photo.longitude is None or \ 69 | photo.sequence_index is None or \ 70 | photo.timestamp is None: 71 | return None 72 | 73 | photo.photo_id = int(photo.photo_id) 74 | photo.latitude = float(photo.latitude) 75 | photo.longitude = float(photo.longitude) 76 | photo.sequence_index = int(photo.sequence_index) 77 | 78 | photo.date_added = json.get('dateAdded', None) 79 | photo.status = json.get("status", None) 80 | photo.processing_status = json.get("autoImgProcessingStatus", None) 81 | photo._file_url = json.get("fileurl", None) 82 | photo.field_of_view = int(json.get("fieldOfView")) if json.get("fieldOfView") else None 83 | photo.projection = json.get("projection", None) 84 | photo.yaw = json.get("projectionYaw", None) 85 | 86 | return photo 87 | 88 | def photo_url(self): 89 | return self._proc_url() if self.projection == "PLANE" else self._wrapped_proc_url 90 | 91 | def _wrapped_proc_url(self): 92 | if self._file_url is None: 93 | return None 94 | return self._file_url.replace("{{sizeprefix}}", "wrapped_proc") 95 | 96 | def _proc_url(self): 97 | if self._file_url is None: 98 | return None 99 | return self._file_url.replace("{{sizeprefix}}", "proc") 100 | 101 | def __eq__(self, other): 102 | if isinstance(other, OSCPhoto): 103 | return self.photo_id == other.photo_id 104 | return False 105 | 106 | def __hash__(self): 107 | return hash(self.photo_id) 108 | 109 | 110 | class OSCCameraParameters: 111 | def __init__(self, focal_length, horizontal_fov, vertical_fov, 112 | aperture: Optional[Decimal] = None): 113 | self.focal_length: Decimal = focal_length 114 | self.horizontal_fov: Decimal = horizontal_fov 115 | self.vertical_fov: Decimal = vertical_fov 116 | self.aperture: Optional[Decimal] = aperture 117 | 118 | @classmethod 119 | def from_json(cls, json): 120 | if json is None: 121 | return None 122 | camera = OSCCameraParameters( 123 | json["fLen"], 124 | json["hFoV"], 125 | json["vFoV"], 126 | json["aperture"] 127 | ) 128 | return camera 129 | 130 | 131 | class OSCSequence: 132 | """this is a model class for a sequence from OSC API""" 133 | 134 | def __init__(self): 135 | self.photos: [OSCPhoto] = [] 136 | self.local_id: Optional[str] = None 137 | self.online_id: Optional[str] = None 138 | self.device: Optional[str] = None 139 | self.platform: Optional[str] = None 140 | self.path: Optional[str] = None 141 | self.metadata_url = None 142 | self.latitude = None 143 | self.longitude = None 144 | self.processing_status: Optional[str] = None 145 | self.status: Optional[str] = None 146 | self.recording_type: Optional[str] = None 147 | self.upload_status: Optional[str] = None 148 | self.processing_list: Optional[list] = None 149 | self.recording_date: Optional[str] = None 150 | self.length: Optional[int] = None 151 | self.app_version: Optional[str] = None 152 | self.points: Optional[str] = None 153 | self.total_images: Optional[int] = None 154 | self.date_added: Optional[str] = None 155 | self.is_video: Optional[bool] = None 156 | self.camera_parameters: Optional[OSCCameraParameters] = None 157 | 158 | 159 | @classmethod 160 | def sequence_from_json(cls, json): 161 | """factory method to build a sequence form json representation""" 162 | sequence = OSCSequence() 163 | if 'id' in json: 164 | sequence.online_id = json['id'] 165 | if 'meta_data_filename' in json: 166 | sequence.metadata_url = json['meta_data_filename'] 167 | if 'photos' in json: 168 | photos = [] 169 | photos_json = json['photos'] 170 | for photo_json in photos_json: 171 | photo = OSCPhoto.photo_from_json(photo_json) 172 | photos.append(photo) 173 | sequence.photos = photos 174 | 175 | return sequence 176 | 177 | @classmethod 178 | def from_json(cls, json): 179 | sequence = OSCSequence() 180 | sequence.online_id = json.get('id', None) 181 | sequence.metadata_url = json.get('metadataFileUrl', None) 182 | sequence.processing_status = json.get('processingStatus', None) 183 | sequence.status = json.get('status', None) 184 | sequence.device = json.get('deviceName', None) 185 | sequence.platform = json.get('platformName', None) 186 | sequence.latitude = json.get('currentLat', None) 187 | sequence.longitude = json.get('currentLng', None) 188 | sequence.app_version = json.get('app_version', None) 189 | sequence_date_added = json.get('dateAdded', None) 190 | sequence.total_images = int(json.get('countActivePhotos')) if json.get( 191 | 'countActivePhotos') else None 192 | sequence.is_video = bool(json.get('isVideo', None) == "1") 193 | sequence.camera_parameters = OSCCameraParameters.from_json(json["cameraParameters"]) 194 | if sequence_date_added is not None: 195 | sequence.date_added = datetime.datetime.strptime(sequence_date_added, 196 | '%Y-%m-%d %H:%M:%S') 197 | 198 | return sequence 199 | 200 | def __eq__(self, other): 201 | if isinstance(other, OSCSequence): 202 | return self.local_id == other.local_id and self.online_id == other.online_id 203 | return False 204 | 205 | def __hash__(self): 206 | return hash((self.local_id, self.online_id)) 207 | 208 | def location(self) -> str: 209 | """this method returns the string representation of a OSCSequence location""" 210 | if self.latitude is not None and self.longitude is not None: 211 | return str(self.latitude) + "," + str(self.longitude) 212 | 213 | if self.photos: 214 | photo = self.photos[0] 215 | return str(photo.latitude) + "," + str(photo.longitude) 216 | return "" 217 | 218 | # pylint: enable=R0902 219 | -------------------------------------------------------------------------------- /osc_discoverer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """This script is used to discover video files, sensor info and return Sequences.""" 3 | import os 4 | import json 5 | import logging 6 | from typing import List, cast 7 | 8 | import constants 9 | from common.models import GPS, OSCDevice 10 | from io_storage.storage import Local 11 | from parsers.custom_data_parsers.custom_mapillary import MapillaryExif 12 | from parsers.exif.exif import ExifParser 13 | from parsers.osc_metadata.parser import metadata_parser 14 | from visual_data_discover import VisualDataDiscoverer 15 | from visual_data_discover import ExifPhotoDiscoverer 16 | from visual_data_discover import MapillaryExifDiscoverer 17 | from visual_data_discover import PhotoMetadataDiscoverer 18 | from visual_data_discover import VideoDiscoverer 19 | from validators import SequenceValidator, SequenceMetadataValidator, SequenceFinishedValidator 20 | from osc_utils import unzip_metadata 21 | from osc_models import Sequence, Photo, VisualData 22 | 23 | LOGGER = logging.getLogger('osc_tools.osc_discoverer') 24 | 25 | 26 | class OSCUploadProgressDiscoverer: 27 | """This class is responsible with finding a upload progress file""" 28 | def __eq__(self, other): 29 | if isinstance(other, OSCUploadProgressDiscoverer): 30 | return self == other 31 | return False 32 | 33 | @classmethod 34 | def discover(cls, path: str) -> List[str]: 35 | """this method will discover a upload progress file and parse it to get a progress list.""" 36 | LOGGER.debug("will read uploaded indexes") 37 | progress_file_path = path + "/" + constants.PROGRESS_FILE_NAME 38 | if not os.path.isfile(progress_file_path): 39 | return [] 40 | with open(progress_file_path, 'r') as input_file: 41 | line = input_file.readline() 42 | indexes = list(filter(None, line.split(";"))) 43 | return indexes 44 | 45 | 46 | class OSCMetadataDiscoverer: 47 | """this class will discover a metadata file""" 48 | 49 | def __eq__(self, other): 50 | if isinstance(other, OSCMetadataDiscoverer): 51 | return self == other 52 | return False 53 | 54 | @classmethod 55 | def discover(cls, path: str) -> str: 56 | """This method will discover osc metadata path""" 57 | files = os.listdir(path) 58 | for file_path in files: 59 | file_name, file_extension = os.path.splitext(file_path) 60 | if ".txt" in file_extension and "track" in file_name: 61 | return path + "/" + file_path 62 | if ".gz" in file_extension and "track" in file_name: 63 | return unzip_metadata(path) 64 | return None 65 | # if no metadata found generate metadata from gpx or exif 66 | 67 | 68 | class OnlineIDDiscoverer: 69 | """This class will discover online id of a sequence""" 70 | 71 | @classmethod 72 | def discover(cls, path: str) -> str: 73 | """This method will discover online id""" 74 | LOGGER.debug("searching for metadata %s", path) 75 | sequence_file_path = path + "/osc_sequence_id.txt" 76 | if not os.path.isfile(sequence_file_path): 77 | return None 78 | 79 | try: 80 | with open(sequence_file_path) as json_file: 81 | data = json.load(json_file) 82 | if "id" in data and data["id"] and str.isdigit(data["id"]): 83 | return int(data["id"]) 84 | except FileNotFoundError: 85 | return None 86 | return None 87 | 88 | @classmethod 89 | def discover_using_type(cls, path: str, osc_type: str): 90 | """this method is discovering the online id""" 91 | print(path) 92 | print(osc_type) 93 | 94 | 95 | class SequenceDiscoverer: 96 | """Seq discoverer base class""" 97 | def __init__(self): 98 | self.ignored_for_upload: bool = False 99 | self.name = "default" 100 | self.online_id: OnlineIDDiscoverer = OnlineIDDiscoverer() 101 | self.visual_data: VisualDataDiscoverer = VisualDataDiscoverer() 102 | self.osc_metadata: OSCMetadataDiscoverer = OSCMetadataDiscoverer() 103 | self.upload_progress: OSCUploadProgressDiscoverer = OSCUploadProgressDiscoverer() 104 | self.validator: SequenceValidator = SequenceValidator() 105 | 106 | def discover(self, path: str) -> [Sequence]: 107 | """This method will discover a valid sequence""" 108 | files = os.listdir(path) 109 | sequences = [] 110 | for file_path in files: 111 | full_path = os.path.join(path, file_path) 112 | if os.path.isdir(full_path): 113 | sequences = sequences + self.discover(full_path) 114 | sequence = self.create_sequence(path) 115 | if self.validator.validate(sequence): 116 | sequences.append(sequence) 117 | else: 118 | LOGGER.debug("This sequence (%s) does not conform to this discoverer %s.", path, 119 | self.name) 120 | return sequences 121 | 122 | def create_sequence(self, path): 123 | """This method will discover all attributes af a sequence""" 124 | sequence = Sequence() 125 | if self.online_id: 126 | sequence.online_id = self.online_id.discover(path) 127 | 128 | if self.visual_data: 129 | (visual_data, data_type) = self.visual_data.discover(path) 130 | sequence.visual_items = visual_data 131 | sequence.visual_data_type = data_type 132 | 133 | if self.osc_metadata: 134 | sequence.osc_metadata = self.osc_metadata.discover(path) 135 | 136 | if self.upload_progress: 137 | sequence.progress = self.upload_progress.discover(path) 138 | sequence.path = path 139 | self._find_latitude_longitude_device_info(sequence) 140 | 141 | return sequence 142 | 143 | def _find_latitude_longitude_device_info(self, sequence: Sequence): 144 | if not sequence.online_id: 145 | if sequence.osc_metadata and isinstance(self.validator, SequenceMetadataValidator): 146 | parser = metadata_parser(sequence.osc_metadata, Local()) 147 | gps = cast(GPS, parser.next_item_with_class(GPS)) 148 | if gps: 149 | sequence.latitude = gps.latitude 150 | sequence.longitude = gps.longitude 151 | device_info: OSCDevice = cast(OSCDevice, parser.next_item_with_class(OSCDevice)) 152 | if device_info: 153 | sequence.device = device_info.device_raw_name 154 | sequence.platform = device_info.platform_name 155 | elif sequence.visual_items: 156 | visual_item: VisualData = sequence.visual_items[0] 157 | if isinstance(self.visual_data, ExifPhotoDiscoverer): 158 | parser = ExifParser(visual_item.path, Local()) 159 | device_info: OSCDevice = parser.next_item_with_class(OSCDevice) 160 | if device_info: 161 | sequence.device = device_info.device_raw_name 162 | sequence.platform = device_info.platform_name 163 | if isinstance(self.visual_data, MapillaryExifDiscoverer): 164 | parser = MapillaryExif(visual_item.path, Local()) 165 | device_info: OSCDevice = parser.next_item_with_class(OSCDevice) 166 | if device_info: 167 | sequence.device = device_info.device_raw_name 168 | sequence.platform = device_info.platform_name 169 | if isinstance(visual_item, Photo): 170 | sequence.latitude = visual_item.latitude 171 | sequence.longitude = visual_item.longitude 172 | 173 | 174 | class SequenceDiscovererFactory: 175 | """Class that builds a list of sequence discoverers ready to use.""" 176 | 177 | @classmethod 178 | def discoverers(cls) -> [SequenceDiscoverer]: 179 | """This is a factory method that will return Sequence Discoverers""" 180 | return [cls.finished_discoverer(), 181 | cls.photo_metadata_discoverer(), 182 | cls.exif_discoverer(), 183 | cls.mapillary_exif_discoverer(), 184 | cls.video_discoverer()] 185 | 186 | @classmethod 187 | def photo_metadata_discoverer(cls) -> SequenceDiscoverer: 188 | """This method will return a photo discoverer""" 189 | photo_metadata_finder = SequenceDiscoverer() 190 | photo_metadata_finder.name = "Metadata-Photo" 191 | photo_metadata_finder.visual_data = PhotoMetadataDiscoverer() 192 | photo_metadata_finder.validator = SequenceMetadataValidator() 193 | return photo_metadata_finder 194 | 195 | @classmethod 196 | def exif_discoverer(cls) -> SequenceDiscoverer: 197 | """This method will return a photo discoverer""" 198 | exif_photo_finder = SequenceDiscoverer() 199 | exif_photo_finder.name = "Exif-Photo" 200 | exif_photo_finder.visual_data = ExifPhotoDiscoverer() 201 | exif_photo_finder.osc_metadata = None 202 | return exif_photo_finder 203 | 204 | @classmethod 205 | def mapillary_exif_discoverer(cls) -> SequenceDiscoverer: 206 | """This method will return a photo discoverer""" 207 | exif_photo_finder = SequenceDiscoverer() 208 | exif_photo_finder.name = "MapillaryExif-Photo" 209 | exif_photo_finder.visual_data = MapillaryExifDiscoverer() 210 | exif_photo_finder.osc_metadata = None 211 | return exif_photo_finder 212 | 213 | @classmethod 214 | def video_discoverer(cls) -> SequenceDiscoverer: 215 | """this method will return a video discoverer""" 216 | video_finder = SequenceDiscoverer() 217 | video_finder.name = "Metadata-Video" 218 | video_finder.visual_data = VideoDiscoverer() 219 | video_finder.validator = SequenceMetadataValidator() 220 | return video_finder 221 | 222 | @classmethod 223 | def finished_discoverer(cls) -> SequenceDiscoverer: 224 | """this method will return a discoverer that finds all the sequences that finished upload""" 225 | finished_finder = SequenceDiscoverer() 226 | finished_finder.name = "Done Uploading" 227 | finished_finder.ignored_for_upload = True 228 | finished_finder.visual_data = None 229 | finished_finder.osc_metadata = None 230 | finished_finder.validator = SequenceFinishedValidator() 231 | return finished_finder 232 | -------------------------------------------------------------------------------- /osc_models.py: -------------------------------------------------------------------------------- 1 | """osc_models module contains all the application level models""" 2 | # pylint: disable=R0902 3 | 4 | from typing import Optional 5 | 6 | from common.models import CameraProjection 7 | 8 | 9 | class Sequence: 10 | """Sequence is a model class containing a list of visual items""" 11 | 12 | def __init__(self): 13 | self.path: str = "" 14 | self.online_id: str = "" 15 | self.progress: [str] = [] 16 | self.visual_items: [VisualData] = [] 17 | self.osc_metadata: str = "" 18 | self.visual_data_type: str = "" 19 | self.latitude: float = None 20 | self.longitude: float = None 21 | self.platform: Optional[str] = None 22 | self.device: Optional[str] = None 23 | 24 | @property 25 | def description(self) -> str: 26 | """this method returns a string description of a sequence""" 27 | return self.online_id + self.osc_metadata + self.visual_data_type 28 | 29 | def visual_data_count(self) -> int: 30 | """this method returns the count of visual data""" 31 | return len(self.visual_items) 32 | 33 | def __eq__(self, other): 34 | """Overrides the default implementation""" 35 | if isinstance(other, Sequence): 36 | return self.path == other.path 37 | return False 38 | 39 | def __hash__(self): 40 | return hash(self.path) 41 | 42 | 43 | class VisualData: 44 | """VisualData is a model class for a visual item""" 45 | def __init__(self, path): 46 | self.path: str = path 47 | self.index: int = None 48 | 49 | def __eq__(self, other): 50 | if isinstance(other, VisualData): 51 | return self.path == other.path and \ 52 | self.index == other.index 53 | return False 54 | 55 | def __hash__(self): 56 | return hash((self.path, self.index)) 57 | 58 | 59 | class Photo(VisualData): 60 | """Photo is a VisualData model for a photo item""" 61 | 62 | def __init__(self, path): 63 | super().__init__(path) 64 | self.latitude: float = None 65 | self.longitude: float = None 66 | self.exif_timestamp: float = None 67 | self.gps_timestamp: float = None 68 | self.gps_speed: float = None 69 | self.gps_altitude: float = None 70 | self.gps_compass: float = None 71 | self.fov: Optional[float] = None 72 | self.projection: CameraProjection = None 73 | 74 | def __eq__(self, other): 75 | if isinstance(other, Photo): 76 | return self.gps_timestamp == other.gps_timestamp and \ 77 | self.latitude == other.path and \ 78 | self.longitude == other.longitude 79 | return False 80 | 81 | def __hash__(self): 82 | return hash((self.gps_timestamp, 83 | self.latitude, 84 | self.longitude)) 85 | 86 | 87 | class Video(VisualData): 88 | """Video is a VisualData model for a video item""" 89 | def __eq__(self, other): 90 | if isinstance(other, Video): 91 | return self.path == other.path and self.index == other.index 92 | return False 93 | 94 | def __hash__(self): 95 | return hash((self.path, self.index)) 96 | 97 | # pylint: enable=R0902 98 | -------------------------------------------------------------------------------- /osc_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Tools developed by KartaView to help contributors.""" 3 | 4 | import logging 5 | import os 6 | from argparse import ArgumentParser, RawTextHelpFormatter, SUPPRESS, Namespace 7 | 8 | from download import download_user_images 9 | from login_controller import LoginController 10 | from osc_api_config import OSCAPISubDomain 11 | from osc_uploader import OSCUploadManager 12 | from osc_utils import create_exif 13 | from osc_discoverer import SequenceDiscovererFactory 14 | 15 | LOGGER = logging.getLogger('osc_tools') 16 | OSC_LOG_FILE = 'OSC_logs.log' 17 | 18 | 19 | def main(): 20 | """Entry point for the OSC scripts""" 21 | args = get_args() 22 | configure_log(args) 23 | # call the right sub command 24 | args.func(args) 25 | 26 | 27 | def configure_login(args) -> LoginController: 28 | """Method to configure upload environment""" 29 | if args.env == 'p': 30 | LOGGER.debug("environment production") 31 | controller = LoginController(OSCAPISubDomain.PRODUCTION) 32 | elif args.env == 't': 33 | LOGGER.debug("environment testing") 34 | controller = LoginController(OSCAPISubDomain.TESTING) 35 | elif args.env == 's': 36 | LOGGER.debug("environment staging") 37 | controller = LoginController(OSCAPISubDomain.STAGING) 38 | elif args.env == 'b': 39 | LOGGER.debug("environment beta") 40 | controller = LoginController(OSCAPISubDomain.BETA) 41 | else: 42 | LOGGER.debug("environment default production") 43 | controller = LoginController(OSCAPISubDomain.PRODUCTION) 44 | return controller 45 | 46 | 47 | def configure_log(args): 48 | """Method to configure logging level""" 49 | LOGGER.setLevel(logging.DEBUG) 50 | 51 | # create file handler which logs even debug messages 52 | file = logging.FileHandler(OSC_LOG_FILE) 53 | file.setLevel(logging.DEBUG) 54 | formatter = logging.Formatter('%(asctime)s %(name)-35s: %(levelname)-8s %(message)s') 55 | file.setFormatter(formatter) 56 | 57 | # create console handler with a higher log level 58 | console = logging.StreamHandler() 59 | console.setLevel(logging.WARNING) 60 | 61 | if args.log_level == 'd': 62 | console.setLevel(logging.DEBUG) 63 | console.setFormatter(formatter) 64 | LOGGER.debug("Debug logging selected") 65 | elif args.log_level == 'i': 66 | console.setLevel(logging.INFO) 67 | formatter = logging.Formatter('%(message)s') 68 | console.setFormatter(formatter) 69 | LOGGER.debug("info logging selected") 70 | elif args.log_level == 'w': 71 | console.setLevel(logging.WARNING) 72 | formatter = logging.Formatter('%(message)s') 73 | console.setFormatter(formatter) 74 | LOGGER.debug("Warning logging selected") 75 | 76 | # add the handlers to the logger 77 | LOGGER.addHandler(file) 78 | LOGGER.addHandler(console) 79 | 80 | 81 | def upload_command(args): 82 | """Upload sequence from a given path""" 83 | path = args.path 84 | if not os.path.exists(path): 85 | LOGGER.warning("This is an invalid path.") 86 | return 87 | 88 | login_controller = configure_login(args) 89 | upload_manager = OSCUploadManager(login_controller) 90 | discoverers = SequenceDiscovererFactory.discoverers() 91 | finished_list = [] 92 | LOGGER.warning("Searching for sequences...") 93 | for discoverer in discoverers: 94 | LOGGER.debug("Searching for %s", discoverer.name) 95 | sequences = discoverer.discover(path) 96 | if discoverer.ignored_for_upload: 97 | finished_list += sequences 98 | for sequence in sequences: 99 | LOGGER.warning(" Found sequence at path %s that is already uploaded at %s", 100 | sequence.path, 101 | login_controller.osc_api.sequence_link(sequence)) 102 | continue 103 | for sequence in sequences: 104 | if sequence not in finished_list and \ 105 | sequence not in upload_manager.sequences: 106 | LOGGER.warning(" Found sequence at path %s. Sequence type %s.", 107 | sequence.path, 108 | discoverer.name) 109 | upload_manager.add_sequence_to_upload(sequence) 110 | else: 111 | LOGGER.debug("No sequence found for %s", discoverer.name) 112 | 113 | LOGGER.warning("Search completed.") 114 | if not upload_manager.sequences: 115 | if finished_list: 116 | LOGGER.warning(" No sequence to upload.") 117 | else: 118 | LOGGER.warning(" No sequence found.") 119 | else: 120 | LOGGER.warning("\n") 121 | upload_manager.start_upload() 122 | 123 | 124 | def exif_generation_command(args): 125 | """Generate Exif from metadata""" 126 | path = args.path 127 | LOGGER.warning("Trying to generating exif for images at path...") 128 | create_exif(path, args.exif_source) 129 | LOGGER.warning("Finished.") 130 | 131 | 132 | def download_current_user_data(args): 133 | """Download current user data if no user the user will be promted to login""" 134 | path = args.path 135 | LOGGER.debug("Started download current user data") 136 | download_user_images(path) 137 | LOGGER.warning("Done download current user data.") 138 | 139 | 140 | def get_args() -> Namespace: 141 | """Method to create and configure a argument parser""" 142 | parser: ArgumentParser = ArgumentParser(prog='python osc_tools.py', 143 | formatter_class=RawTextHelpFormatter) 144 | 145 | subparsers = parser.add_subparsers(title='These are the available OSC commands', 146 | description='upload Uploads sequences from ' 147 | 'a given path to KartaView\n' 148 | 'generate_exif Generates Exif info for ' 149 | 'each image from a metadata file\n' 150 | 'download Download the data that was ' 151 | 'uploaded by your user', 152 | dest='sub command') 153 | subparsers.required = True 154 | create_parsers(subparsers) 155 | 156 | return parser.parse_args() 157 | 158 | 159 | def _add_environment_argument(parser: ArgumentParser): 160 | parser.add_argument('-e', 161 | '--env', 162 | default='p', 163 | required=False, 164 | help=SUPPRESS, 165 | choices=['p', 't', 's', 'b']) 166 | 167 | 168 | def _add_logging_argument(parser: ArgumentParser): 169 | parser.add_argument('-l', 170 | '--log_level', 171 | required=False, 172 | default='i', 173 | help='Level of logging to console:\n' 174 | ' d level (debug) will log every event to the console\n' 175 | ' i level (info) will log every event more severe than ' 176 | 'debug level to the console\n' 177 | ' w level (warning) will log every event more severe than ' 178 | 'info level to the console', 179 | choices=['d', 'i', 'w']) 180 | 181 | 182 | def create_parsers(subparsers: ArgumentParser): 183 | """Add all available parsers""" 184 | add_upload_parser(subparsers) 185 | add_generate_exif_parser(subparsers) 186 | add_download_parser(subparsers) 187 | 188 | 189 | def add_upload_parser(subparsers): 190 | """Adds upload parser""" 191 | upload_parser = subparsers.add_parser('upload', formatter_class=RawTextHelpFormatter) 192 | upload_parser.set_defaults(func=upload_command) 193 | upload_parser.add_argument('-p', 194 | '--path', 195 | required=True, 196 | help='Full path directory that contains sequence(s) ' 197 | 'folder(s) to upload') 198 | upload_parser.add_argument('-w', 199 | '--workers', 200 | required=False, 201 | type=int, 202 | default=10, 203 | choices=range(1, 21), 204 | metavar="[1-20]", 205 | help='Number of parallel workers used to upload files. ' 206 | 'Default number is 10.') 207 | _add_environment_argument(upload_parser) 208 | _add_logging_argument(upload_parser) 209 | 210 | 211 | def add_generate_exif_parser(subparsers): 212 | """Adds generate exif parser""" 213 | generate_parser = subparsers.add_parser('generate_exif', formatter_class=RawTextHelpFormatter) 214 | generate_parser.set_defaults(func=exif_generation_command) 215 | generate_parser.add_argument('-p', 216 | '--path', 217 | required=True, 218 | help='Folder PATH with metadata file ' 219 | '(OSC metadata, or custom geojson) and images') 220 | generate_parser.add_argument('--exif_source', 221 | required=True, 222 | choices=['metadata', "custom_geojson"]) 223 | _add_logging_argument(generate_parser) 224 | 225 | return subparsers 226 | 227 | 228 | def add_download_parser(subparsers): 229 | """Adds download user data parser""" 230 | download_parser = subparsers.add_parser('download', 231 | formatter_class=RawTextHelpFormatter) 232 | download_parser.set_defaults(func=download_current_user_data) 233 | download_parser.add_argument('-p', 234 | '--path', 235 | required=True, 236 | help='Folder PATH to download your data') 237 | _add_logging_argument(download_parser) 238 | 239 | return subparsers 240 | 241 | 242 | if __name__ == "__main__": 243 | main() 244 | -------------------------------------------------------------------------------- /osc_uploader.py: -------------------------------------------------------------------------------- 1 | """this module will be used to upload files to osc server""" 2 | 3 | import logging 4 | import json 5 | import threading 6 | from concurrent.futures import as_completed, ThreadPoolExecutor 7 | # third party 8 | from typing import List 9 | 10 | from tqdm import tqdm 11 | # local imports 12 | import constants 13 | from common.models import CameraProjection 14 | from osc_discoverer import Sequence 15 | from visual_data_discover import Photo, Video 16 | from login_controller import LoginController 17 | from osc_api_models import OSCPhoto, OSCSequence 18 | 19 | LOGGER = logging.getLogger('osc_uploader') 20 | THREAD_LOCK = threading.Lock() 21 | 22 | 23 | class OSCUploadManager: 24 | """OSCUploadManager is a manager that is responsible with managing the upload of the 25 | sequences received as input""" 26 | def __init__(self, login_controller: LoginController, max_workers: int = 10): 27 | self.progress_bar: tqdm = None 28 | self.sequences: List[Sequence] = [] 29 | self.visual_data_count = 0 30 | self.login_controller: LoginController = login_controller 31 | self.max_workers = max_workers 32 | 33 | def add_sequence_to_upload(self, sequence: Sequence): 34 | """Method to add a sequence to upload queue""" 35 | self.sequences.append(sequence) 36 | 37 | def add_sequences_to_upload(self, sequences: [Sequence]): 38 | """Method to add a list of sequences to the upload queue""" 39 | self.sequences = self.sequences + sequences 40 | 41 | def start_upload(self): 42 | """Method to start upload""" 43 | LOGGER.warning("Starting to upload %d sequences...", len(self.sequences)) 44 | user = self.login_controller.login() 45 | with THREAD_LOCK: 46 | total = 0 47 | for sequence in self.sequences: 48 | total = total + len(sequence.visual_items) 49 | self.progress_bar = tqdm(total=total, dynamic_ncols=True) 50 | 51 | sequence_operation = SequenceUploadOperation(self, 52 | user.access_token, 53 | self.max_workers) 54 | 55 | with ThreadPoolExecutor(max_workers=1) as executors: 56 | futures = [executors.submit(sequence_operation.upload, 57 | sequence) for sequence in self.sequences] 58 | report = [] 59 | for future in as_completed(futures): 60 | success, sequence = future.result() 61 | report.append((success, sequence)) 62 | if success: 63 | LOGGER.warning(" Uploaded sequence from %s, " 64 | "the sequence will be available after " 65 | "processing at %s", sequence.path, 66 | self.login_controller.osc_api.sequence_link(sequence)) 67 | else: 68 | LOGGER.warning(" Failed to upload sequence at %s. Restart the script in " 69 | "order to finish you upload for this sequence.", sequence.path) 70 | LOGGER.warning("Finished uploading") 71 | self.progress_bar.close() 72 | 73 | 74 | class SequenceUploadOperation: 75 | """SequenceUploadOperation is a class that is responsible with uploading a sequence to 76 | OSC servers""" 77 | def __init__(self, manager: OSCUploadManager, user_token: str, workers: int = 5): 78 | self.user_token = user_token 79 | self.workers = workers 80 | self.manager = manager 81 | 82 | def __eq__(self, other): 83 | if isinstance(other, SequenceUploadOperation): 84 | return self.user_token == other.user_token and self.workers == other.workers 85 | return False 86 | 87 | def __hash__(self): 88 | return hash((self.user_token, self.workers)) 89 | 90 | def upload(self, sequence: Sequence) -> (bool, Sequence): 91 | """"This method will upload a sequence of video items to OSC servers. 92 | It returns a success status as bool and the sequence model that was used for the request""" 93 | if constants.UPLOAD_FINISHED in sequence.progress: 94 | return True, sequence 95 | 96 | if not sequence.online_id: 97 | result, online_id = self._create_online_sequence_id(sequence) 98 | if result: 99 | sequence.online_id = online_id 100 | else: 101 | return False, sequence 102 | 103 | visual_item_upload_operation = PhotoUploadOperation(self.manager, 104 | self.user_token, 105 | sequence.online_id) 106 | if sequence.visual_data_type == "video": 107 | visual_item_upload_operation = VideoUploadOperation(self.manager, 108 | self.user_token, 109 | sequence.online_id) 110 | self._visual_items_upload_with_operation(sequence, visual_item_upload_operation) 111 | 112 | if len(sequence.progress) == len(sequence.visual_items): 113 | osc_api = self.manager.login_controller.osc_api 114 | response, _ = osc_api.finish_upload(sequence, self.user_token) 115 | if response: 116 | self.__persist_upload_index(constants.UPLOAD_FINISHED, sequence.path) 117 | return True, sequence 118 | return False, sequence 119 | 120 | def _create_online_sequence_id(self, sequence) -> (bool, Sequence): 121 | osc_sequence = OSCSequence() 122 | osc_sequence.local_id = sequence.path 123 | osc_sequence.metadata_url = sequence.osc_metadata 124 | osc_sequence.latitude = sequence.latitude 125 | osc_sequence.longitude = sequence.longitude 126 | osc_sequence.platform = sequence.platform 127 | osc_sequence.device = sequence.device 128 | osc_api = self.manager.login_controller.osc_api 129 | online_id, error = osc_api.create_sequence(osc_sequence, self.user_token) 130 | sequence.online_id = online_id 131 | if error: 132 | return False, online_id 133 | self.__persist_sequence_id(sequence.online_id, sequence.path) 134 | return True, online_id 135 | 136 | def _visual_items_upload_with_operation(self, sequence, visual_item_upload_operation): 137 | items_to_upload = [] 138 | for visual_item in sequence.visual_items: 139 | if str(visual_item.index) not in sequence.progress: 140 | items_to_upload.append(visual_item) 141 | 142 | with THREAD_LOCK: 143 | self.manager.progress_bar.update(len(sequence.visual_items) - len(items_to_upload)) 144 | 145 | with ThreadPoolExecutor(max_workers=self.workers) as executor: 146 | future_events = [executor.submit(visual_item_upload_operation.upload, 147 | visual_item) for visual_item in items_to_upload] 148 | for completed_event in as_completed(future_events): 149 | uploaded, index = completed_event.result() 150 | with THREAD_LOCK: 151 | if uploaded: 152 | self.__persist_upload_index(index, sequence.path) 153 | sequence.progress.append(index) 154 | self.manager.progress_bar.update(1) 155 | 156 | @classmethod 157 | def __persist_sequence_id(cls, sequence_id, path): 158 | LOGGER.debug("will save sequence_id into file") 159 | sequence_dict = {"id": sequence_id} 160 | with open(path + "/osc_sequence_id.txt", 'w') as output: 161 | json.dump(sequence_dict, output) 162 | LOGGER.debug("Did write data to sequence_id file") 163 | 164 | @classmethod 165 | def __persist_upload_index(cls, sequence_index, path): 166 | LOGGER.debug("will save upload index into file") 167 | with open(path + "/osc_sequence_upload_progress.txt", 'a') as output: 168 | output.write(str(sequence_index) + ";") 169 | 170 | 171 | class VideoUploadOperation: 172 | """VideoUploadOperation is a class responsible with making a video upload.""" 173 | 174 | def __init__(self, manager: OSCUploadManager, user_token: str, sequence_id: str): 175 | self.sequence_id = sequence_id 176 | self.user_token = user_token 177 | self.manager = manager 178 | 179 | def __eq__(self, other): 180 | if isinstance(other, VideoUploadOperation): 181 | return self.user_token == other.user_token and self.sequence_id == other.sequence_id 182 | return False 183 | 184 | def __hash__(self): 185 | return hash((self.user_token, self.sequence_id)) 186 | 187 | def upload(self, video: Video) -> (bool, int): 188 | """This method will upload the video corresponding to the video model 189 | received as parameter. It returns a tuple: success as bool and video index as int""" 190 | user = self.manager.login_controller.user 191 | api = self.manager.login_controller.osc_api 192 | 193 | uploaded = False 194 | for _ in range(0, 10): 195 | uploaded, _ = api.upload_video(user.access_token, 196 | self.sequence_id, 197 | video.path, 198 | video.index) 199 | if uploaded: 200 | break 201 | LOGGER.debug("Will request upload %s", video.path) 202 | 203 | return uploaded, video.index 204 | 205 | 206 | class PhotoUploadOperation: 207 | """PhotoUploadOperation is a class responsible with making a photo upload.""" 208 | 209 | def __init__(self, manager: OSCUploadManager, user_token: str, sequence_id: str): 210 | self.user_token = user_token 211 | self.sequence_id = sequence_id 212 | self.manager = manager 213 | 214 | def __eq__(self, other): 215 | if isinstance(other, PhotoUploadOperation): 216 | return self.user_token == other.user_token and self.sequence_id == other.sequence_id 217 | return False 218 | 219 | def __hash__(self): 220 | return hash((self.user_token, self.sequence_id)) 221 | 222 | def upload(self, photo: Photo) -> (bool, int): 223 | """This method will upload the image corresponding to the photo model 224 | received as parameter. It returns a tuple: success as bool and photo index as int""" 225 | user = self.manager.login_controller.user 226 | api = self.manager.login_controller.osc_api 227 | osc_photo = OSCPhoto() 228 | osc_photo.timestamp = photo.gps_timestamp 229 | osc_photo.image_name = str(photo.index) + ".jpg" 230 | osc_photo.latitude = photo.latitude 231 | osc_photo.longitude = photo.longitude 232 | osc_photo.compass = photo.gps_compass 233 | osc_photo.sequence_index = photo.index 234 | 235 | projection = "PLAIN" 236 | if photo.projection == CameraProjection.EQUIRECTANGULAR: 237 | projection = "SPHERE" 238 | 239 | uploaded = False 240 | for _ in range(0, 10): 241 | uploaded, _ = api.upload_photo(user.access_token, 242 | self.sequence_id, 243 | osc_photo, 244 | photo.path, 245 | photo.fov, 246 | projection) 247 | if uploaded: 248 | break 249 | LOGGER.debug("Will request upload %s", photo.path) 250 | 251 | return uploaded, photo.index 252 | -------------------------------------------------------------------------------- /osc_utils.py: -------------------------------------------------------------------------------- 1 | """utils module that contains useful functions""" 2 | import logging 3 | import os 4 | import gzip 5 | import shutil 6 | from typing import Type, Dict 7 | 8 | import constants 9 | from common.models import GPS 10 | from exif_data_generators.custom_geojson_to_exif import ExifCustomGeoJson 11 | from exif_data_generators.exif_generator_interface import ExifGenerator 12 | from exif_data_generators.metadata_to_exif import ExifMetadataGenerator 13 | from io_storage.storage import Local 14 | from parsers.gpx import GPXParser 15 | from parsers.osc_metadata.parser import metadata_parser 16 | 17 | LOGGER = logging.getLogger('osc_tools.osc_utils') 18 | 19 | 20 | def create_exif(path: str, exif_source: str): 21 | exif_generators: Dict[str, Type[ExifGenerator]] = {"metadata": ExifMetadataGenerator, 22 | "custom_geojson": ExifCustomGeoJson} 23 | if exif_generators[exif_source].has_necessary_data(path): 24 | exif_generators[exif_source].create_exif(path) 25 | return 26 | LOGGER.info("Exif generation is not possible since necessary data was not found") 27 | 28 | 29 | def convert_metadata_to_gpx(base_path, sequence_path_ids): 30 | for sequence_path, sequence_id in sequence_path_ids: 31 | metadata_handle = metadata_parser(os.path.join(sequence_path, "track.txt"), 32 | Local()) 33 | output_handle = GPXParser(os.path.join(base_path, str(sequence_id) + ".gpx"), Local()) 34 | output_handle.add_items(metadata_handle.items_with_class(GPS)) 35 | output_handle.serialize() 36 | 37 | 38 | def unzip_metadata(path: str) -> str: 39 | """Method to unzip the metadata file""" 40 | with gzip.open(os.path.join(path, constants.METADATA_ZIP_NAME), 'rb') as f_in: 41 | unzip_path = os.path.join(path, constants.METADATA_NAME) 42 | 43 | with open(unzip_path, 'wb') as f_out: 44 | shutil.copyfileobj(f_in, f_out) 45 | return unzip_path 46 | -------------------------------------------------------------------------------- /osm_access.py: -------------------------------------------------------------------------------- 1 | """This script is used to get osm authentication token and secret token.""" 2 | from typing import Tuple, Optional 3 | 4 | from requests_oauthlib import OAuth2Session 5 | from oauthlib.oauth2 import OAuth2Token 6 | 7 | OSM_AUTH_URL = 'https://www.openstreetmap.org/oauth2/authorize' 8 | OSM_TOKEN_URL = 'https://www.openstreetmap.org/oauth2/token' 9 | 10 | # Credentials dedicated for GitHub community 11 | CLIENT_ID = 'xCDlXoN-gVeXsXMHu8N5VArN4iWBDwuMgZAlf5PlC7c' 12 | CLIENT_SECRET = "UICSTaxQkQsSl-osmcbqd5CXJIak5fvw9BF_F152BeE" 13 | 14 | 15 | def __osm_auth_service() -> OAuth2Session: 16 | """Factory method that builds osm auth service""" 17 | oauth = OAuth2Session(client_id=CLIENT_ID, 18 | redirect_uri="urn:ietf:wg:oauth:2.0:oob", 19 | scope=["openid", "read_prefs"]) 20 | return oauth 21 | 22 | 23 | def osm_auth(request_user_action, error_handle) -> Tuple[Optional[str], Optional[str]]: 24 | service = __osm_auth_service() 25 | authorization_url, _ = service.authorization_url(OSM_AUTH_URL) 26 | response = request_user_action(authorization_url) 27 | # pylint: disable=W0703 28 | try: 29 | access_token_response: OAuth2Token = service.fetch_token(OSM_TOKEN_URL, 30 | code=response, 31 | client_id=CLIENT_ID, 32 | client_secret=CLIENT_SECRET) 33 | 34 | return access_token_response.get("access_token"), CLIENT_SECRET 35 | except Exception as ex: 36 | error_handle(ex) 37 | return None, None 38 | -------------------------------------------------------------------------------- /parsers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartaview/upload-scripts/28650a631e398ebe8991c1c965248d41ae68dd87/parsers/__init__.py -------------------------------------------------------------------------------- /parsers/base.py: -------------------------------------------------------------------------------- 1 | """This file contains parsers for osc metadata file to other file formats.""" 2 | import abc 3 | from typing import Optional, List, Any, Type 4 | 5 | from common.models import SensorItem 6 | from io_storage.storage import Storage 7 | 8 | 9 | class BaseParser(metaclass=abc.ABCMeta): 10 | """this class is a base class for every parser""" 11 | def __init__(self, path: str, storage: Storage): 12 | self.file_path: str = path 13 | self._sensors: Optional[List[SensorItem]] = None 14 | self._body_pointer: Optional[int] = None 15 | self._data_pointer: Optional[int] = None 16 | self._storage = storage 17 | 18 | @abc.abstractmethod 19 | def next_item_with_class(self, item_class: Type[SensorItem]) -> Optional[SensorItem]: 20 | """this method will return a the next SensorItem found in the current file, 21 | of instance item_class""" 22 | 23 | @abc.abstractmethod 24 | def items_with_class(self, item_class: Type[SensorItem]) -> List[SensorItem]: 25 | """this method will return all SensorItems found in the current file, 26 | of instance item_class""" 27 | 28 | @abc.abstractmethod 29 | def next_item(self) -> Optional[SensorItem]: 30 | """this method will return a the next SensorItem found in the current file""" 31 | 32 | @abc.abstractmethod 33 | def items(self) -> List[SensorItem]: 34 | """this method will return all SensorItems found in the current file""" 35 | 36 | @abc.abstractmethod 37 | def format_version(self) -> Optional[str]: 38 | """this method will return the format version""" 39 | 40 | @abc.abstractmethod 41 | def serialize(self): 42 | """this method will write all the added items to file""" 43 | 44 | @classmethod 45 | @abc.abstractmethod 46 | def compatible_sensors(cls) -> List[Any]: 47 | """this method will return all SensorItem classes that are compatible 48 | with the current parser""" 49 | 50 | def add_items(self, items: List[SensorItem]): 51 | """this method will add items to be serialized""" 52 | self._sensors = items 53 | self._sensors.sort(key=lambda sensor_item: sensor_item.timestamp) 54 | 55 | def start_new_reading(self): 56 | """This method sets the reading file pointer to the body section of the metadata""" 57 | self._data_pointer = self._body_pointer 58 | -------------------------------------------------------------------------------- /parsers/custom_data_parsers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartaview/upload-scripts/28650a631e398ebe8991c1c965248d41ae68dd87/parsers/custom_data_parsers/__init__.py -------------------------------------------------------------------------------- /parsers/custom_data_parsers/custom_geojson.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains custom geojson parsers. 3 | """ 4 | from typing import Type, List, Optional 5 | import datetime 6 | 7 | from geojson import load 8 | from io_storage.storage import Storage 9 | from parsers.geojson import GeoJsonParser 10 | from parsers.custom_data_parsers.custom_models import PhotoGeoJson 11 | from common.models import GPS, Compass, SensorItem 12 | 13 | 14 | class FeaturePhotoGeoJsonParser(GeoJsonParser): 15 | 16 | def __init__(self, file_path: str, storage: Storage): 17 | super().__init__(file_path, storage) 18 | self._sensors: List[PhotoGeoJson] = [] 19 | self._data_pointer: int = 0 20 | self._version = "unknown" 21 | with self._storage.open(self.file_path, 'r') as geo_json_file: 22 | geo_json = load(geo_json_file) 23 | for feature in geo_json["features"]: 24 | string_time = feature.properties['Timestamp'] 25 | utc_time = datetime.datetime.strptime(string_time, "%Y-%m-%dT%H:%M:%SZ") 26 | utc_time = utc_time.replace(tzinfo=datetime.timezone.utc) 27 | gps = GPS.gps(utc_time.timestamp(), 28 | float(feature.properties['Lat']), 29 | float(feature.properties['Lon'])) 30 | photo = PhotoGeoJson(gps, 31 | int(feature.properties['order']), 32 | feature.properties['path'].replace("\\", "/")) 33 | photo.compass = Compass() 34 | photo.compass.compass = feature.properties['direction'] 35 | self._sensors.append(photo) 36 | crs_string = geo_json.get('crs', {}).get('properties', {}).get("name", "") 37 | crs_values = crs_string.split(":") 38 | if len(crs_values) == 7: 39 | self._version = crs_values[5] 40 | 41 | self._sensors.sort(key=lambda x: x.frame_index) 42 | 43 | def items(self) -> List[SensorItem]: 44 | return self._sensors 45 | 46 | def items_with_class(self, item_class: Type[SensorItem]) -> List[SensorItem]: 47 | if item_class not in self.compatible_sensors(): 48 | return [] 49 | if item_class is PhotoGeoJson: 50 | return self._sensors 51 | if item_class is GPS: 52 | return [photo.gps for photo in self._sensors] 53 | if item_class is Compass: 54 | return [photo.compass for photo in self._sensors] 55 | return [] 56 | 57 | def next_item(self) -> Optional[SensorItem]: 58 | if len(self._sensors) < self._data_pointer + 1: 59 | self._data_pointer += 1 60 | return self._sensors[self._data_pointer] 61 | 62 | return None 63 | 64 | def next_item_with_class(self, item_class: Type[SensorItem]) -> Optional[SensorItem]: 65 | items = self.items_with_class(item_class) 66 | if len(items) < self._data_pointer + 1: 67 | self._data_pointer += 1 68 | return items[self._data_pointer] 69 | return None 70 | 71 | def format_version(self) -> Optional[str]: 72 | return self._version 73 | 74 | def start_new_reading(self): 75 | self._data_pointer = 0 76 | 77 | @classmethod 78 | def compatible_sensors(cls): 79 | return [PhotoGeoJson, GPS, Compass] 80 | -------------------------------------------------------------------------------- /parsers/custom_data_parsers/custom_mapillary.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains custom mapillary images parsers. 3 | """ 4 | import json 5 | from typing import Dict, Optional 6 | 7 | from exifread.classes import IfdTag 8 | 9 | from common.models import OSCDevice 10 | from parsers.exif.exif import ExifParser 11 | from parsers.exif.utils import ExifTags, datetime_from_string 12 | 13 | 14 | class MapillaryExif(ExifParser): 15 | 16 | @classmethod 17 | def _gps_timestamp(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 18 | if ExifTags.DESCRIPTION.value in gps_data: 19 | # description exists 20 | description = str(gps_data[ExifTags.DESCRIPTION.value]) 21 | try: 22 | description = json.loads(description) 23 | except json.JSONDecodeError as _: 24 | return None 25 | mapillary_timestamp = description.get("MAPCaptureTime", None) # 2021_07_24_14_24_04_000 26 | if mapillary_timestamp is not None: 27 | # mapillary timestamp exists 28 | return datetime_from_string(mapillary_timestamp, "%Y_%m_%d_%H_%M_%S_%f") 29 | return None 30 | 31 | @classmethod 32 | def _gps_latitude(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 33 | if ExifTags.DESCRIPTION.value in gps_data: 34 | description = str(gps_data[ExifTags.DESCRIPTION.value]) 35 | try: 36 | description = json.loads(description) 37 | except json.JSONDecodeError as _: 38 | return None 39 | # mapillary latitude exists return it or None 40 | latitude = description.get("MAPLatitude", None) 41 | return latitude 42 | return None 43 | 44 | @classmethod 45 | def _gps_longitude(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 46 | if ExifTags.DESCRIPTION.value in gps_data: 47 | description = str(gps_data[ExifTags.DESCRIPTION.value]) 48 | try: 49 | description = json.loads(description) 50 | except json.JSONDecodeError as _: 51 | return None 52 | # mapillary longitude exists return it or None 53 | latitude = description.get("MAPLongitude", None) 54 | return latitude 55 | return None 56 | 57 | @classmethod 58 | def _gps_compass(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 59 | if ExifTags.DESCRIPTION.value in gps_data: 60 | description = str(gps_data[ExifTags.DESCRIPTION.value]) 61 | try: 62 | description = json.loads(description) 63 | except json.JSONDecodeError as _: 64 | return None 65 | # mapillary compass exists return it or None 66 | compass_dict = description.get("MAPCompassHeading", {}) 67 | compass = compass_dict.get("TrueHeading", None) 68 | return compass 69 | return None 70 | 71 | @classmethod 72 | def _gps_altitude(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 73 | if ExifTags.DESCRIPTION.value in gps_data: 74 | description = str(gps_data[ExifTags.DESCRIPTION.value]) 75 | try: 76 | description = json.loads(description) 77 | except json.JSONDecodeError as _: 78 | return None 79 | # mapillary altitude exists return it or None 80 | altitude = description.get("MAPAltitude", None) 81 | return altitude 82 | return None 83 | 84 | @classmethod 85 | def _gps_speed(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 86 | if ExifTags.DESCRIPTION.value in gps_data: 87 | description = str(gps_data[ExifTags.DESCRIPTION.value]) 88 | try: 89 | description = json.loads(description) 90 | except json.JSONDecodeError as _: 91 | return None 92 | # mapillary speed exists return it or None 93 | speed = description.get("MAPGPSSpeed", None) 94 | return speed 95 | return None 96 | 97 | @classmethod 98 | def _device_model(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 99 | if ExifTags.DESCRIPTION.value in gps_data: 100 | description = str(gps_data[ExifTags.DESCRIPTION.value]) 101 | try: 102 | description = json.loads(description) 103 | except json.JSONDecodeError as _: 104 | return None 105 | # mapillary device model exists return it or None 106 | speed = description.get("MAPDeviceModel", None) 107 | return speed 108 | return None 109 | 110 | @classmethod 111 | def _device_make(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 112 | if ExifTags.DESCRIPTION.value in gps_data: 113 | description = str(gps_data[ExifTags.DESCRIPTION.value]) 114 | try: 115 | description = json.loads(description) 116 | except json.JSONDecodeError as _: 117 | return None 118 | # mapillary device make exists return it or None 119 | make = description.get("MAPDeviceMake", None) 120 | return make 121 | return None 122 | 123 | def _device_item(self, tags_data=None) -> OSCDevice: 124 | device = super()._device_item(tags_data) 125 | if device.platform_name is None or "iPhone" in device.platform_name: 126 | if "iPhone" in device.device_raw_name: 127 | device.platform_name = "Apple" 128 | else: 129 | device.platform_name = "Unknown" 130 | 131 | return device 132 | -------------------------------------------------------------------------------- /parsers/custom_data_parsers/custom_models.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a module for declaring any new custom data classes. 3 | """ 4 | from common.models import PhotoMetadata, GPS 5 | 6 | 7 | class PhotoGeoJson(PhotoMetadata): 8 | def __init__(self, gps: GPS, index, relative_path: str): 9 | super().__init__() 10 | self.gps = gps 11 | self.frame_index = index 12 | self.relative_image_path = relative_path 13 | -------------------------------------------------------------------------------- /parsers/exif/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartaview/upload-scripts/28650a631e398ebe8991c1c965248d41ae68dd87/parsers/exif/__init__.py -------------------------------------------------------------------------------- /parsers/exif/exif.py: -------------------------------------------------------------------------------- 1 | """Module responsible to parse Exif information from an image""" 2 | 3 | import os 4 | from typing import Optional, Dict, List, Type 5 | # third party 6 | import exifread 7 | import imagesize 8 | 9 | from exifread.classes import Ratio, IfdTag 10 | 11 | from common.models import ( 12 | PhotoMetadata, 13 | GPS, 14 | Compass, 15 | SensorItem, 16 | OSCDevice, 17 | ExifParameters, 18 | RecordingType) 19 | 20 | from parsers.base import BaseParser 21 | from parsers.exif.utils import ( 22 | ExifTags, 23 | CardinalDirection, 24 | SpeedUnit, 25 | SeaLevel, 26 | datetime_from_string, 27 | create_required_gps_tags, 28 | add_optional_gps_tags, 29 | add_gps_tags, 30 | dms_to_dd 31 | ) 32 | 33 | from io_storage.storage import Storage 34 | 35 | 36 | class ExifParser(BaseParser): 37 | """This class is a BaseParser that can parse images having exif""" 38 | 39 | def __init__(self, file_path, storage: Storage): 40 | super().__init__(file_path, storage) 41 | self._data_pointer = 0 42 | self._body_pointer = 0 43 | self.tags: Dict[str, IfdTag] = self._all_tags() 44 | 45 | def next_item_with_class(self, item_class: Type[SensorItem]) -> Optional[SensorItem]: 46 | if item_class == PhotoMetadata: 47 | return self._photo_item(self.tags) 48 | if item_class == GPS: 49 | return self._gps_item(self.tags) 50 | if item_class == Compass: 51 | return self._compass_item(self.tags) 52 | if item_class == OSCDevice: 53 | return self._device_item(self.tags) 54 | if item_class == ExifParameters: 55 | return self._exif_item(self.tags) 56 | return None 57 | 58 | def items_with_class(self, item_class: Type[SensorItem]) -> List[SensorItem]: 59 | item = self.next_item_with_class(item_class) 60 | if item: 61 | return [item] 62 | return [] 63 | 64 | def next_item(self) -> Optional[SensorItem]: 65 | if self._data_pointer == 0: 66 | self._data_pointer = 1 67 | return self._photo_item(self.tags) 68 | if self._data_pointer == 1: 69 | self._data_pointer = 2 70 | return self._gps_item(self.tags) 71 | if self._data_pointer == 2: 72 | self._data_pointer = 3 73 | return self._compass_item(self.tags) 74 | if self._data_pointer == 3: 75 | self._data_pointer = 4 76 | return self._device_item(self.tags) 77 | return None 78 | 79 | def items(self) -> List[SensorItem]: 80 | return [item for item in [self._photo_item(self.tags), 81 | self._gps_item(self.tags), 82 | self._compass_item(self.tags), 83 | self._device_item(self.tags), 84 | self._exif_item(self.tags)] if item] 85 | 86 | def format_version(self): 87 | return self._exif_version(self.tags) 88 | 89 | def serialize(self): 90 | tags = None 91 | for item in self._sensors: 92 | gps = None 93 | compass = None 94 | if isinstance(item, PhotoMetadata): 95 | gps, compass = (item.gps, item.compass) 96 | if isinstance(item, GPS): 97 | gps = item 98 | if gps: 99 | tags = create_required_gps_tags(gps.timestamp, 100 | gps.latitude, 101 | gps.longitude) 102 | add_optional_gps_tags(tags, 103 | gps.speed, 104 | gps.altitude, 105 | None) 106 | if isinstance(item, Compass): 107 | add_optional_gps_tags(tags, 108 | None, 109 | None, 110 | compass.compass) 111 | if tags is not None: 112 | add_gps_tags(self.file_path, tags) 113 | 114 | @classmethod 115 | def compatible_sensors(cls): 116 | return [PhotoMetadata, GPS, Compass, OSCDevice] 117 | 118 | # private methods 119 | 120 | def _all_tags(self) -> Dict[str, IfdTag]: 121 | """Method to return Exif tags""" 122 | with self._storage.open(self.file_path, "rb") as file: 123 | tags = exifread.process_file(file, details=False) 124 | return tags 125 | 126 | def _gps_item(self, data=None) -> Optional[GPS]: 127 | if data is not None: 128 | tags_data = data 129 | else: 130 | tags_data = self._all_tags() 131 | gps = GPS() 132 | # required gps timestamp or exif timestamp 133 | gps.timestamp = self._gps_timestamp(tags_data) 134 | exif_timestamp = self._timestamp(tags_data) 135 | if (not gps.timestamp or 136 | (exif_timestamp is not None and exif_timestamp > 31556952 137 | and abs(gps.timestamp - exif_timestamp) > 31556952)): 138 | # if there is no gps timestamp or gps timestamp differs with more than 1 year 139 | # compared to exif_timestamp we choose exif_timestamp 140 | gps.timestamp = exif_timestamp 141 | 142 | # required latitude and longitude 143 | gps.latitude = self._gps_latitude(tags_data) 144 | gps.longitude = self._gps_longitude(tags_data) 145 | if not gps.latitude or \ 146 | not gps.longitude or \ 147 | not gps.timestamp: 148 | return None 149 | 150 | # optional data 151 | gps.speed = self._gps_speed(tags_data) 152 | gps.altitude = self._gps_altitude(tags_data) 153 | return gps 154 | 155 | def _compass_item(self, data=None) -> Optional[Compass]: 156 | if data is not None: 157 | tags_data = data 158 | else: 159 | tags_data = self._all_tags() 160 | compass = Compass() 161 | compass.compass = self._gps_compass(tags_data) 162 | 163 | if compass.compass: 164 | return compass 165 | return None 166 | 167 | def _photo_item(self, tags_data=None) -> Optional[PhotoMetadata]: 168 | if tags_data is None: 169 | tags_data = self._all_tags() 170 | 171 | gps = self._gps_item(tags_data) 172 | if gps is None: 173 | return None 174 | 175 | compass = self._compass_item(tags_data) or Compass() 176 | photo = PhotoMetadata() 177 | photo.gps = gps 178 | photo.compass = compass 179 | photo.timestamp = self._timestamp(tags_data) 180 | file_name = os.path.basename(self.file_path) 181 | if file_name.isdigit(): 182 | photo.frame_index = int(file_name) 183 | if photo.timestamp is None: 184 | photo.timestamp = photo.gps.timestamp 185 | return photo 186 | return photo 187 | 188 | def _device_item(self, tags_data=None) -> OSCDevice: 189 | if tags_data is None: 190 | tags_data = self._all_tags() 191 | 192 | device = OSCDevice() 193 | valid_timestamp = self._timestamp(tags_data) 194 | if valid_timestamp is None: 195 | valid_timestamp = self._gps_timestamp(tags_data) 196 | device.timestamp = valid_timestamp 197 | device.device_raw_name = self._device_model(tags_data) 198 | device.platform_name = self._maker_name(tags_data) 199 | device.recording_type = RecordingType.PHOTO 200 | 201 | return device 202 | 203 | def _exif_item(self, tag_data=None) -> Optional[ExifParameters]: 204 | if tag_data is None: 205 | tag_data = self._all_tags() 206 | if ExifTags.WIDTH.value in tag_data and ExifTags.HEIGHT.value in tag_data: 207 | exif_item = ExifParameters() 208 | exif_item.width = int(tag_data[ExifTags.WIDTH.value].values[0]) 209 | exif_item.height = int(tag_data[ExifTags.HEIGHT.value].values[0]) 210 | 211 | return exif_item 212 | width, height = imagesize.get(self.file_path) 213 | if width > 0 and height > 0: 214 | exif_item = ExifParameters() 215 | exif_item.width = width 216 | exif_item.height = height 217 | 218 | return exif_item 219 | return None 220 | 221 | @classmethod 222 | def _gps_compass(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 223 | """Exif compass from gps_data represented by gps tags found in image exif. 224 | reference relative to true north""" 225 | if ExifTags.GPS_DIRECTION.value in gps_data: 226 | # compass exists 227 | compass_ratio = gps_data[ExifTags.GPS_DIRECTION.value].values[0] 228 | if ExifTags.GPS_DIRECTION_REF.value in gps_data and \ 229 | gps_data[ExifTags.GPS_DIRECTION_REF.value] == CardinalDirection.MAGNETIC_NORTH: 230 | # if we find magnetic north then we don't consider a valid compass 231 | return None 232 | return compass_ratio.num / compass_ratio.den 233 | # no compass found 234 | return None 235 | 236 | @classmethod 237 | def _gps_timestamp(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 238 | """Exif gps time from gps_data represented by gps tags found in image exif. 239 | In exif there are values giving the hour, minute, and second. 240 | This is UTC time""" 241 | if ExifTags.GPS_TIMESTAMP.value in gps_data: 242 | # timestamp exists 243 | _timestamp = gps_data[ExifTags.GPS_TIMESTAMP.value] 244 | hours: Ratio = _timestamp.values[0] 245 | minutes: Ratio = _timestamp.values[1] 246 | seconds: Ratio = _timestamp.values[2] 247 | 248 | if hours.den == 0.0 or minutes.den == 0.0 or seconds.den == 0.0: 249 | return None 250 | 251 | day_timestamp = \ 252 | hours.num / hours.den * 3600 + \ 253 | minutes.num / minutes.den * 60 + \ 254 | seconds.num / seconds.den 255 | 256 | if ExifTags.GPS_DATE_STAMP.value in gps_data: 257 | # this tag is the one present in the exif documentation 258 | # but from experience ExifTags.GPS_DATE is replacing this tag 259 | gps_date = gps_data[ExifTags.GPS_DATE_STAMP.value].values 260 | gps_date_time = datetime_from_string(gps_date, "%Y:%m:%d") 261 | if gps_date_time is None: 262 | return None 263 | date_timestamp = gps_date_time.timestamp() 264 | 265 | return day_timestamp + date_timestamp 266 | 267 | if ExifTags.GPS_DATE.value in gps_data: 268 | # this tag is a replacement for ExifTags.GPS_DATE_STAMP 269 | gps_date = gps_data[ExifTags.GPS_DATE.value].values 270 | gps_date_time = datetime_from_string(gps_date, "%Y:%m:%d") 271 | if gps_date_time is None: 272 | return None 273 | date_timestamp = gps_date_time.timestamp() 274 | 275 | return day_timestamp + date_timestamp 276 | 277 | # no date information only hour minutes second of day -> no valid gps timestamp 278 | # no gps timestamp found 279 | return None 280 | 281 | @classmethod 282 | def _timestamp(cls, tags: Dict[str, IfdTag]) -> Optional[float]: 283 | """Original timestamp determined by the digital still camera. This is timezone corrected.""" 284 | if ExifTags.DATE_TIME_ORIGINAL.value in tags: 285 | date_taken = tags[ExifTags.DATE_TIME_ORIGINAL.value].values 286 | date_time_value = datetime_from_string(date_taken, "%Y:%m:%d %H:%M:%S") 287 | if date_time_value is None: 288 | return None 289 | _timestamp = date_time_value.timestamp() 290 | 291 | return _timestamp 292 | if ExifTags.DATE_TIME_DIGITIZED.value in tags: 293 | date_taken = tags[ExifTags.DATE_TIME_DIGITIZED.value].values 294 | date_time_value = datetime_from_string(date_taken, "%Y:%m:%d %H:%M:%S") 295 | if date_time_value is None: 296 | return None 297 | _timestamp = date_time_value.timestamp() 298 | 299 | return _timestamp 300 | # no timestamp information found 301 | return None 302 | 303 | @classmethod 304 | def _gps_altitude(cls, gps_tags: Dict[str, IfdTag]) -> Optional[float]: 305 | """GPS altitude form exif """ 306 | if ExifTags.GPS_ALTITUDE.value in gps_tags: 307 | # altitude exists 308 | altitude_ratio = gps_tags[ExifTags.GPS_ALTITUDE.value].values[0] 309 | altitude = altitude_ratio.num / altitude_ratio.den 310 | if ExifTags.GPS_ALTITUDE_REF.value in gps_tags and \ 311 | gps_tags[ExifTags.GPS_ALTITUDE_REF.value] == SeaLevel.BELOW.value: 312 | altitude = -1 * altitude 313 | return altitude 314 | return None 315 | 316 | @classmethod 317 | def _gps_speed(cls, gps_tags: Dict[str, IfdTag]) -> Optional[float]: 318 | """Returns GPS speed from exif in km per hour or None if no gps speed tag found""" 319 | if ExifTags.GPS_SPEED.value in gps_tags: 320 | # gps speed exist 321 | speed_ratio = gps_tags[ExifTags.GPS_SPEED.value].values[0] 322 | speed = speed_ratio.num / speed_ratio.den 323 | if ExifTags.GPS_SPEED_REF.value in gps_tags: 324 | if gps_tags[ExifTags.GPS_SPEED_REF.value] == SpeedUnit.MPH.value: 325 | speed = SpeedUnit.convert_mph_to_kmh(speed) 326 | if gps_tags[ExifTags.GPS_SPEED_REF.value] == SpeedUnit.KNOTS.value: 327 | speed = SpeedUnit.convert_knots_to_kmh(speed) 328 | return speed 329 | # no gps speed tag found 330 | return None 331 | 332 | @classmethod 333 | def _maker_name(cls, tags: Dict[str, IfdTag]) -> Optional[str]: 334 | """this method returns a platform name""" 335 | device_make = None 336 | if ExifTags.DEVICE_MAKE.value in tags: 337 | device_make = str(tags[ExifTags.DEVICE_MAKE.value]) 338 | 339 | return device_make 340 | 341 | @classmethod 342 | def _device_model(cls, tags: Dict[str, IfdTag]) -> Optional[str]: 343 | """this method returns a device name""" 344 | model = None 345 | if ExifTags.DEVICE_MODEL.value in tags: 346 | model = str(tags[ExifTags.DEVICE_MODEL.value]) 347 | 348 | return model 349 | 350 | @classmethod 351 | def _exif_version(cls, tags: Dict[str, IfdTag]) -> Optional[str]: 352 | """this method returns exif version""" 353 | if ExifTags.FORMAT_VERSION.value in tags: 354 | return str(tags[ExifTags.FORMAT_VERSION.value]) 355 | return None 356 | 357 | @classmethod 358 | def _gps_latitude(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 359 | """Exif latitude from gps_data represented by gps tags found in image exif""" 360 | if ExifTags.GPS_LATITUDE.value in gps_data: 361 | # latitude exists 362 | dms_values = gps_data[ExifTags.GPS_LATITUDE.value] 363 | _latitude = dms_to_dd(dms_values) 364 | if _latitude is None: 365 | return None 366 | 367 | if ExifTags.GPS_LATITUDE_REF.value in gps_data and \ 368 | (str(gps_data[ExifTags.GPS_LATITUDE_REF.value]) == str(CardinalDirection.S.value)): 369 | # cardinal direction is S so the latitude should be negative 370 | _latitude = -1 * _latitude 371 | 372 | if abs(_latitude) > 90: 373 | return None 374 | 375 | return _latitude 376 | # no latitude info found 377 | return None 378 | 379 | @classmethod 380 | def _gps_longitude(cls, gps_data: Dict[str, IfdTag]) -> Optional[float]: 381 | """Exif longitude from gps_data represented by gps tags found in image exif""" 382 | if ExifTags.GPS_LONGITUDE.value in gps_data: 383 | # longitude exists 384 | dms_values = gps_data[ExifTags.GPS_LONGITUDE.value] 385 | _longitude = dms_to_dd(dms_values) 386 | if _longitude is None: 387 | return None 388 | 389 | if ExifTags.GPS_LONGITUDE_REF.value in gps_data and \ 390 | str(gps_data[ExifTags.GPS_LONGITUDE_REF.value]) == str(CardinalDirection.W.value): 391 | # cardinal direction is W so the longitude should be negative 392 | _longitude = -1 * _longitude 393 | 394 | if abs(_longitude) > 180: 395 | return None 396 | 397 | return _longitude 398 | # no longitude info found 399 | return None 400 | -------------------------------------------------------------------------------- /parsers/exif/utils.py: -------------------------------------------------------------------------------- 1 | """Module containing Exif object helpers""" 2 | 3 | import math 4 | import datetime 5 | from enum import Enum 6 | from typing import Tuple, Any, Optional, List, Dict 7 | import piexif 8 | 9 | MPH_TO_KMH_FACTOR = 1.60934 10 | """miles per hour to kilometers per hour conversion factor""" 11 | KNOTS_TO_KMH_FACTOR = 1.852 12 | """knots to kilometers per hour conversion factor""" 13 | 14 | 15 | class ExifTags(Enum): 16 | """This is an enumeration of exif tags. More info here 17 | http://owl.phy.queensu.ca/~phil/exiftool/TagNames/GPS.html """ 18 | DATE_TIME_ORIGINAL = "EXIF DateTimeOriginal" 19 | DATE_TIME_DIGITIZED = "EXIF DateTimeDigitized" 20 | # latitude 21 | GPS_LATITUDE = "GPS GPSLatitude" 22 | GPS_LATITUDE_REF = "GPS GPSLatitudeRef" 23 | # longitude 24 | GPS_LONGITUDE = "GPS GPSLongitude" 25 | GPS_LONGITUDE_REF = "GPS GPSLongitudeRef" 26 | # altitude 27 | GPS_ALTITUDE_REF = "GPS GPSAltitudeRef" 28 | GPS_ALTITUDE = "GPS GPSAltitude" 29 | # timestamp 30 | GPS_TIMESTAMP = "GPS GPSTimeStamp" 31 | GPS_DATE_STAMP = "GPS GPSDateStamp" 32 | GPS_DATE = "GPS GPSDate" 33 | # speed 34 | GPS_SPEED_REF = "GPS GPSSpeedRef" 35 | GPS_SPEED = "GPS GPSSpeed" 36 | # direction 37 | GPS_DIRECTION_REF = "GPS GPSImgDirectionRef" 38 | GPS_DIRECTION = "GPS GPSImgDirection" 39 | # device name 40 | DEVICE_MAKE = "Image Make" 41 | DEVICE_MODEL = "Image Model" 42 | FORMAT_VERSION = "EXIF ExifVersion" 43 | WIDTH = "Image ImageWidth" 44 | HEIGHT = "Image ImageLength" 45 | DESCRIPTION = "Image ImageDescription" 46 | 47 | 48 | class CardinalDirection(Enum): 49 | """Exif Enum with all cardinal directions""" 50 | N = "N" 51 | S = "S" 52 | E = "E" 53 | W = "W" 54 | TRUE_NORTH = "T" 55 | MAGNETIC_NORTH = "M" 56 | 57 | 58 | class SeaLevel(Enum): 59 | """Exif Enum 60 | If the reference is sea level and the 61 | altitude is above sea level, 0 is given. 62 | If the altitude is below sea level, a value of 1 is given and 63 | the altitude is indicated as an absolute value in the GPSAltitude tag. 64 | The reference unit is meters. Note that this tag is BYTE type, 65 | unlike other reference tags.""" 66 | ABOVE = 0 67 | BELOW = 1 68 | 69 | 70 | class SpeedUnit(Enum): 71 | """Exif speed unit enum""" 72 | KMH = "K" 73 | MPH = "M" 74 | KNOTS = "N" 75 | 76 | @classmethod 77 | def convert_mph_to_kmh(cls, mph) -> float: 78 | """This method converts from miles per hour to kilometers per hour""" 79 | return mph * MPH_TO_KMH_FACTOR 80 | 81 | @classmethod 82 | def convert_knots_to_kmh(cls, knots) -> float: 83 | """This method converts from knots to kilometers per hour""" 84 | return knots * KNOTS_TO_KMH_FACTOR 85 | 86 | 87 | def dms_to_dd(dms_value) -> Optional[float]: 88 | """DMS is Degrees Minutes Seconds, DD is Decimal Degrees. 89 | A typical format would be dd/1,mm/1,ss/1. 90 | When degrees and minutes are used and, for example, 91 | fractions of minutes are given up to two decimal places, 92 | the format would be dd/1,mmmm/100,0/1 """ 93 | # degrees 94 | degrees_nominator = dms_value.values[0].num 95 | degrees_denominator = dms_value.values[0].den 96 | if float(degrees_denominator) == 0.0: 97 | return None 98 | degrees = float(degrees_nominator) / float(degrees_denominator) 99 | # minutes 100 | minutes_nominator = dms_value.values[1].num 101 | minutes_denominator = dms_value.values[1].den 102 | if float(minutes_denominator) == 0.0: 103 | return None 104 | minutes = float(minutes_nominator) / float(minutes_denominator) 105 | # seconds 106 | seconds_nominator = dms_value.values[2].num 107 | seconds_denominator = dms_value.values[2].den 108 | if float(seconds_denominator) == 0.0: 109 | return None 110 | seconds = float(seconds_nominator) / float(seconds_denominator) 111 | # decimal degrees 112 | return degrees + (minutes / 60.0) + (seconds / 3600.0) 113 | 114 | 115 | def dd_to_dms(decimal_degree) -> List[Tuple[float, int]]: 116 | decimal_degree_abs = abs(decimal_degree) 117 | 118 | degrees = math.floor(decimal_degree_abs) 119 | minute_float = (decimal_degree_abs - degrees) * 60 120 | minute = math.floor(minute_float) 121 | seconds = round((minute_float - minute) * 60 * 100) 122 | 123 | return [(degrees, 1), (minute, 1), (seconds, 100)] 124 | 125 | 126 | def datetime_from_string(date_taken, string_format): 127 | try: 128 | tmp = str(date_taken).replace("-", ":") 129 | time_value = datetime.datetime.strptime(tmp, string_format) 130 | if time_value.tzinfo is None: 131 | time_value = time_value.replace(tzinfo=datetime.timezone.utc) 132 | return time_value 133 | except ValueError as error: 134 | # this is are workarounds for wrong timestamp format e.g. 135 | 136 | # date_taken = "????:??:?? ??:??:??" 137 | if isinstance(date_taken, str): 138 | return None 139 | 140 | # date_taken=b'\\xf2\\xf0\\xf1\\xf9:\\xf0\\xf4:\\xf0\\xf5 \\xf1\\xf1:\\xf2\\xf9:\\xf5\\xf4' 141 | try: 142 | date_taken = str(date_taken.decode("utf-8", "backslashreplace")).replace("\\xf", "") 143 | time_value = datetime.datetime.strptime(date_taken, string_format) 144 | if time_value.tzinfo is None: 145 | time_value = time_value.replace(tzinfo=datetime.timezone.utc) 146 | return time_value 147 | except ValueError: 148 | raise ValueError from error 149 | 150 | 151 | def add_gps_tags(path: str, gps_tags: Dict[str, Any]): 152 | """This method will add gps tags to the photo found at path""" 153 | exif_dict = piexif.load(path) 154 | for tag, tag_value in gps_tags.items(): 155 | exif_dict["GPS"][tag] = tag_value 156 | 157 | exif_bytes = piexif.dump(exif_dict) 158 | piexif.insert(exif_bytes, path) 159 | 160 | 161 | def create_required_gps_tags(timestamp_gps: Optional[float], 162 | latitude: float, 163 | longitude: float) -> Dict[str, Any]: 164 | """This method will create gps required tags """ 165 | exif_gps: Dict[str, Any] = {} 166 | if timestamp_gps is not None: 167 | day = int(timestamp_gps / 86400) * 86400 168 | hour = int((timestamp_gps - day) / 3600) 169 | minutes = int((timestamp_gps - day - hour * 3600) / 60) 170 | seconds = int(timestamp_gps - day - hour * 3600 - minutes * 60) 171 | 172 | day_timestamp_str = datetime.date.fromtimestamp(day).strftime("%Y:%m:%d") 173 | exif_gps[piexif.GPSIFD.GPSTimeStamp] = [(hour, 1), 174 | (minutes, 1), 175 | (seconds, 1)] 176 | exif_gps[piexif.GPSIFD.GPSDateStamp] = day_timestamp_str 177 | 178 | dms_latitude = dd_to_dms(latitude) 179 | dms_longitude = dd_to_dms(longitude) 180 | exif_gps[piexif.GPSIFD.GPSLatitudeRef] = "S" if latitude < 0 else "N" 181 | exif_gps[piexif.GPSIFD.GPSLatitude] = dms_latitude 182 | exif_gps[piexif.GPSIFD.GPSLongitudeRef] = "W" if longitude < 0 else "E" 183 | exif_gps[piexif.GPSIFD.GPSLongitude] = dms_longitude 184 | return exif_gps 185 | 186 | 187 | def add_optional_gps_tags(exif_gps: Dict[str, Any], 188 | speed: Optional[float], 189 | altitude: Optional[float], 190 | compass: Optional[float]): 191 | """This method will append optional tags to exif_gps tags dictionary""" 192 | precision = 10000 193 | if speed: 194 | exif_gps[piexif.GPSIFD.GPSSpeed] = (int(speed * precision), precision) 195 | exif_gps[piexif.GPSIFD.GPSSpeedRef] = SpeedUnit.KMH.value 196 | if altitude: 197 | exif_gps[piexif.GPSIFD.GPSAltitude] = (int(altitude * precision), precision) 198 | sea_level = SeaLevel.BELOW.value if altitude < 0 else SeaLevel.ABOVE.value 199 | exif_gps[piexif.GPSIFD.GPSAltitudeRef] = sea_level 200 | if compass: 201 | exif_gps[piexif.GPSIFD.GPSImgDirection] = (int(compass * precision), precision) 202 | exif_gps[piexif.GPSIFD.GPSImgDirectionRef] = CardinalDirection.TRUE_NORTH.value 203 | -------------------------------------------------------------------------------- /parsers/geojson.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module constains custom OSC geojson parsing. This file is generated by KartaView Android app. 3 | """ 4 | from typing import Optional, List, Type 5 | import time 6 | 7 | from geojson import load 8 | 9 | from parsers.base import BaseParser 10 | from common.models import SensorItem, GPS 11 | 12 | TWO_WAY_KEY = "osctagging" 13 | TWO_WAY_VALUE = "twoway" 14 | 15 | ONE_WAY_KEY = "osctagging" 16 | ONE_WAY_VALUE = "oneway" 17 | 18 | CLOSED_KEY = "osctagging" 19 | CLOSED_VALUE = "closedRoad" 20 | 21 | NARROW_ROAD_KEY = "osctagging" 22 | NARROW_ROAD_VALUE = "narrowRoad" 23 | 24 | OTHER_KEY = "osctagging" 25 | OTHER_VALUE = "notes" 26 | 27 | 28 | class GeoJsonParser(BaseParser): 29 | """this class is a BaseParser that can parse a GPX""" 30 | 31 | def serialize(self): 32 | """serialize the sensors into geojson format""" 33 | print("This method is not implemeted for GeoJsonParser", self) 34 | 35 | def next_item_with_class(self, _: Type[SensorItem]) -> Optional[SensorItem]: 36 | print("This method is not implemented for GeoJsonParser", self) 37 | 38 | def items_with_class(self, _: Type[SensorItem]) -> List[SensorItem]: 39 | print("This method is not implemeted for GeoJsonParser", self) 40 | return [] 41 | 42 | def next_item(self) -> Optional[SensorItem]: 43 | print("This method is not implemeted for GeoJsonParser", self) 44 | 45 | def items(self) -> List[SensorItem]: 46 | with self._storage.open(self.file_path, 'r') as geo_json_file: 47 | geo_json = load(geo_json_file) 48 | index = 0 49 | sensors: List[SensorItem] = [] 50 | for feature in geo_json["features"]: 51 | geometry = feature["geometry"] 52 | coordinates = geometry["coordinates"] 53 | 54 | for geometry_coordinate in coordinates: 55 | if isinstance(geometry_coordinate, float) and len(coordinates) == 2: 56 | # this is a point 57 | gps = GPS() 58 | gps.timestamp = time.time() + index 59 | gps.latitude = coordinates[1] 60 | gps.longitude = coordinates[0] 61 | sensors.append(gps) 62 | break 63 | 64 | if (isinstance(geometry_coordinate[0], float) 65 | and len(geometry_coordinate) == 2): 66 | # this is a list of points 67 | longitude = geometry_coordinate[0] 68 | latitude = geometry_coordinate[1] 69 | gps = GPS() 70 | gps.timestamp = time.time() + index 71 | gps.latitude = latitude 72 | gps.longitude = longitude 73 | sensors.append(gps) 74 | index += 1 75 | else: 76 | # this is a list of list of points 77 | for geometry_point_coordinate in geometry_coordinate[0]: 78 | longitude = geometry_point_coordinate[0] 79 | latitude = geometry_point_coordinate[1] 80 | gps = GPS() 81 | gps.timestamp = time.time() + index 82 | gps.latitude = latitude 83 | gps.longitude = longitude 84 | sensors.append(gps) 85 | index += 1 86 | return sensors 87 | 88 | def format_version(self) -> Optional[str]: 89 | print("GeoJsonParser version", self) 90 | return "unknown" 91 | 92 | @classmethod 93 | def compatible_sensors(cls): 94 | return [GPS] 95 | -------------------------------------------------------------------------------- /parsers/gpx.py: -------------------------------------------------------------------------------- 1 | """Module responsible to parse Exif information from a image""" 2 | from typing import Optional, List, Type 3 | from datetime import datetime 4 | 5 | import gpxpy.gpx 6 | 7 | from io_storage.storage import Storage 8 | from parsers.base import BaseParser 9 | from common.models import SensorItem, GPS 10 | 11 | 12 | class GPXParser(BaseParser): 13 | """this class is a BaseParser that can parse a GPX""" 14 | def __init__(self, path: str, storage: Storage): 15 | super().__init__(path, storage) 16 | self._data_pointer = 0 17 | 18 | def next_item_with_class(self, item_class: Type[SensorItem]) -> Optional[SensorItem]: 19 | if item_class != GPS: 20 | return None 21 | with self._storage.open(self.file_path, 'r') as gpx_file: 22 | gpx = gpxpy.parse(gpx_file) 23 | index = 0 24 | for track in gpx.tracks: 25 | for segment in track.segments: 26 | for point in segment.points: 27 | if index == self._data_pointer: 28 | gps = GPS() 29 | gps.speed = point.speed 30 | gps.timestamp = point.time.timestamp() 31 | gps.latitude = point.latitude 32 | gps.longitude = point.longitude 33 | gps.altitude = point.elevation 34 | self._data_pointer += 1 35 | return gps 36 | return None 37 | 38 | def items_with_class(self, item_class: Type[SensorItem]) -> List[SensorItem]: 39 | if item_class != GPS: 40 | return [] 41 | with open(self.file_path, 'r') as gpx_file: 42 | gpx = gpxpy.parse(gpx_file) 43 | sensors: List[SensorItem] = [] 44 | for track in gpx.tracks: 45 | for segment in track.segments: 46 | for point in segment.points: 47 | gps = GPS() 48 | gps.speed = point.speed 49 | gps.timestamp = point.time.timestamp() 50 | gps.latitude = point.latitude 51 | gps.longitude = point.longitude 52 | gps.altitude = point.elevation 53 | sensors.append(gps) 54 | return sensors 55 | 56 | def next_item(self) -> Optional[SensorItem]: 57 | with open(self.file_path, 'r') as gpx_file: 58 | gpx = gpxpy.parse(gpx_file) 59 | index = 0 60 | for track in gpx.tracks: 61 | for segment in track.segments: 62 | for point in segment.points: 63 | if index == self._data_pointer: 64 | gps = GPS() 65 | gps.speed = point.speed 66 | gps.timestamp = point.time.timestamp() 67 | gps.latitude = point.latitude 68 | gps.longitude = point.longitude 69 | gps.altitude = point.elevation 70 | self._data_pointer += 1 71 | return gps 72 | return None 73 | 74 | def items(self) -> List[SensorItem]: 75 | with open(self.file_path, 'r') as gpx_file: 76 | gpx = gpxpy.parse(gpx_file) 77 | sensors: List[SensorItem] = [] 78 | for track in gpx.tracks: 79 | for segment in track.segments: 80 | for point in segment.points: 81 | gps = GPS() 82 | gps.speed = point.speed 83 | gps.timestamp = point.time.timestamp() 84 | gps.latitude = point.latitude 85 | gps.longitude = point.longitude 86 | gps.altitude = point.elevation 87 | sensors.append(gps) 88 | return sensors 89 | 90 | def format_version(self) -> Optional[str]: 91 | with open(self.file_path, 'r') as gpx_file: 92 | gpx = gpxpy.parse(gpx_file) 93 | return gpx.version 94 | 95 | def serialize(self): 96 | my_gpx = gpxpy.gpx.GPX() 97 | # Create first track in our GPX: 98 | gpx_track = gpxpy.gpx.GPXTrack() 99 | my_gpx.tracks.append(gpx_track) 100 | 101 | # Create first segment in our GPX track: 102 | gpx_segment = gpxpy.gpx.GPXTrackSegment() 103 | gpx_track.segments.append(gpx_segment) 104 | for item in self._sensors: 105 | if isinstance(item, GPS): 106 | gpx_segment.points.append(self._gpx_track_point(item)) 107 | # elif isinstance(item, PhotoMetadata): 108 | # gpx_segment.points.append(gpxpy.gpx.GPXTrackPoint(latitude=item.gps.latitude, 109 | # longitude=item.gps.longitude, 110 | # elevation=item.gps.altitude, 111 | # time=item.gps.timestamp, 112 | # speed=item.gps.speed)) 113 | xml_data = my_gpx.to_xml() 114 | with open(self.file_path, "w+") as file: 115 | file.write(xml_data) 116 | file.close() 117 | 118 | @classmethod 119 | def _gpx_track_point(cls, item: GPS) -> Optional[gpxpy.gpx.GPXTrackPoint]: 120 | point = None 121 | if item.timestamp is None: 122 | return None 123 | 124 | item.timestamp += 1526814822 125 | if item.latitude and item.longitude and item.timestamp: 126 | time = datetime.fromtimestamp(float(item.timestamp)) 127 | point = gpxpy.gpx.GPXTrackPoint(latitude=float(item.latitude), 128 | longitude=float(item.longitude), 129 | time=time) 130 | if point and item.speed: 131 | point.speed = float(item.speed) 132 | 133 | if point and item.altitude: 134 | point.elevation = float(item.altitude) 135 | 136 | return point 137 | 138 | @classmethod 139 | def compatible_sensors(cls): 140 | return [GPS] 141 | -------------------------------------------------------------------------------- /parsers/osc_metadata/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kartaview/upload-scripts/28650a631e398ebe8991c1c965248d41ae68dd87/parsers/osc_metadata/__init__.py -------------------------------------------------------------------------------- /parsers/osc_metadata/item_factory.py: -------------------------------------------------------------------------------- 1 | """this file contains all item parsers for Metadata2.0""" 2 | from typing import List, Optional 3 | 4 | from common.models import SensorItem, PhotoMetadata, GPS, Acceleration, Compass, OBD, Pressure 5 | from common.models import Attitude, Gravity, DeviceMotion, OSCDevice, RecordingType 6 | from common.models import CameraParameters, ExifParameters, CameraProjection 7 | 8 | 9 | class ItemParser: 10 | """ItemParser is a parser class that can parse a Metadata2.0 row and return a SensorItem""" 11 | 12 | # pylint: disable=R0913 13 | def __init__(self, version: int, 14 | formats: dict, 15 | item_class, 16 | item_name, 17 | post_processing=None): 18 | self.version: int = version 19 | self.format: dict = formats 20 | self.item_class = item_class 21 | self.item_name = item_name 22 | self.post_processing = post_processing 23 | 24 | # pylint: enable=R0913 25 | 26 | def __eq__(self, other): 27 | if isinstance(other, ItemParser): 28 | return self.version == other.version and \ 29 | self.format == other.format and \ 30 | self.item_class == other.item_class 31 | return False 32 | 33 | def __hash__(self): 34 | return hash((self.version, self.format, self.item_class)) 35 | 36 | def parse(self, row, timestamp) -> Optional[SensorItem]: 37 | """This method will return a complete Metadata Item instance that was found at the row 38 | received as parameter""" 39 | _elements = row.replace("\n", "").split(";") 40 | if len(_elements) != len(self.format): 41 | return None 42 | 43 | item_instance = self.item_class() 44 | item_instance.timestamp = float(timestamp) 45 | for attribute_key, attribute_value in self.format.items(): 46 | if "." in attribute_key: 47 | sub_attributes = attribute_key.split(".") 48 | # get the sub item that has the property that needs to be set 49 | tmp_item = item_instance 50 | for level in range(len(sub_attributes) - 1): 51 | tmp_item = getattr(tmp_item, sub_attributes[level]) 52 | setattr(tmp_item, 53 | sub_attributes[len(sub_attributes) - 1], 54 | ItemParser._value(_elements, attribute_value)) 55 | 56 | else: 57 | setattr(item_instance, 58 | attribute_key, 59 | ItemParser._value(_elements, attribute_value)) 60 | if self.post_processing is not None: 61 | self.post_processing(item_instance) 62 | 63 | return item_instance 64 | 65 | @classmethod 66 | def _value(cls, elements, key) -> Optional[str]: 67 | if elements[key] != '': 68 | return elements[key] 69 | return None 70 | 71 | 72 | class SensorItemDefinition: 73 | """SensorItemDefinition is a model class for the Metadata2.0 header rows""" 74 | def __init__(self): 75 | self.alias = None 76 | self.item_name = None 77 | self.version = None 78 | self.min_compatible_version = None 79 | self.parsers: [ItemParser] = None 80 | 81 | def __eq__(self, other): 82 | if isinstance(other, SensorItemDefinition): 83 | return self.alias == other.alias and \ 84 | self.item_name == other.item_name and \ 85 | self.version == other.version and \ 86 | self.min_compatible_version == other.min_compatible_version and \ 87 | len(self.parsers) == len(other.parsers) 88 | return False 89 | 90 | def __hash__(self): 91 | return hash((self.alias, 92 | self.item_name, 93 | self.version, 94 | self.min_compatible_version, 95 | len(self.parsers))) 96 | 97 | @classmethod 98 | def definition_from_row(cls, row): 99 | """This method will return a SensorItemDefinition compatible with the row sent as 100 | parameter. If no compatible SensorItemDefinition is found then it will return None""" 101 | if ";" not in row or ":" not in row: 102 | return None 103 | elements = row.split(":") 104 | if len(elements) != 2: 105 | return None 106 | elements = elements[1].split(";") 107 | if len(elements) != 4: 108 | return None 109 | definition = SensorItemDefinition() 110 | definition.alias = elements[0] 111 | definition.item_name = elements[1] 112 | definition.version = elements[2] 113 | definition.min_compatible_version = elements[3] 114 | definition.parsers = _compatible_parser(definition) 115 | if not definition.parsers: 116 | return None 117 | # setting the definition on the first parser will set class 118 | class_name = definition.parsers[0].item_class 119 | class_name.definition = definition 120 | 121 | return definition 122 | 123 | 124 | def _available_parsers() -> List[ItemParser]: 125 | """This function returns all the available item parsers""" 126 | parsers = [photo_v1(), 127 | gps_v1(), 128 | acceleration_v1(), 129 | compass_v1(), 130 | obd_v1(), 131 | pressure_v1(), 132 | attitude_v1(), 133 | gravity_v1(), 134 | device_v1(), 135 | device_motion_v1(), 136 | camera_v1(), 137 | camera_v2(), 138 | exif_v1(), 139 | exif_v2()] 140 | return parsers 141 | 142 | 143 | def _compatible_parser(definition: SensorItemDefinition) -> List[ItemParser]: 144 | """This function returns all the compatible item parsers for a specific 145 | SensorItemDefinition""" 146 | 147 | item_name = definition.item_name 148 | min_version = definition.min_compatible_version 149 | compatible = [] 150 | for parser in _available_parsers(): 151 | if parser.version >= int(min_version) and parser.item_name == item_name: 152 | compatible.append(parser) 153 | return compatible 154 | 155 | 156 | def photo_v1(): 157 | """This is a factory method that returns a item parser for version 1 of the photo row""" 158 | 159 | def type_conversions(photo_metadata: PhotoMetadata): 160 | photo_metadata.video_index = int(photo_metadata.video_index) 161 | photo_metadata.frame_index = int(photo_metadata.frame_index) 162 | photo_metadata.gps.timestamp = float(photo_metadata.gps.timestamp) 163 | photo_metadata.gps.latitude = float(photo_metadata.gps.latitude) 164 | photo_metadata.gps.longitude = float(photo_metadata.gps.longitude) 165 | photo_metadata.gps.horizontal_accuracy = float(photo_metadata.gps.horizontal_accuracy) 166 | photo_metadata.gps.speed = float(photo_metadata.gps.speed) 167 | 168 | if photo_metadata.obd.speed is not None and photo_metadata.obd.timestamp is not None: 169 | photo_metadata.obd.timestamp = float(photo_metadata.obd.timestamp) 170 | photo_metadata.obd.speed = float(photo_metadata.obd.speed) 171 | 172 | if (photo_metadata.compass.compass is not None 173 | and photo_metadata.compass.timestamp is not None): 174 | photo_metadata.compass.compass = float(photo_metadata.compass.compass) 175 | photo_metadata.compass.timestamp = float(photo_metadata.compass.timestamp) 176 | 177 | photo_parser = ItemParser(1, {'video_index': 0, 178 | 'frame_index': 1, 179 | 'gps.timestamp': 2, 180 | 'gps.latitude': 3, 181 | 'gps.longitude': 4, 182 | 'gps.horizontal_accuracy': 5, 183 | 'gps.speed': 6, 184 | 'compass.timestamp': 7, 185 | 'compass.compass': 8, 186 | 'obd.timestamp': 9, 187 | 'obd.speed': 10}, 188 | PhotoMetadata, 189 | "PHOTO", 190 | type_conversions) 191 | 192 | return photo_parser 193 | 194 | 195 | def gps_v1(): 196 | """This is a factory method to get a ItemParser for the version 1 of the GPS found in 197 | Metadata format""" 198 | 199 | def type_conversions(gps: GPS): 200 | gps.latitude = float(gps.latitude) 201 | gps.longitude = float(gps.longitude) 202 | gps.altitude = float(gps.altitude) 203 | gps.horizontal_accuracy = float(gps.horizontal_accuracy) 204 | gps.vertical_accuracy = float(gps.vertical_accuracy) 205 | gps.speed = float(gps.speed) 206 | 207 | gps_parser = ItemParser(1, {'latitude': 0, 208 | 'longitude': 1, 209 | 'altitude': 2, 210 | 'horizontal_accuracy': 3, 211 | 'vertical_accuracy': 4, 212 | 'speed': 5}, 213 | GPS, 214 | "GPS", 215 | type_conversions) 216 | return gps_parser 217 | 218 | 219 | def acceleration_v1() -> ItemParser: 220 | """This method is returns a Acceleration ItemParser for Acceleration row version 1""" 221 | 222 | def type_conversions(acceleration: Acceleration): 223 | acceleration.acc_x = float(acceleration.acc_x) 224 | acceleration.acc_y = float(acceleration.acc_y) 225 | acceleration.acc_z = float(acceleration.acc_z) 226 | 227 | parser = ItemParser(1, {'acc_x': 0, 228 | 'acc_y': 1, 229 | 'acc_z': 2}, 230 | Acceleration, 231 | "ACCELERATION", 232 | type_conversions) 233 | return parser 234 | 235 | 236 | def compass_v1() -> ItemParser: 237 | """This method is returns a Compass ItemParser for Compass row version 1""" 238 | 239 | def type_conversions(compass: Compass): 240 | compass.compass = float(compass.compass) 241 | 242 | parser = ItemParser(1, {'compass': 0}, Compass, "COMPASS", type_conversions) 243 | return parser 244 | 245 | 246 | def obd_v1() -> ItemParser: 247 | """This method is returns a OBD ItemParser for OBD row version 1""" 248 | 249 | def type_conversions(obd: OBD): 250 | obd.speed = float(obd.speed) 251 | 252 | parser = ItemParser(1, {'speed': 0}, OBD, "OBD", type_conversions) 253 | return parser 254 | 255 | 256 | def pressure_v1() -> ItemParser: 257 | """This method is returns a Pressure ItemParser for Pressure row version 1""" 258 | 259 | def type_conversions(pressure: Pressure): 260 | pressure.pressure = float(pressure.pressure) 261 | 262 | parser = ItemParser(1, {'pressure': 0}, Pressure, "PRESSURE", type_conversions) 263 | return parser 264 | 265 | 266 | def attitude_v1() -> ItemParser: 267 | """This method is returns a Attitude ItemParser for Attitude row version 1""" 268 | 269 | def type_conversions(attitude: Attitude): 270 | attitude.yaw = float(attitude.yaw) 271 | attitude.pitch = float(attitude.pitch) 272 | attitude.roll = float(attitude.roll) 273 | 274 | parser = ItemParser(1, {'yaw': 0, 275 | 'pitch': 1, 276 | 'roll': 2}, 277 | Attitude, "ATTITUDE", 278 | type_conversions) 279 | return parser 280 | 281 | 282 | def gravity_v1() -> ItemParser: 283 | """This method is returns a Gravity ItemParser for Gravity row version 1""" 284 | 285 | def type_conversions(gravity: Gravity): 286 | gravity.acc_x = float(gravity.acc_x) 287 | gravity.acc_y = float(gravity.acc_y) 288 | gravity.acc_z = float(gravity.acc_z) 289 | 290 | parser = ItemParser(1, {'acc_x': 0, 291 | 'acc_y': 1, 292 | 'acc_z': 2}, 293 | Gravity, "GRAVITY", 294 | type_conversions) 295 | return parser 296 | 297 | 298 | def device_motion_v1() -> ItemParser: 299 | """This method is returns a DeviceMotion ItemParser for DeviceMotion row version 1""" 300 | 301 | parser = ItemParser(1, {'gyroscope.yaw': 0, 302 | 'gyroscope.pitch': 1, 303 | 'gyroscope.roll': 2, 304 | 'acceleration.acc_x': 3, 305 | 'acceleration.acc_y': 4, 306 | 'acceleration.acc_z': 5, 307 | 'gravity.acc_x': 6, 308 | 'gravity.acc_y': 7, 309 | 'gravity.acc_z': 8}, 310 | DeviceMotion, "DEVICEMOTION", 311 | DeviceMotion.type_conversions) 312 | return parser 313 | 314 | 315 | def device_v1() -> ItemParser: 316 | """This method is returns a OSCDevice ItemParser for OSCDevice row version 1""" 317 | 318 | def type_conversions(device: OSCDevice): 319 | if "photo" in device.recording_type: 320 | device.recording_type = RecordingType.PHOTO 321 | elif "video" in device.recording_type: 322 | device.recording_type = RecordingType.VIDEO 323 | else: 324 | device.recording_type = RecordingType.UNKNOWN 325 | 326 | parser = ItemParser(1, {'platform_name': 0, 327 | 'os_raw_name': 1, 328 | 'os_version': 2, 329 | 'device_raw_name': 3, 330 | 'app_version': 4, 331 | 'app_build_number': 5, 332 | 'recording_type': 6}, 333 | OSCDevice, "DEVICE", 334 | type_conversions) 335 | return parser 336 | 337 | 338 | def camera_v1() -> ItemParser: 339 | """This method is returns a CameraParameters ItemParser for CameraParameters row 340 | version 1""" 341 | def type_conversion(camera: CameraParameters): 342 | camera.h_fov = float(camera.h_fov) 343 | camera.v_fov = None 344 | camera.projection = CameraProjection.PLAIN 345 | 346 | parser = ItemParser(1, {'h_fov': 0, 347 | 'v_fov': 1, 348 | 'aperture': 2}, 349 | CameraParameters, "CAMERA", type_conversion) 350 | 351 | return parser 352 | 353 | 354 | def camera_v2() -> ItemParser: 355 | """This method is returns a CameraParameters ItemParser for CameraParameters row 356 | version 2 like hFoV;vFoV;aperture """ 357 | def type_conversion(camera: CameraParameters): 358 | camera.h_fov = float(camera.h_fov) 359 | camera.v_fov = float(camera.v_fov) 360 | 361 | parser = ItemParser(2, {'h_fov': 0, 362 | 'v_fov': 1, 363 | 'aperture': 2}, 364 | CameraParameters, "CAMERA", type_conversion) 365 | return parser 366 | 367 | 368 | def exif_v1() -> ItemParser: 369 | """This method is returns a ExifParameters ItemParser for ExifParameters row version 1""" 370 | def type_conversion(exif: ExifParameters): 371 | exif.focal_length = float(exif.focal_length) 372 | 373 | parser = ItemParser(1, {'focal_length': 1}, 374 | ExifParameters, "EXIF", type_conversion) 375 | return parser 376 | 377 | 378 | def exif_v2() -> ItemParser: 379 | """This method is returns a ExifParameters ItemParser for ExifParameters row version 2""" 380 | def type_conversion(exif: ExifParameters): 381 | exif.focal_length = float(exif.focal_length) 382 | exif.width = int(exif.width) 383 | exif.height = int(exif.height) 384 | 385 | parser = ItemParser(2, {'focal_length': 0, 386 | 'width': 1, 387 | 'height': 2}, 388 | ExifParameters, "EXIF", type_conversion) 389 | return parser 390 | -------------------------------------------------------------------------------- /parsers/osc_metadata/legacy_item_factory.py: -------------------------------------------------------------------------------- 1 | """This file contains all the Metadata 1.x item parser definitions""" 2 | from typing import Optional, Dict, Any 3 | 4 | from common.models import SensorItem, Pressure, PhotoMetadata, OBD, DeviceMotion, Acceleration, GPS 5 | from common.models import Attitude, Gravity, Compass 6 | 7 | 8 | class ItemLegacyParser: 9 | """ItemLegacyParser is a parser class that can parse a Metadata1.x row and 10 | return a SensorItem""" 11 | 12 | # pylint: disable=R0913 13 | def __init__(self, 14 | metadata_format, 15 | item_class, 16 | required_attributes_mapping, 17 | optional_attributes_mapping=None, 18 | post_processing=None): 19 | if optional_attributes_mapping is None: 20 | optional_attributes_mapping = {} 21 | 22 | self._metadata_format = metadata_format 23 | self._item_class = item_class 24 | self._attributes_element_names = required_attributes_mapping 25 | self._optional_attributes_element_names = optional_attributes_mapping 26 | self._post_processing = post_processing 27 | 28 | # pylint: enable=R0913 29 | 30 | def __eq__(self, other): 31 | if isinstance(other, ItemLegacyParser): 32 | return self == other 33 | return False 34 | 35 | def __hash__(self): 36 | return hash((self._item_class, self._metadata_format)) 37 | 38 | def parse(self, elements) -> Optional[SensorItem]: 39 | """parse a list of elements""" 40 | # search for required attributes 41 | element_values: Dict[str, Any] = {} 42 | for _, element_name in self._attributes_element_names.items(): 43 | name_value = self._value(elements, element_name) 44 | if not name_value: 45 | return None 46 | element_values[element_name] = name_value 47 | # set required attributes 48 | item_instance = self._item_class() 49 | self._set_values_for_attributes(element_values, 50 | self._attributes_element_names, 51 | item_instance) 52 | 53 | # set optional attributes 54 | element_values = {} 55 | for _, element_name in self._optional_attributes_element_names.items(): 56 | name_value = self._value(elements, element_name) 57 | element_values[element_name] = name_value 58 | self._set_values_for_attributes(element_values, 59 | self._optional_attributes_element_names, 60 | item_instance) 61 | # make post processing 62 | if self._post_processing: 63 | self._post_processing(item_instance) 64 | 65 | return item_instance 66 | 67 | @classmethod 68 | def _set_values_for_attributes(cls, element_values, attributes_element_names, item_instance): 69 | for attribute_name, element_name in attributes_element_names.items(): 70 | if "." in attribute_name: 71 | sub_attributes = attribute_name.split(".") 72 | # get the sub item that has the property that needs to be set 73 | tmp_item = item_instance 74 | for level in range(len(sub_attributes) - 1): 75 | tmp_item = getattr(tmp_item, sub_attributes[level]) 76 | setattr(tmp_item, 77 | sub_attributes[len(sub_attributes)-1], 78 | element_values[element_name]) 79 | else: 80 | setattr(item_instance, 81 | attribute_name, 82 | element_values[element_name]) 83 | 84 | def _value(self, elements, key): 85 | if key not in self._metadata_format: 86 | return None 87 | 88 | if elements[self._metadata_format[key]] != '': 89 | return elements[self._metadata_format[key]] 90 | return None 91 | 92 | 93 | def timestamp_error(item: SensorItem): 94 | """this function fixes an error when timestamp is logged in a smaller unit then seconds.""" 95 | if float(item.timestamp) / 3600 * 24 * 356 > 2019 and \ 96 | "." not in str(item.timestamp) and \ 97 | len(str(item.timestamp)) > 10: 98 | # this bug has fixed in 2018 99 | # 1471117570183 -> 1471117570.183 100 | item.timestamp = item.timestamp[:10] + "." + item.timestamp[10:] 101 | item.timestamp = float(item.timestamp) 102 | 103 | 104 | def gps_parser(metadata_format, device_item): 105 | """gps parser""" 106 | def waylens_device(gps: GPS): 107 | """this function fixes an error for waylens metadata when gps speed is logged in m/s""" 108 | timestamp_error(gps) 109 | if "waylens" in device_item.device_raw_name: 110 | gps.speed = str(float(gps.speed) / 3.6) 111 | gps.latitude = float(gps.latitude) 112 | gps.longitude = float(gps.longitude) 113 | gps.horizontal_accuracy = float(gps.horizontal_accuracy) 114 | if gps.altitude is not None: 115 | gps.altitude = float(gps.altitude) 116 | if gps.vertical_accuracy is not None: 117 | gps.vertical_accuracy = float(gps.vertical_accuracy) 118 | if gps.speed is not None: 119 | gps.speed = float(gps.speed) 120 | 121 | return ItemLegacyParser(metadata_format, 122 | GPS, 123 | {"timestamp": "time", 124 | "latitude": "latitude", 125 | "longitude": "longitude", 126 | "horizontal_accuracy": "horizontal_accuracy"}, 127 | {"altitude": "elevation", 128 | "vertical_accuracy": "vertical_accuracy", 129 | "gps_speed": "gps.speed"}, 130 | waylens_device) 131 | 132 | 133 | def obd_parser(metadata_format): 134 | """OBD parser""" 135 | def type_conversions(obd: OBD): 136 | timestamp_error(obd) 137 | obd.speed = float(obd.speed) 138 | 139 | return ItemLegacyParser(metadata_format, 140 | OBD, 141 | {"timestamp": "time", 142 | "speed": "OBDs"}, 143 | {}, 144 | type_conversions) 145 | 146 | 147 | def pressure_parser(metadata_format): 148 | """pressure parser""" 149 | def type_conversions(pressure: Pressure): 150 | timestamp_error(pressure) 151 | pressure.pressure = float(pressure.pressure) 152 | 153 | return ItemLegacyParser(metadata_format, 154 | Pressure, 155 | {"timestamp": "time", 156 | "pressure": "pressure"}, 157 | {}, 158 | type_conversions) 159 | 160 | 161 | def compass_parser(metadata_format): 162 | """compass parser""" 163 | def type_conversions(compass: Compass): 164 | timestamp_error(compass) 165 | compass.compass = float(compass.compass) 166 | 167 | return ItemLegacyParser(metadata_format, 168 | Compass, 169 | {"timestamp": "time", 170 | "compass": "compass"}, 171 | {}, 172 | type_conversions) 173 | 174 | 175 | def attitude_parser(metadata_format): 176 | """attitude parser""" 177 | def type_conversions(attitude: Attitude): 178 | timestamp_error(attitude) 179 | attitude.yaw = float(attitude.yaw) 180 | attitude.pitch = float(attitude.pitch) 181 | attitude.roll = float(attitude.roll) 182 | 183 | return ItemLegacyParser(metadata_format, 184 | Attitude, 185 | {"timestamp": "time", 186 | "yaw": "yaw", 187 | "pitch": "pitch", 188 | "roll": "roll"}, 189 | {}, 190 | type_conversions) 191 | 192 | 193 | def gravity_parser(metadata_format): 194 | """gravity parser""" 195 | def type_conversions(gravity: Gravity): 196 | timestamp_error(gravity) 197 | gravity.acc_x = float(gravity.acc_x) 198 | gravity.acc_y = float(gravity.acc_y) 199 | gravity.acc_z = float(gravity.acc_z) 200 | 201 | return ItemLegacyParser(metadata_format, 202 | Gravity, 203 | {"timestamp": "time", 204 | "acc_x": "gravity.x", 205 | "acc_y": "gravity.y", 206 | "acc_z": "gravity.z"}, 207 | {}, 208 | type_conversions) 209 | 210 | 211 | def acceleration_parser(metadata_format): 212 | """acceleration parser""" 213 | def type_conversions(acceleration: Acceleration): 214 | timestamp_error(acceleration) 215 | acceleration.acc_x = float(acceleration.acc_x) 216 | acceleration.acc_y = float(acceleration.acc_y) 217 | acceleration.acc_z = float(acceleration.acc_z) 218 | 219 | return ItemLegacyParser(metadata_format, 220 | Acceleration, 221 | {"timestamp": "time", 222 | "acc_x": "acceleration.x", 223 | "acc_y": "acceleration.y", 224 | "acc_z": "acceleration.z"}, 225 | {}, 226 | type_conversions) 227 | 228 | 229 | def incomplete_photo_parser(metadata_format): 230 | """photo parser""" 231 | def type_conversions(photo_metadata: PhotoMetadata): 232 | timestamp_error(photo_metadata) 233 | if photo_metadata.video_index is not None: 234 | photo_metadata.video_index = int(photo_metadata.video_index) 235 | photo_metadata.frame_index = int(photo_metadata.frame_index) 236 | 237 | return ItemLegacyParser(metadata_format, 238 | PhotoMetadata, 239 | {"timestamp": "time", 240 | "frame_index": "frame_index"}, 241 | {"video_index": "video_index"}, 242 | type_conversions) 243 | 244 | 245 | def device_motion_parse(metadata_format): 246 | """device motion parser""" 247 | def type_conversions(device_motion: DeviceMotion): 248 | timestamp_error(device_motion) 249 | DeviceMotion.type_conversions(device_motion) 250 | 251 | return ItemLegacyParser(metadata_format, 252 | DeviceMotion, 253 | {"acceleration.acc_x": "acceleration.x", 254 | "acceleration.acc_y": "acceleration.y", 255 | "acceleration.acc_z": "acceleration.z", 256 | "gravity.acc_x": "gravity.x", 257 | "gravity.acc_y": "gravity.y", 258 | "gravity.acc_z": "gravity.z", 259 | "gyroscope.yaw": "yaw", 260 | "gyroscope.pitch": "pitch", 261 | "gyroscope.roll": "roll"}, 262 | {}, 263 | type_conversions) 264 | -------------------------------------------------------------------------------- /parsers/xmp.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module is used to read XMP data from images. 3 | """ 4 | import struct 5 | from typing import Optional, Tuple, List, Any, Type 6 | from xml.etree.ElementTree import fromstring, ParseError 7 | 8 | from io_storage.storage import Storage 9 | from parsers.base import BaseParser 10 | from common.models import SensorItem, CameraParameters, projection_type_from_name, ExifParameters 11 | 12 | 13 | class XMPParser(BaseParser): 14 | """xmp parser for xmp image header""" 15 | 16 | def __init__(self, file_path: str, storage: Storage): 17 | super().__init__(file_path, storage) 18 | self._data_pointer = 0 19 | self._body_pointer = 0 20 | self.xmp_str = self._read_xmp() 21 | 22 | def _read_xmp(self) -> str: 23 | with self._storage.open(self.file_path, "rb") as image: 24 | data = image.read() 25 | xmp_start = data.find(b' Optional[SensorItem]: 32 | if item_class == CameraParameters: 33 | return self._camera_item() 34 | return None 35 | 36 | def items_with_class(self, item_class: Type[SensorItem]) -> List[SensorItem]: 37 | next_item = self.next_item_with_class(item_class) 38 | if next_item is not None: 39 | return [next_item] 40 | return [] 41 | 42 | def next_item(self) -> Optional[SensorItem]: 43 | if self._data_pointer == 0: 44 | self._data_pointer = 1 45 | return self._camera_item() 46 | 47 | def items(self) -> List[SensorItem]: 48 | camera = self._camera_item() 49 | if camera is not None: 50 | return [camera] 51 | return [] 52 | 53 | def format_version(self) -> Optional[str]: 54 | raise NotImplementedError(f"XMP format version - {self}") 55 | 56 | @classmethod 57 | def compatible_sensors(cls) -> List[Any]: 58 | return [CameraParameters] 59 | 60 | def _recursive_get_camera_item(self, root, 61 | full_pano_image_width=None, 62 | cropped_area_image_width_pixels=None, 63 | projection=None): 64 | elements = root.findall("*") 65 | for element in elements: 66 | [full_pano_image_width, cropped_area_image_width_pixels, 67 | projection] = self.compute_camera_items(element, full_pano_image_width, 68 | cropped_area_image_width_pixels, 69 | projection) 70 | if None not in [cropped_area_image_width_pixels, full_pano_image_width, 71 | projection]: 72 | return full_pano_image_width, cropped_area_image_width_pixels, projection 73 | [full_pano_image_width, cropped_area_image_width_pixels, 74 | projection] = self.compute_camera_items_for_garmin(element, 75 | full_pano_image_width, 76 | cropped_area_image_width_pixels, 77 | projection) 78 | if None not in [cropped_area_image_width_pixels, full_pano_image_width, 79 | projection]: 80 | return full_pano_image_width, cropped_area_image_width_pixels, projection 81 | 82 | sub_elements = element.findall("*") 83 | for sub_element in sub_elements: 84 | [f_pano_image_width, c_area_image_width_pixels, 85 | sub_projection] = self._recursive_get_camera_item(sub_element, 86 | full_pano_image_width, 87 | cropped_area_image_width_pixels, 88 | projection) 89 | if None not in [cropped_area_image_width_pixels or c_area_image_width_pixels, 90 | full_pano_image_width or f_pano_image_width, 91 | projection or sub_projection]: 92 | return full_pano_image_width or f_pano_image_width, \ 93 | cropped_area_image_width_pixels or c_area_image_width_pixels, \ 94 | projection or sub_projection 95 | 96 | return full_pano_image_width, cropped_area_image_width_pixels, projection 97 | 98 | def _camera_item(self) -> Optional[CameraParameters]: 99 | try: 100 | root = fromstring(self.xmp_str) 101 | [full_pano_image_width, cropped_area_image_width_pixels, 102 | projection] = self._recursive_get_camera_item(root) 103 | if cropped_area_image_width_pixels is not None \ 104 | and full_pano_image_width is not None \ 105 | and projection is not None: 106 | parameters = CameraParameters() 107 | parameters.h_fov = cropped_area_image_width_pixels * 360 / full_pano_image_width 108 | parameters.projection = projection_type_from_name(projection) 109 | return parameters 110 | return None 111 | except ParseError: 112 | return None 113 | 114 | @staticmethod 115 | def compute_camera_items(xml_tags, 116 | full_pano_image_width, 117 | cropped_area_image_width_pixels, 118 | projection) -> Tuple[Optional[int], Optional[int], Optional[str]]: 119 | for attr_name, attr_value in xml_tags.items(): 120 | if "FullPanoWidthPixels" in attr_name: 121 | full_pano_image_width = int(attr_value) 122 | if "CroppedAreaImageWidthPixels" in attr_name: 123 | cropped_area_image_width_pixels = int(attr_value) 124 | if "ProjectionType" in attr_name: 125 | projection = attr_value 126 | 127 | return full_pano_image_width, cropped_area_image_width_pixels, projection 128 | 129 | @staticmethod 130 | def compute_camera_items_for_garmin(xml_child, 131 | full_pano_image_width, 132 | cropped_area_image_width_pixels, 133 | projection) -> Tuple[Optional[int], 134 | Optional[int], 135 | Optional[str]]: 136 | if "FullPanoWidthPixels" in xml_child.tag: 137 | full_pano_image_width = int(xml_child.text) 138 | if "CroppedAreaImageWidthPixels" in xml_child.tag: 139 | cropped_area_image_width_pixels = int(xml_child.text) 140 | if "ProjectionType" in xml_child.tag: 141 | projection = xml_child.text 142 | return full_pano_image_width, cropped_area_image_width_pixels, projection 143 | 144 | def serialize(self): 145 | with self._storage.open(self.file_path, "rb") as image: 146 | data = image.read() 147 | start = data.find(b'\xff\xe1') 148 | height = 0 149 | width = 0 150 | for item in self._sensors: 151 | if isinstance(item, ExifParameters): 152 | height = item.height 153 | width = item.width 154 | # print(str(hex_val) + xmp_header) 155 | # pylint: disable=C0301 156 | xmp_header = '''\n\n\nTrue\nequirectangular\n0.0\n0.0\n0.0\n0.0\n0.0\n0.0\n0.0\n0.0\n0\n0\n{imagewidth}\n{imageheight}\n{imagewidth}\n{imageheight}\n\n\n'''.format( 157 | imageheight=height, imagewidth=width) 158 | # pylint: enable=C0301 159 | string_len = len(xmp_header.encode('utf-8')) + 2 160 | xmp_header = b'\xff\xe1' + struct.pack('>h', string_len) + xmp_header.encode('utf-8') 161 | if start == -1: 162 | with self._storage.open(self.file_path, "wb") as out_image: 163 | out_image.write(data[:2] + xmp_header + data[2:]) 164 | elif len(self.xmp_str) > 0: 165 | raise NotImplementedError("Adding information to existing XMP header is currently " 166 | "not supported") 167 | else: 168 | with self._storage.open(self.file_path, "wb") as out_image: 169 | out_image.write(data[:start] + xmp_header + data[start:]) 170 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests>=2.20.0 2 | ExifRead>=2.3.2 3 | tqdm>=4.62.3 4 | piexif>=1.1.3 5 | pycodestyle>=2.8.0 6 | 7 | gpxpy>=1.4.2 8 | geojson~=2.5.0 9 | 10 | requests-oauthlib 11 | 12 | 13 | imagesize~=1.3.0 14 | oauthlib~=3.2.2 -------------------------------------------------------------------------------- /validators.py: -------------------------------------------------------------------------------- 1 | """This file will contain all validators used to validate a sequence""" 2 | 3 | import logging 4 | from typing import cast 5 | 6 | import constants 7 | from common.models import PhotoMetadata, OSCDevice, RecordingType 8 | from io_storage.storage import Local 9 | from parsers.osc_metadata.parser import MetadataParser, metadata_parser 10 | from osc_models import Sequence, Video, Photo 11 | 12 | LOGGER = logging.getLogger('osc_tools.validators') 13 | 14 | 15 | class SequenceValidator: 16 | """This class checks if a Sequence will be accepted on the OSC server as a valid sequence""" 17 | 18 | def __eq__(self, other): 19 | if isinstance(other, SequenceValidator): 20 | return self == other 21 | return False 22 | 23 | def validate(self, sequence: Sequence) -> bool: 24 | """This method returns is a bool. If it returns True the sequence is valid if returns 25 | False the sequence is not valid and it is not usable for OSC servers. 26 | """ 27 | LOGGER.debug(" Validating sequence using %s", str(self.__class__)) 28 | if not sequence.visual_items: 29 | LOGGER.debug(" Sequence at %s will not be uploaded since we did not find " 30 | "any compatible visual data", sequence.path) 31 | return False 32 | 33 | if (not sequence.latitude or not sequence.longitude) and not sequence.online_id: 34 | LOGGER.warning(" WARNING: Sequence at %s will not be uploaded. No " 35 | "GPS info was found.", sequence.path) 36 | return False 37 | 38 | return True 39 | 40 | 41 | class SequenceMetadataValidator(SequenceValidator): 42 | """SequenceMetadataValidator is a SequenceValidator responsible of validating if a sequence 43 | that has metadata""" 44 | 45 | def __eq__(self, other): 46 | if isinstance(other, SequenceMetadataValidator): 47 | return self == other 48 | return False 49 | 50 | def validate(self, sequence: Sequence) -> bool: 51 | """This method returns is a bool, If it returns True the sequence is valid if returns 52 | False the sequence is not valid and it is not usable for OSC servers. 53 | """ 54 | if not super().validate(sequence): 55 | return False 56 | # is sequence has metadata 57 | if sequence.osc_metadata and not sequence.online_id: 58 | metadata_path = sequence.osc_metadata 59 | LOGGER.debug(" Validating Metadata %s", metadata_path) 60 | parser: MetadataParser = metadata_parser(metadata_path, Local()) 61 | photo_item = parser.next_item_with_class(PhotoMetadata) 62 | if not photo_item: 63 | LOGGER.debug(" No photo in metadata") 64 | return False 65 | device: OSCDevice = cast(OSCDevice, parser.next_item_with_class(OSCDevice)) 66 | visual_item = sequence.visual_items[0] 67 | 68 | if device is not None and device.recording_type is not None: 69 | if device.recording_type == RecordingType.VIDEO and not isinstance(visual_item, 70 | Video): 71 | return False 72 | if device.recording_type == RecordingType.PHOTO and not isinstance(visual_item, 73 | Photo): 74 | return False 75 | return True 76 | 77 | 78 | class SequenceFinishedValidator(SequenceValidator): 79 | """SequenceFinishedValidator is a SequenceValidator that is responsible to validate a 80 | finished sequence""" 81 | 82 | def __eq__(self, other): 83 | if isinstance(other, SequenceFinishedValidator): 84 | return self == other 85 | return False 86 | 87 | def validate(self, sequence: Sequence) -> bool: 88 | """this method will return true if a sequence is already uploaded and was flagged 89 | as finished""" 90 | if sequence.progress and \ 91 | constants.UPLOAD_FINISHED in sequence.progress: 92 | return True 93 | return False 94 | -------------------------------------------------------------------------------- /visual_data_discover.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """This script is used to discover video files, or photo files""" 3 | 4 | import os 5 | import logging 6 | from typing import Optional, Tuple, List, cast 7 | 8 | import constants 9 | 10 | from io_storage.storage import Local 11 | from parsers.custom_data_parsers.custom_mapillary import MapillaryExif 12 | from parsers.osc_metadata.parser import metadata_parser 13 | from parsers.exif.exif import ExifParser 14 | from parsers.xmp import XMPParser 15 | from osc_models import VisualData, Photo, Video 16 | from common.models import PhotoMetadata, CameraParameters 17 | 18 | LOGGER = logging.getLogger('osc_tools.visual_data_discoverer') 19 | 20 | 21 | class VisualDataDiscoverer: 22 | """This class is an abstract discoverer of visual data files""" 23 | 24 | @classmethod 25 | def discover(cls, path: str) -> Tuple[List[VisualData], str]: 26 | """This method will discover visual data and will return paths and type""" 27 | 28 | @classmethod 29 | def discover_using_type(cls, path: str, osc_type: str): 30 | """this method is discovering the online visual data knowing the type""" 31 | 32 | 33 | class PhotoDiscovery(VisualDataDiscoverer): 34 | """This class will discover all photo files""" 35 | 36 | @classmethod 37 | def discover(cls, path: str) -> Tuple[List[VisualData], str]: 38 | """This method will discover photos""" 39 | LOGGER.debug("searching for photos %s", path) 40 | if not os.path.isdir(path): 41 | return [], "photo" 42 | 43 | files = os.listdir(path) 44 | photos = [] 45 | for file_path in files: 46 | file_name, file_extension = os.path.splitext(file_path) 47 | if ("jpg" in file_extension.lower() or "jpeg" in file_extension.lower()) and \ 48 | "thumb" not in file_name.lower(): 49 | LOGGER.debug("found a photo: %s", file_path) 50 | photo = cls._photo_from_path(os.path.join(path, file_path)) 51 | if photo: 52 | photos.append(photo) 53 | # Sort photo list 54 | cls._sort_photo_list(photos) 55 | # Add index to the photo objects 56 | index = 0 57 | for photo in photos: 58 | photo.index = index 59 | index += 1 60 | 61 | return cast(List[VisualData], photos), "photo" 62 | 63 | @classmethod 64 | def _photo_from_path(cls, path) -> Optional[Photo]: 65 | photo = Photo(path) 66 | return photo 67 | 68 | @classmethod 69 | def _sort_photo_list(cls, photos): 70 | photos.sort(key=lambda p: int("".join(filter(str.isdigit, os.path.basename(p.path))))) 71 | 72 | 73 | class ExifPhotoDiscoverer(PhotoDiscovery): 74 | """This class will discover all photo files having exif data""" 75 | 76 | @classmethod 77 | def _photo_from_path(cls, path) -> Optional[Photo]: 78 | photo = Photo(path) 79 | exif_parser = ExifParser(path, Local()) 80 | photo_metadata: PhotoMetadata = cast(PhotoMetadata, 81 | exif_parser.next_item_with_class(PhotoMetadata)) 82 | 83 | if photo_metadata is None: 84 | return None 85 | 86 | # required gps timestamp or exif timestamp 87 | photo.gps_timestamp = photo_metadata.gps.timestamp 88 | photo.exif_timestamp = photo_metadata.timestamp 89 | if not photo.gps_timestamp and photo.exif_timestamp: 90 | photo.gps_timestamp = photo.exif_timestamp 91 | 92 | # required latitude and longitude 93 | photo.latitude = photo_metadata.gps.latitude 94 | photo.longitude = photo_metadata.gps.longitude 95 | if not photo.latitude or \ 96 | not photo.longitude or \ 97 | not photo.gps_timestamp: 98 | return None 99 | 100 | # optional data 101 | photo.gps_speed = photo_metadata.gps.speed 102 | photo.gps_altitude = photo_metadata.gps.altitude 103 | photo.gps_compass = photo_metadata.compass.compass 104 | 105 | # pylint: disable=W0703 106 | try: 107 | xmp_parser = XMPParser(path, Local()) 108 | params: CameraParameters = cast(CameraParameters, 109 | xmp_parser.next_item_with_class(CameraParameters)) 110 | if params is not None: 111 | photo.fov = params.h_fov 112 | photo.projection = params.projection 113 | except Exception: 114 | pass 115 | 116 | LOGGER.debug("lat/lon: %f/%f", photo.latitude, photo.longitude) 117 | return photo 118 | 119 | @classmethod 120 | def _sort_photo_list(cls, photos): 121 | photos.sort(key=lambda p: (p.gps_timestamp, os.path.basename(p.path))) 122 | 123 | 124 | class MapillaryExifDiscoverer(ExifPhotoDiscoverer): 125 | @classmethod 126 | def _photo_from_path(cls, path) -> Optional[Photo]: 127 | photo = Photo(path) 128 | exif_parser = MapillaryExif(path, Local()) 129 | photo_metadata: PhotoMetadata = cast(PhotoMetadata, 130 | exif_parser.next_item_with_class(PhotoMetadata)) 131 | 132 | if photo_metadata is None: 133 | return None 134 | 135 | # required gps timestamp or exif timestamp 136 | photo.gps_timestamp = photo_metadata.gps.timestamp 137 | photo.exif_timestamp = photo_metadata.timestamp 138 | if not photo.gps_timestamp and photo.exif_timestamp: 139 | photo.gps_timestamp = photo.exif_timestamp 140 | 141 | # required latitude and longitude 142 | photo.latitude = photo_metadata.gps.latitude 143 | photo.longitude = photo_metadata.gps.longitude 144 | if not photo.latitude or \ 145 | not photo.longitude or \ 146 | not photo.gps_timestamp: 147 | return None 148 | 149 | # optional data 150 | photo.gps_speed = photo_metadata.gps.speed 151 | photo.gps_altitude = photo_metadata.gps.altitude 152 | photo.gps_compass = photo_metadata.compass.compass 153 | 154 | # pylint: disable=W0703 155 | try: 156 | xmp_parser = XMPParser(path, Local()) 157 | params: CameraParameters = cast(CameraParameters, 158 | xmp_parser.next_item_with_class(CameraParameters)) 159 | if params is not None: 160 | photo.fov = params.h_fov 161 | photo.projection = params.projection 162 | except Exception: 163 | pass 164 | 165 | LOGGER.debug("lat/lon: %f/%f", photo.latitude, photo.longitude) 166 | return photo 167 | 168 | 169 | class PhotoMetadataDiscoverer(PhotoDiscovery): 170 | 171 | @classmethod 172 | def discover(cls, path: str): 173 | photos, visual_type = super().discover(path) 174 | metadata_file = os.path.join(path, constants.METADATA_NAME) 175 | if os.path.exists(metadata_file): 176 | parser = metadata_parser(metadata_file, Local()) 177 | parser.start_new_reading() 178 | metadata_photos = cast(List[PhotoMetadata], parser.items_with_class(PhotoMetadata)) 179 | remove_photos = [] 180 | for photo in photos: 181 | if not isinstance(photo, Photo): 182 | continue 183 | for tmp_photo in metadata_photos: 184 | if int(tmp_photo.frame_index) == photo.index: 185 | metadata_photo_to_photo(tmp_photo, photo) 186 | break 187 | if not photo.latitude or not photo.longitude or not photo.gps_timestamp: 188 | remove_photos.append(photo) 189 | return [x for x in photos if x not in remove_photos], visual_type 190 | return [], visual_type 191 | 192 | 193 | def metadata_photo_to_photo(metadata_photo: PhotoMetadata, photo: Photo): 194 | if metadata_photo.gps.latitude: 195 | photo.latitude = float(metadata_photo.gps.latitude) 196 | if metadata_photo.gps.longitude: 197 | photo.longitude = float(metadata_photo.gps.longitude) 198 | if metadata_photo.gps.speed: 199 | photo.gps_speed = round(float(metadata_photo.gps.speed) * 3.6) 200 | if metadata_photo.gps.altitude: 201 | photo.gps_altitude = float(metadata_photo.gps.altitude) 202 | if metadata_photo.frame_index: 203 | photo.index = int(metadata_photo.frame_index) 204 | if metadata_photo.timestamp: 205 | photo.gps_timestamp = float(metadata_photo.timestamp) 206 | 207 | 208 | class VideoDiscoverer(VisualDataDiscoverer): 209 | """This class will discover any sequence having a list of videos""" 210 | 211 | @classmethod 212 | def discover(cls, path: str) -> Tuple[List[VisualData], str]: 213 | if not os.path.isdir(path): 214 | return [], "video" 215 | 216 | files = os.listdir(path) 217 | videos = [] 218 | 219 | for file_path in files: 220 | _, file_extension = os.path.splitext(file_path) 221 | if "mp4" in file_extension: 222 | video = Video(os.path.join(path, file_path)) 223 | videos.append(video) 224 | cls._sort_list(videos) 225 | index = 0 226 | for video in videos: 227 | video.index = index 228 | index += 1 229 | 230 | return videos, "video" 231 | 232 | @classmethod 233 | def _sort_list(cls, videos): 234 | videos.sort(key=lambda v: int("".join(filter(str.isdigit, os.path.basename(v.path))))) 235 | --------------------------------------------------------------------------------