├── .devcontainer ├── Dockerfile └── devcontainer.json ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.md ├── README.md ├── docs └── get-authentication-cookie.png ├── requirements.txt └── src ├── CaptureDate.py ├── LocationHistory.py └── extract.py /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.140.1/containers/python-3/.devcontainer/base.Dockerfile 2 | 3 | # [Choice] Python version: 3, 3.8, 3.7, 3.6 4 | ARG VARIANT="3" 5 | FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} 6 | 7 | # [Option] Install Node.js 8 | ARG INSTALL_NODE="true" 9 | ARG NODE_VERSION="lts/*" 10 | RUN if [ "${INSTALL_NODE}" = "true" ]; then su vscode -c "source /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi 11 | 12 | # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. 13 | COPY requirements.txt /tmp/pip-tmp/ 14 | RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ 15 | && rm -rf /tmp/pip-tmp 16 | 17 | # [Optional] Uncomment this section to install additional OS packages. 18 | # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 19 | # && apt-get -y install --no-install-recommends 20 | 21 | # [Optional] Uncomment this line to install global node packages. 22 | # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 23 | 24 | 25 | RUN pip3 install mypy 26 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: 2 | // https://github.com/microsoft/vscode-dev-containers/tree/v0.140.1/containers/python-3 3 | { 4 | "name": "Python 3", 5 | "build": { 6 | "dockerfile": "Dockerfile", 7 | "context": "..", 8 | "args": { 9 | // Update 'VARIANT' to pick a Python version: 3, 3.6, 3.7, 3.8 10 | "VARIANT": "3", 11 | // Options 12 | "INSTALL_NODE": "false", 13 | "NODE_VERSION": "lts/*" 14 | } 15 | }, 16 | 17 | // Set *default* container specific settings.json values on container create. 18 | "settings": { 19 | "terminal.integrated.shell.linux": "/bin/bash", 20 | "python.pythonPath": "/usr/local/bin/python", 21 | "python.linting.enabled": true, 22 | "python.linting.pylintEnabled": true, 23 | "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", 24 | "python.formatting.blackPath": "/usr/local/py-utils/bin/black", 25 | "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", 26 | "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", 27 | "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", 28 | "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", 29 | "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", 30 | "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", 31 | "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" 32 | }, 33 | 34 | // Add the IDs of extensions you want installed when the container is created. 35 | "extensions": [ 36 | "ms-python.python" 37 | ] 38 | 39 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 40 | // "forwardPorts": [], 41 | 42 | // Use 'postCreateCommand' to run commands after the container is created. 43 | // "postCreateCommand": "pip3 install --user -r requirements.txt", 44 | 45 | // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. 46 | // "remoteUser": "vscode" 47 | } 48 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to `TimelineExtractor` will be documented in this file. 4 | 5 | Updates should follow the [Keep a CHANGELOG](http://keepachangelog.com/) principles. 6 | 7 | ## [Unreleased](https://github.com/Stadly/TimelineExtractor/compare/v1.2.0...HEAD) 8 | 9 | ### Added 10 | - Nothing 11 | 12 | ### Changed 13 | - Nothing 14 | 15 | ### Fixed 16 | - Nothing 17 | 18 | ### Deprecated 19 | - Nothing 20 | 21 | ### Removed 22 | - Nothing 23 | 24 | ### Security 25 | - Nothing 26 | 27 | ## [v1.2.0](https://github.com/Stadly/TimelineExtractor/compare/v1.1.0...v1.2.0) 28 | 29 | ### Added 30 | - Download progress. Thanks @GRWalter! 31 | 32 | ### Fixed 33 | - Authentication to Google Maps. Thanks @GRWalter! 34 | 35 | ## [v1.1.0](https://github.com/Stadly/TimelineExtractor/compare/v1.0.0...v1.1.0) 36 | 37 | ### Added 38 | - Better error handling. 39 | 40 | ## v1.0.0 - 2020-05-08 41 | 42 | ### Added 43 | - Possibility to extract location history for one or more dates. 44 | - Possibility to extract location history for date range. 45 | - Possibility to extract location history for the caputure dates of one or more photos. 46 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-alpine 2 | 3 | RUN apk update && apk upgrade 4 | 5 | COPY . /TimelineExtractor 6 | 7 | WORKDIR /TimelineExtractor/src 8 | 9 | RUN pip install -r ../requirements.txt 10 | 11 | ENTRYPOINT ["python", "extract.py"] 12 | CMD ["-h"] 13 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | 3 | Copyright (c) 2020 Magnar Ovedal Myrtveit 4 | 5 | > Permission is hereby granted, free of charge, to any person obtaining a copy 6 | > of this software and associated documentation files (the "Software"), to deal 7 | > in the Software without restriction, including without limitation the rights 8 | > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | > copies of the Software, and to permit persons to whom the Software is 10 | > furnished to do so, subject to the following conditions: 11 | > 12 | > The above copyright notice and this permission notice shall be included in 13 | > all copies or substantial portions of the Software. 14 | > 15 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | > THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TimelineExtractor 2 | 3 | [![Software License][ico-license]](LICENSE.md) 4 | 5 | Extract location history from Google Maps Timeline. 6 | 7 | ## Introduction 8 | 9 | By enabling Location History in Google Maps, Google will save your location data and processes it in order to create your personal [Timeline](https://www.google.com/maps/timeline). 10 | 11 | You can easily [download the raw data](https://takeout.google.com/) saved by Google; this will put your timeline data in `JSON` files that do not contain the same level of detail as downloading the `kml` files. It is also possible to download a `kml` file of the processed location history for a single day, but it is not possible to download more than one day at a time. 12 | 13 | `TimelineExtractor` lets you easily download your location history. There are multiple options for specifying which dates to download: 14 | - Specify one or more dates. 15 | - Specify a date range. 16 | - Specify one or more photos or directories, to download location history for the capture dates of the photos and contained photos. 17 | 18 | Google Timeline exports your location history using the `kml` format. There are some issues with how Google formats the files, making them incompatible with software such as [GPSBabel](https://www.gpsbabel.org). `TimelineExtractor` takes care of these issues, generating valid `kml` files. 19 | 20 | ## Installation 21 | 22 | Use the following commands to set up `TimelineExtractor`. 23 | 24 | Download `TimelineExtractor`: 25 | 26 | ``` bash 27 | git clone -b v1.2.0 --depth 1 https://github.com/Stadly/TimelineExtractor.git 28 | ``` 29 | 30 | Install dependencies: 31 | 32 | ``` bash 33 | pip install -r TimelineExtractor/requirements.txt 34 | ``` 35 | 36 | Change working directory: 37 | 38 | ``` bash 39 | cd TimelineExtractor/src 40 | ``` 41 | 42 | ### Authentication 43 | 44 | In order to download location history from Google Maps, you must be authenticated. Authentication is done by passing an authentication cookie, authenticated user number, and a reauth proof token to `TimelineExtractor`. 45 | 46 | Follow the steps below to get your authentication cookie from Google Maps Timeline: 47 | 48 | 1. Go to [Timeline](https://www.google.com/maps/timeline) in [Firefox](https://www.mozilla.org/en-US/firefox/) (steps have been tested in Firefox, though other browsers should work similarly). 49 | 2. Open `Developer tools` (`F12`). 50 | 3. In the console, run this command to download the currently selected day's KML file `document.querySelector('.goog-menuitem.export-kml').click();`. 51 | 4. A new tab will briefly open and your KML download will start. 52 | - Depending on your browser settings, you may get a message saying that a pop-up has been prevented from opening; click the message and open the pop-up. 53 | - If you are prompted to save or cancel the download, be sure to save it (otherwise the URL may not be logged in the history and we'll need that). 54 | 5. Open the browser history in Firefox (`CTRL` + `Shift` + `H`) or the download history in Chrome (`CTRL` + `J`). 55 | 6. Copy the most recent URL, which should be something like this: `https://timeline.google.com/maps/timeline/kml?authuser=&pb=%211m8%211m3%211i1990%212i0%213i1%212m3%211i2090%212i0%213i1&pli=1&rapt=`. 56 | 7. With the Developer tools still open, paste that URL into the address bar of your browser. 57 | 8. A new request will appear in the `Network` tab. Click on it. 58 | 9. Details about the request should appear; look for the `Cookie` in the request headers and copy the cookie value. 59 | 10. Save the cookie's content so you can use it to authenticate requests sent by `TimelineExtractor` when downloading location history. It is recommended to store it in a file called `cookie` in the directory `src`, as that will be assumed in most of the examples further down. 60 | 11. Make note of the `authuser` number in the URL. Pass this to the `TimelineExtractor` script with the `-u` or `--authuser` argument. 61 | 12. Copy the value of the `rapt` parameter in the URL. Pass this to the `TimelineExtractor` script with the `-r` or `--rapt` argument. 62 | 63 | Please note that valid cookie and reauth proof tokens change frequently, so these steps may need to be repeated, depending on your usage of `TimelineExtractor`. 64 | 65 | ### Install in Docker container 66 | 67 | It is also possible to set up and use `TimelineExtractor` in a docker container instead of installing it locally. See the section [Using Docker](#using-docker) for details. 68 | 69 | ## Usage 70 | 71 | `TimelineExtractor` is run with the python file `extract.py`: 72 | 73 | ``` 74 | python extract.py 75 | ``` 76 | 77 | ### Authenticate 78 | 79 | To authenticate, specify the following arguments when running `TimelineExtractor`: 80 | - The path to your authentication cookie using the `-c` or `--cookie` argument. 81 | - The authuser number using the `-u` or `--authuser` argument. 82 | - The reauth proof token using the `-r` or `--rapt` argument. 83 | - The output file path using the `-o` or `--output` argument. 84 | 85 | ``` 86 | python extract.py -c path/to/cookie -u 1 -r token_value -o path/to/output/file 87 | ``` 88 | 89 | ### Get location history 90 | 91 | There are three ways to specify which dates to extract location history for: 92 | 1. [Specify one or more dates](#get-location-history-for-one-or-more-dates). 93 | 2. [Specify a date range](#get-location-history-for-a-date-range). 94 | 3. [Specify one or more photos or directories](#get-location-history-for-one-or-more-photos-or-directories), to download location history for the capture dates of the photos and contained photos. 95 | 96 | #### Get location history for one or more dates 97 | 98 | To download location history for a date, simply use the `date` mode and specify the date in `YYYY-MM-DD` format: 99 | 100 | ``` bash 101 | python extract.py -c cookie -u 1 -r token -o output.kml date 2020-01-01 102 | ``` 103 | 104 | If you specify multiple dates, location history will be downloaded for all of them: 105 | 106 | ``` bash 107 | python extract.py -c cookie -u 1 -r token -o output.kml date 2020-01-01 2020-01-05 2020-02-10 108 | ``` 109 | 110 | #### Get location history for a date range 111 | 112 | To download location history for a date range, simply use the `range` mode and specify the first date and last date in `YYYY-MM-DD` format: 113 | 114 | ``` bash 115 | python extract.py -c cookie -u 1 -r token -o output.kml range 2020-01-01 2020-01-31 116 | ``` 117 | 118 | #### Get location history for one or more photos or directories 119 | 120 | To download location history for the capture date of a photo, simply use the `photo` mode and specify the path to the photo: 121 | 122 | ``` bash 123 | python extract.py -c cookie -u 1 -r token -o output.kml photo path/to/photo.jpg 124 | ``` 125 | 126 | If you specify a directory, location history will be downloaded for all the photos in the directory: 127 | 128 | ``` bash 129 | python extract.py -c cookie -u 1 -r token -o output.kml photo path/to/directory 130 | ``` 131 | 132 | If you specify multiple paths, location history will be downloaded for all of them: 133 | 134 | ``` bash 135 | python extract.py -c cookie -u 1 -r token -o output.kml photo path/to/photo.jpg path/to/directory 136 | ``` 137 | 138 | Use the `-s` or `--subdir` argument to download location history also for photos in subdirectories of the specified directories: 139 | 140 | ``` bash 141 | python extract.py -c cookie -u 1 -r token -o output.kml photo -s path/to/directory-tree 142 | ``` 143 | 144 | ### Logging output 145 | 146 | Any logging output generated by `TimelineExtractor` is written to `stderr`. There are five levels of logging: 147 | 148 | 1. debug 149 | 2. info 150 | 3. warning 151 | 4. error 152 | 5. critical 153 | 154 | By default, `info` and higher log messages are output. Use the `-l` or `--log` argument to specify which levels of log messages to output: 155 | 156 | ``` bash 157 | python extract.py -l debug -c cookie date 2020-01-01 158 | ``` 159 | 160 | ### Using Docker 161 | 162 | [Docker](https://www.docker.com) makes setting up and using `TimelineExtractor` really easy. All you have to do is build the docker image, and you can use `TimelineExtractor` without installing any dependencies (even Python!) locally. 163 | 164 | #### Build the docker image 165 | 166 | Build the docker image using the following command. Note that you should [save you authentication cookie](#get-authentication-cookie) before building the docker image so that it becomes part of the image. 167 | 168 | ``` bash 169 | docker build -t extract-timeline . 170 | ``` 171 | 172 | #### Run the docker container 173 | 174 | After the image is built, just run it to use `TimelineExtractor`. The syntax when running `TimelineExtractor` inside the docker container is the same as when running it locally, except that `python extract.py` is replaced by `docker run extract-timeline`. 175 | 176 | For example, the following command will extract location history for the date `2020-01-01` and store it in the file `timeline.kml` in your current working directory: 177 | 178 | ``` bash 179 | docker run extract-timeline -c cookie -u 1 -r token -o timeline.kml date 2020-01-01 180 | ``` 181 | 182 | When extracting location history for photos, the docker container must be able to access to the photos in order to get their capture dates. This is achieved by mounting the directories containing the photos to the docker container. To mount a directory, use the `-v` or `--volume` argument and specify the absolute path of the directory in the local file system, followed by `:` and the absolute path of where it should be accessible in the container. Use the latter paths when specifying the photos and directories to get location history for. 183 | 184 | In the following example, the local directory `/path/to/photos` is mounted to `/photos` in the container. Location history is then calculated for the photo `/photos/my-image.jpg` (refers to `/path/to/photos/my-image.jpg` in the local file system) and the photos contained in `/photos/more-photos` (refers to `/path/to/photos/more-photos` in the local file system). 185 | 186 | ``` bash 187 | docker run -v /path/to/photos:/photos extract-timeline -c cookie -u 1 -r token -o output.kml photo /photos/my-image.jpg /photos/more-photos 188 | ``` 189 | 190 | If you want to mount a directory using a relative path, you can use `$(pwd)` to denote the current working directory: 191 | 192 | ``` bash 193 | docker run -v "$(pwd):/photos" extract-timeline -c cookie -u 1 -r token -o output.kml photo /photos 194 | ``` 195 | 196 | ## Change log 197 | 198 | Please see [CHANGELOG](CHANGELOG.md) for information on what has changed recently. 199 | 200 | ## Security 201 | 202 | If you discover any security related issues, please email magnar@myrtveit.com instead of using the issue tracker. 203 | 204 | ## Credits 205 | 206 | - [Magnar Ovedal Myrtveit][link-author] 207 | - [All contributors][link-contributors] 208 | 209 | ## License 210 | 211 | The MIT License (MIT). Please see [License file](LICENSE.md) for more information. 212 | 213 | [ico-license]: https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square 214 | 215 | [link-author]: https://github.com/Stadly 216 | [link-contributors]: ../../contributors 217 | -------------------------------------------------------------------------------- /docs/get-authentication-cookie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Stadly/TimelineExtractor/50c969340e605f829c7a4ba3aa007a7dddb75776/docs/get-authentication-cookie.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | piexif~=1.1.3 2 | requests~=2.32.3 -------------------------------------------------------------------------------- /src/CaptureDate.py: -------------------------------------------------------------------------------- 1 | import datetime as DT 2 | import logging 3 | import os 4 | import piexif 5 | import re 6 | from typing import List, Optional, Tuple 7 | import xml.etree.ElementTree as ET 8 | 9 | def GetFromPictureFile(FilePath: str) -> Optional[DT.datetime]: 10 | """ 11 | Get capture date for picture file. 12 | """ 13 | try: 14 | Exif = piexif.load(FilePath) 15 | except: 16 | return None 17 | 18 | if 'Exif' not in Exif or 36867 not in Exif['Exif']: 19 | return None 20 | 21 | DateTimeString = Exif['Exif'][36867].decode('utf-8') 22 | return DT.datetime.strptime(DateTimeString, '%Y:%m:%d %H:%M:%S') 23 | 24 | 25 | def GetFromXmpFile(FilePath: str) -> Optional[DT.datetime]: 26 | """ 27 | Get capture date for xmp file. 28 | """ 29 | try: 30 | XmpTree = ET.parse(FilePath) 31 | except: 32 | return None 33 | 34 | XmpNs = { 35 | 'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', 36 | 'exif': 'http://ns.adobe.com/exif/1.0/', 37 | } 38 | 39 | Exif = XmpTree.find('rdf:RDF/rdf:Description', XmpNs) 40 | if Exif is None: 41 | return None 42 | 43 | DateTimeString = Exif.get('{' + XmpNs['exif'] + '}DateTimeOriginal') 44 | if DateTimeString is None: 45 | return None 46 | 47 | if re.match('\\d+-\\d+-\\d+T\\d+:\\d+:\\d+\\.\\d+', DateTimeString): 48 | return DT.datetime.strptime(DateTimeString, '%Y-%m-%dT%H:%M:%S.%f') 49 | 50 | return DT.datetime.fromisoformat(DateTimeString) 51 | 52 | 53 | def GetFromFile(FilePath: str) -> Optional[DT.datetime]: 54 | """ 55 | Get capture date for file. 56 | """ 57 | DateTime = None 58 | FileExt = os.path.splitext(FilePath)[1].lower() 59 | if FileExt in [ 60 | '.cr2', 61 | '.dng', 62 | '.jpg', 63 | '.nef', 64 | '.tif', 65 | ]: 66 | DateTime = GetFromPictureFile(FilePath) 67 | elif FileExt in ['.xmp']: 68 | DateTime = GetFromXmpFile(FilePath) 69 | else: 70 | logging.debug('Skipping file with extension %s: %s', FileExt, FilePath) 71 | return None 72 | 73 | if(DateTime is None): 74 | logging.warning('Could not get capture date from %s.', FilePath) 75 | 76 | return DateTime 77 | 78 | 79 | def GetFromPath(Path: str, VisitSubdirectories: bool = False) -> List[DT.datetime]: 80 | """ 81 | Get list of capture dates for files in path. The path can be either file or directory. 82 | """ 83 | DateTimes = set() 84 | if os.path.isdir(Path): 85 | for File in os.scandir(Path): 86 | if File.is_file(): 87 | DateTime = GetFromFile(File.path) 88 | if DateTime is not None: 89 | DateTimes.add(DateTime) 90 | elif File.is_dir() and VisitSubdirectories: 91 | DateTimes.update(GetFromPath(File.path, VisitSubdirectories)) 92 | elif os.path.isfile(Path): 93 | DateTime = GetFromFile(Path) 94 | if DateTime is not None: 95 | DateTimes.add(DateTime) 96 | else: 97 | logging.warning('%s is not a valid path.', Path) 98 | 99 | return list(DateTimes) 100 | 101 | 102 | def GetMinAndMaxFromPath(Path: str, VisitSubdirectories: bool = False) -> Tuple[Optional[DT.datetime], Optional[DT.datetime]]: 103 | """ 104 | Get minimum and maximum capture date for files in path. The path can be either file or directory. 105 | """ 106 | MinDateTime = None 107 | MaxDateTime = None 108 | if os.path.isdir(Path): 109 | for File in os.scandir(Path): 110 | NextMinDateTime = None 111 | NextMaxDateTime = None 112 | 113 | if File.is_file(): 114 | DateTime = GetFromFile(File.path) 115 | NextMinDateTime = DateTime 116 | NextMaxDateTime = DateTime 117 | elif File.is_dir() and VisitSubdirectories: 118 | NextMinDateTime, NextMaxDateTime = GetMinAndMaxFromPath(File.path, VisitSubdirectories) 119 | 120 | if MinDateTime is None: 121 | MinDateTime = NextMinDateTime 122 | elif NextMinDateTime is not None: 123 | MinDateTime = min(MinDateTime, NextMinDateTime) 124 | 125 | if MaxDateTime is None: 126 | MaxDateTime = NextMaxDateTime 127 | elif NextMaxDateTime is not None: 128 | MaxDateTime = max(MaxDateTime, NextMaxDateTime) 129 | elif os.path.isfile(Path): 130 | DateTime = GetFromFile(Path) 131 | MinDateTime = DateTime 132 | MaxDateTime = DateTime 133 | else: 134 | logging.warning('%s is not a valid path.', Path) 135 | 136 | return MinDateTime, MaxDateTime 137 | -------------------------------------------------------------------------------- /src/LocationHistory.py: -------------------------------------------------------------------------------- 1 | import datetime as DT 2 | import re 3 | import requests 4 | from typing import List 5 | import xml.etree.ElementTree as ET 6 | import logging 7 | 8 | def ElementsAreEqual(Element1: ET.Element, Element2: ET.Element) -> bool: 9 | """ 10 | Check that two elements are indistinguishable from each other. 11 | """ 12 | if Element1.tag != Element2.tag: 13 | return False 14 | if Element1.text != Element2.text: 15 | return False 16 | if Element1.tail != Element2.tail: 17 | return False 18 | if Element1.attrib != Element2.attrib: 19 | return False 20 | if len(Element1) != len(Element2): 21 | return False 22 | return all(ElementsAreEqual(Child1, Child2) for Child1, Child2 in zip(Element1, Element2)) 23 | 24 | 25 | def Merge(LocationHistory1: ET.ElementTree, LocationHistory2: ET.ElementTree) -> ET.ElementTree: 26 | """ 27 | Merge two location histories. 28 | """ 29 | Ns = { 30 | 'ns': 'http://www.opengis.net/kml/2.2', 31 | } 32 | 33 | Document1 = LocationHistory1.find('ns:Document', Ns) 34 | Document2 = LocationHistory2.find('ns:Document', Ns) 35 | if Document1 is None or Document2 is None: 36 | raise Exception('Location history is malformed.') 37 | 38 | Name1 = Document1.find('ns:name', Ns) 39 | Name2 = Document2.find('ns:name', Ns) 40 | if Name1 is None or Name1.text is None or Name2 is None or Name2.text is None: 41 | raise Exception('Location history is malformed.') 42 | 43 | Name1.text += '\n' + Name2.text 44 | 45 | LastPlacermark1 = Document1.find('ns:Placemark[last()]', Ns) 46 | FirstPlacermark2 = Document2.find('ns:Placemark[1]', Ns) 47 | 48 | if LastPlacermark1 is not None and FirstPlacermark2 is not None: 49 | if ElementsAreEqual(LastPlacermark1, FirstPlacermark2): 50 | Document1.remove(LastPlacermark1) 51 | 52 | for Placemark in Document2.findall('ns:Placemark', Ns): 53 | Document1.append(Placemark) 54 | 55 | return LocationHistory1 56 | 57 | 58 | def GetDate(Date: DT.date, AuthCookie: str, AuthUser: int, Rapt: str) -> ET.ElementTree: 59 | """ 60 | Get location history for a date. 61 | """ 62 | Url = 'https://www.google.com/maps/timeline/kml?authuser={3}&pb=!1m8!1m3!1i{0}!2i{1}!3i{2}!2m3!1i{0}!2i{1}!3i{2}&pli=1&rapt={4}'.format(Date.year, Date.month - 1, Date.day, AuthUser, Rapt) 63 | 64 | Response = requests.get(Url, cookies=dict(cookie=AuthCookie)) 65 | if 200 != Response.status_code: 66 | raise Exception('Could not fetch location history.') 67 | 68 | if not Response.text.startswith(' ET.ElementTree: 75 | """ 76 | Get location history for one or more dates. 77 | """ 78 | if not bool(Dates): 79 | raise Exception('You must specify at least one date.') 80 | 81 | SortedDates = sorted(Dates) 82 | TotalDates = len(SortedDates) 83 | CurrentDate = 1 84 | DisplayProgress(SortedDates[0], CurrentDate, TotalDates) 85 | 86 | LocationHistory = GetDate(SortedDates[0], AuthCookie, AuthUser, Rapt) 87 | 88 | for Date in SortedDates[1:]: 89 | CurrentDate += 1 90 | DisplayProgress(Date, CurrentDate, TotalDates) 91 | try: 92 | LocationHistory = Merge(LocationHistory, GetDate(Date, AuthCookie, AuthUser, Rapt)) 93 | except: 94 | logging.error('Location history could not be downloaded. You may have been unauthenicated or a Captcha may be required in your browser. Your output file is being saved with what was downloaded so far.') 95 | return LocationHistory 96 | 97 | return LocationHistory 98 | 99 | 100 | def GetDateRange(StartDate: DT.date, EndDate: DT.date, AuthCookie: str, AuthUser: int, Rapt: str) -> ET.ElementTree: 101 | """ 102 | Get location history for a date range. 103 | """ 104 | if EndDate < StartDate: 105 | raise Exception('Start date cannot be later than end date.') 106 | 107 | TotalDates = (EndDate - StartDate).days 108 | CurrentDate = 1 109 | DisplayProgress(StartDate, CurrentDate, TotalDates) 110 | LocationHistory = GetDate(StartDate, AuthCookie, AuthUser, Rapt) 111 | 112 | for Delta in range(TotalDates): 113 | Date = StartDate + DT.timedelta(Delta + 1) 114 | CurrentDate += 1 115 | DisplayProgress(Date, CurrentDate, TotalDates) 116 | try: 117 | LocationHistory = Merge(LocationHistory, GetDate(Date, AuthCookie, AuthUser, Rapt)) 118 | except: 119 | logging.error('Location history could not be downloaded. You may have been unauthenicated or a Captcha may be required in your browser. Your output file is being saved with what was downloaded so far.') 120 | return LocationHistory 121 | 122 | 123 | return LocationHistory 124 | 125 | 126 | def ReorderLineStringAndTimeSpan(KmlTree: ET.ElementTree) -> None: 127 | """ 128 | Move LineString last so it follows TimeSpan or TimeStamp. 129 | Workaround for bug in Google Location History: https://github.com/gpsbabel/gpsbabel/issues/482 130 | """ 131 | Ns = { 132 | 'ns': 'http://www.opengis.net/kml/2.2', 133 | } 134 | 135 | for Placemark in KmlTree.findall('ns:Document/ns:Placemark', Ns): 136 | LineString = Placemark.find('ns:LineString', Ns) 137 | if LineString is not None: 138 | Placemark.remove(LineString) 139 | Placemark.append(LineString) 140 | 141 | 142 | def ConvertTimeSpanPointToLineString(KmlTree: ET.ElementTree) -> None: 143 | """ 144 | Convert Point combined with TimeSpan to LineString. 145 | Workaround for limitation in GPSBabel: https://github.com/gpsbabel/gpsbabel/issues/484 146 | """ 147 | Ns = { 148 | 'ns': 'http://www.opengis.net/kml/2.2', 149 | } 150 | 151 | for Placemark in KmlTree.findall('ns:Document/ns:Placemark', Ns): 152 | Point = Placemark.find('ns:Point', Ns) 153 | TimeSpan = Placemark.find('ns:TimeSpan', Ns) 154 | if Point is not None and TimeSpan is not None: 155 | Point.tag = '{' + Ns['ns'] + '}LineString' 156 | Coordinates = Point.find('ns:coordinates', Ns) 157 | if Coordinates is not None and Coordinates.text is not None: 158 | Coordinates.text = Coordinates.text + ' ' + Coordinates.text 159 | 160 | 161 | def RemoveErroneousAltitude(KmlTree: ET.ElementTree) -> None: 162 | """ 163 | Remove altitude information. 164 | Google Location History always reports the altitude as 0. 165 | That is obviously wrong, so it is better to have no altitude information. 166 | """ 167 | Ns = { 168 | 'ns': 'http://www.opengis.net/kml/2.2', 169 | } 170 | 171 | CoordinatesRegEx = '(\\d+(?:\\.\\d*)?,\\d+(?:\\.\\d*)?),0' 172 | RegEx = '^(?:' + CoordinatesRegEx + ' +)*' + CoordinatesRegEx + ' *$' 173 | 174 | for Placemark in KmlTree.findall('ns:Document/ns:Placemark', Ns): 175 | Point = Placemark.find('ns:Point', Ns) 176 | if Point is not None: 177 | Coordinates = Point.find('ns:coordinates', Ns) 178 | if Coordinates is not None and Coordinates.text is not None and re.search(RegEx, Coordinates.text): 179 | # All altitudes are 0. Remove them. 180 | Coordinates.text = re.sub(CoordinatesRegEx, '\\1', Coordinates.text) 181 | 182 | LineString = Placemark.find('ns:LineString', Ns) 183 | if LineString is not None: 184 | Coordinates = LineString.find('ns:coordinates', Ns) 185 | if Coordinates is not None and Coordinates.text is not None and re.search(RegEx, Coordinates.text): 186 | # All altitudes are 0. Remove them. 187 | Coordinates.text = re.sub(CoordinatesRegEx, '\\1', Coordinates.text) 188 | 189 | 190 | def DisplayProgress(date: DT.date, current_download: int, total_downloads: int): 191 | """Display Current Progress""" 192 | print(f'Downloading {date} | {current_download}/{total_downloads} | {current_download/total_downloads*100:.2f}%\r', end='', flush=True) 193 | -------------------------------------------------------------------------------- /src/extract.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import datetime as DT 3 | import logging 4 | from typing import List, Optional 5 | import xml.etree.ElementTree as ET 6 | import CaptureDate 7 | import LocationHistory 8 | 9 | def OutputLocationHistory(History: ET.ElementTree, Output: argparse.FileType) -> None: 10 | LocationHistory.RemoveErroneousAltitude(History) 11 | LocationHistory.ConvertTimeSpanPointToLineString(History) 12 | LocationHistory.ReorderLineStringAndTimeSpan(History) 13 | 14 | # Set the default namespace. Workaround for bug in GPSBabel: https://github.com/gpsbabel/gpsbabel/issues/508 15 | ET.register_namespace('', 'http://www.opengis.net/kml/2.2') 16 | with open(Output.name, 'w', encoding='utf-8') as OutputFile: 17 | OutputFile.write('\n') 18 | OutputFile.write(ET.tostring(History.getroot(), encoding='utf-8').decode('utf-8')) 19 | 20 | logging.info(f'Location history has been saved to {Output.name}') 21 | 22 | 23 | def GetLocationHistoryForDates(Dates: List[DT.date], AuthCookie: str, AuthUser: int, Rapt: str) -> ET.ElementTree: 24 | logging.info(f'Calculating location history for {len(Dates)} date(s)') 25 | return LocationHistory.GetDates(Dates, AuthCookie, AuthUser, Rapt) 26 | 27 | 28 | def GetLocationHistoryForDateRange(StartDate: DT.date, EndDate: DT.date, AuthCookie: str, AuthUser: int, Rapt: str) -> ET.ElementTree: 29 | logging.info(f'Calculating location history for {StartDate:%Y-%m-%d} to {EndDate:%Y-%m-%d}') 30 | return LocationHistory.GetDateRange(StartDate, EndDate, AuthCookie, AuthUser, Rapt) 31 | 32 | 33 | def GetLocationHistoryForPaths(Paths: List[str], VisitSubdirectories: bool, AuthCookie: str, AuthUser: int, Rapt: str) -> Optional[ET.ElementTree]: 34 | DateTimes = set() 35 | for Path in Paths: 36 | logging.info(f'Calculating dates for photos in {Path}') 37 | DateTimes.update(CaptureDate.GetFromPath(Path, VisitSubdirectories)) 38 | 39 | if not bool(DateTimes): 40 | logging.warning('No dates found to extract location history for.') 41 | return None 42 | 43 | Dates = list(set(map(lambda d: d.date(), DateTimes))) 44 | return GetLocationHistoryForDates(Dates, AuthCookie, AuthUser, Rapt) 45 | 46 | 47 | def StringToDate(DateString: str) -> DT.date: 48 | return DT.datetime.strptime(DateString, '%Y-%m-%d').date() 49 | 50 | 51 | def main() -> None: 52 | Parser = argparse.ArgumentParser(description='Extract location history from Google.') 53 | Parser.add_argument('-l', '--log', default='info', choices=['debug', 'info', 'warning', 'error', 'critical'], help='Set the logging level.') 54 | Parser.add_argument('-c', '--cookie', required=True, type=open, help='File containing your Google authentication cookie.') 55 | Parser.add_argument('-u', '--authuser', required=True, type=int, help='The authuser number from the Google Maps URL.') 56 | Parser.add_argument('-r', '--rapt', required=True, type=str, help='The "re-auth proof token" (rapt) from a Google Maps KML file download url.') 57 | Parser.add_argument('-o', '--output', required=True, type=argparse.FileType('w'), help='Output file to write the extracted location history.') 58 | 59 | 60 | 61 | Subparsers = Parser.add_subparsers(title='mode', dest='mode', required=True, help='How to specify dates to extract location history for.') 62 | 63 | # Argument parser for date mode. 64 | DateParser = Subparsers.add_parser('date') 65 | DateParser.add_argument('date', nargs='+', type=StringToDate, help='One or more dates to extract location history for. Format: YYYY-MM-DD') 66 | 67 | # Argument parser for date range mode. 68 | RangeParser = Subparsers.add_parser('range') 69 | RangeParser.add_argument('start', type=StringToDate, help='First date to extract location history for. Format: YYYY-MM-DD') 70 | RangeParser.add_argument('end', type=StringToDate, help='Last date to extract location history for. Format: YYYY-MM-DD') 71 | 72 | # Argument parser for directory mode. 73 | PhotoParser = Subparsers.add_parser('photo') 74 | PhotoParser.add_argument('photo', nargs='+', help='One or more photos or directories to extract location history for. History will be extracted for the capture dates of the photos.') 75 | PhotoParser.add_argument('-s', '--subdir', action='store_true', default=False, help='Also consider photos in subdirectories.') 76 | 77 | Args = Parser.parse_args() 78 | 79 | AuthCookie = Args.cookie.read() 80 | 81 | logging.basicConfig(format='%(levelname)s: %(message)s', level=Args.log.upper()) 82 | 83 | History = None 84 | if Args.mode == 'date': 85 | History = GetLocationHistoryForDates(Args.date, AuthCookie, Args.authuser, Args.rapt) 86 | elif Args.mode == 'range': 87 | History = GetLocationHistoryForDateRange(Args.start, Args.end, AuthCookie, Args.authuser, Args.rapt) 88 | elif Args.mode == 'photo': 89 | History = GetLocationHistoryForPaths(Args.photo, Args.subdir, AuthCookie, Args.authuser, Args.rapt) 90 | 91 | if History is None: 92 | logging.error('Location history could not be calculated.') 93 | else: 94 | OutputLocationHistory(History, Args.output) 95 | 96 | if __name__ == '__main__': 97 | main() 98 | --------------------------------------------------------------------------------