├── LICENSE.md
├── README.md
└── intake.html


/LICENSE.md:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 - present
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # solid-takeout-import
 2 | Import your Google Takeout (etc) data into your solid pod so you can keep it and use it with Solid apps
 3 | 
 4 | Google allow you to take the data you have on Google Plus and similar things
 5 | end export it at the [Google Takeout website](https://takeout.google.com/).  
 6 | 
 7 | You select various sorts of data, then ask for it to be delivered in `.zip` file format or `.tgz` format.  Google then
 8 | does it in the background, then sends you an email with a link to the archive when it is ready to download.
 9 | (You can also ask for it to be stored in various places you control, but not *yet* solid pods.)
10 | 
11 | - You click on the download link on your computer, and get a `.zip` (or `.tgz`) archive file.
12 | 
13 | The goal of this project is to take the file you downloaded and convert it into
14 | an interoperable form for solid apps you may want to use.
15 | 
16 | What you can do on your computer of course is:
17 | 
18 | - Double clicked on the zip file to unwrap the files onto your computer's file system.
19 | - Explore the files on your computer.
20 | 
21 | The future goal is to be able to:
22 | 
23 | - Upload the Zip file onto your solid pod by dragging it onto a folder's green '+' icon
24 | - Run this code to to find your own existing solid assets such as contacts and photos
25 | - Use it to copy from your takeout archive file into your contacts without replication.
26 | - Ideally, if you have already imported the data that copy will be eft, and just new stuff added.
27 | 
28 | Things which we are looking for:
29 | 
30 | - A complete *spec* of Google Takeout, etc  (An LDP graph shape would be ideal)
31 | - The best vcard parser to use
32 | - The best JS ical parser to use
33 | 
34 | 
35 | This as an MIT licensed open source project... contributions are very welcome!  
36 | including taking over the whole project.
37 | 
38 | timbl
39 | 
40 | 
41 | 
42 | ## See also:
43 | 
44 | ### Amazon
45 | - Marc Salzman [Amazon (and Alexa) know a whole lot about you. Here's how download and delete that info](https://www.usatoday.com/story/tech/columnist/saltzman/2018/04/04/amazon-and-alexa-know-whole-lot-you-heres-how-download-and-delete-info/482286002/)
46 | - [Create an Order History Report](https://www.amazon.com/gp/help/customer/display.html?nodeId=200131240)
47 | 
48 | ### Apple
49 | - Tidbits, [How to Download all Your Apple Data](https://tidbits.com/2018/05/31/how-to-download-all-your-apple-data/)
50 | - privacy.apple.com
51 | 
52 | ### Facebook
53 | - [What to look for in your Facebook data - and how to find it](https://www.wired.com/story/download-facebook-data-how-to-read/)
54 | - [Facebook Download your data](https://www.facebook.com/dyi/)
55 | 
56 | ### Google
57 | 
58 | - [Google Takeout](https://takeout.google.com/)
59 | 
60 | ### Microsoft
61 | 
62 | - Todd Haselton, [How to download a copy of everything Microsoft knows about you](https://www.cnbc.com/2018/04/18/how-to-download-a-copy-of-everything-microsoft-knows-about-me.html)
63 | 
64 | ### Twitter
65 | - [How to download your Twitter archive](https://help.twitter.com/en/managing-your-account/how-to-download-your-twitter-archive)
66 | - https://twitter.com/settings/account
67 | - https://twitter.com/settings/your_twitter_data
68 | 


--------------------------------------------------------------------------------
/intake.html:
--------------------------------------------------------------------------------
  1 | <html>
  2 | <head>
  3 | <title>Google Takeout importer</title> <!--
  4 | <script src="https://linkeddata.github.io/mashlib/dist/mashlib.js"></script>
  5 | -->
  6 | <script src="http://localhost:3080/timbl/Automation/Library/Mashup/mashlib.js"></script>
  7 | <script>
  8 | document.addEventListener('DOMContentLoaded', event => {
  9 | 
 10 | // zip file reading https://gildas-lormeau.github.io/zip.js/core-api.html
 11 | 
 12 |   const live = false // live or test mode
 13 | 
 14 |   const dom = document
 15 |   const UI = window.UI
 16 |   const store = UI.store
 17 |   const fetcher = store.fetcher
 18 |   // const load = store.fetcher.load does not work
 19 | 
 20 |   const ns = UI.ns
 21 |   const contains = ns.ldp('contains')
 22 |   const type = ns.rdf('type')
 23 |   console.log('dom loaded')
 24 |   const pre = dom.getElementById('log')
 25 |   const div = dom.getElementById('work')
 26 |   const statusArea = dom.getElementById('statusArea')
 27 |   function log(str) {
 28 |     console.log(str)
 29 |     pre.textContent += str + '\n'
 30 |   }
 31 |   log('hello world')
 32 |   async function importTakeout (root) {
 33 |     function branch(branch, twig) {
 34 |       return $rdf.sym(branch.uri + twig)
 35 |     }
 36 |     function tail(file) { // last part of the path
 37 |       let split = file.uri.split('/') // @@ unencode
 38 |       return decodeURIComponent(split.pop() || split.pop()) // file or folder
 39 |     }
 40 |     function fileIn (folder, tail) {
 41 |       return $rdf.sym(folder.uri + tail)
 42 |     }
 43 |     function happy (response) {
 44 |       if (!response.ok) {
 45 |         let msg = '  HTTP error! Status: ' + response.status + ' for ' + response.url
 46 |         log(msg)
 47 |         response.error = new Error(msg)
 48 |         // statusRow.appendChild(UI.widgets.errorMessageBlock(dom, msg))
 49 |       }
 50 |       return response.ok
 51 |     }
 52 | 
 53 |     /** Convert a JSON like object to RDF
 54 |     */
 55 |     function mapPredicate(p) {
 56 |       return $rdf.sym('http://www.w3.org/ns/pim/json#' + p) // @@ fix Makyby takeout ns?
 57 |     }
 58 |     function jsToRDF (x, newDoc) {
 59 |       //log('     jsToRDF: ' + JSON.stringify(x))
 60 |       if (x instanceof Array) {
 61 |         let y = new $rdf.List()
 62 |         x.forAll(x1 => y.elements.push(jsToRDF(x1, newDoc)))
 63 |         return y
 64 |       } else if (x instanceof Object){
 65 |         let subject = new $rdf.BlankNode()
 66 |         for (let key in x) {
 67 |           let object = jsToRDF(x[key], newDoc)
 68 |           let predicate = mapPredicate(key)
 69 |           log (`         Add {${subject} ${predicate} ${object}} @ ${tail(newDoc)}`)
 70 |           store.add(subject, predicate, object, newDoc)
 71 |         }
 72 |         return subject
 73 |       } else {
 74 |         return $rdf.Literal.fromValue(x)
 75 |       }
 76 |     }
 77 |     /** Convert a JSON file into an RDF file
 78 |     */
 79 |     async function importJSON (doc, newDoc, mapping) {
 80 |       log('Importing JSON doc ' + doc)
 81 |       let response = await fetcher.webOperation('GET', doc.uri)
 82 |       log(' ...  @@ done load ' + doc)
 83 |       if (!happy(response)) return response.error
 84 |       const desc = response.responseText
 85 | 
 86 |       const json = JSON.parse(desc)
 87 |       let subject = $rdf.sym(newDoc.uri + '.ttl#this')
 88 |       log('    regurgitated json:')
 89 |       log(JSON.stringify(json) + '\n')
 90 |       let RDFobject = jsToRDF(json, newDoc)
 91 |       store.add(subject, mapPredicate('data'), RDFobject, newDoc)
 92 |       let text = $rdf.serialize(newDoc, store)
 93 |       log(`      RDF from JSON: doc ${newDoc}\n:text: >>>>${text}<<<<\n`)
 94 |       return newDoc
 95 |     }
 96 |     async function importCSV (doc, namespace) {
 97 |       log('  CSV doc ' + doc)
 98 |       let response = await fetcher.webOperation('GET', doc)
 99 |       if (!happy(response)) return
100 |       const text = response.responseText
101 |       log('   Parsing CSV ' + desc)
102 |       loadCSV(text, store, doc, namespace)
103 |     }
104 | 
105 | 
106 |       //  @@@ parse CSV////////////////////////////////////////
107 | 
108 |       function parseCSV (text, delim) {
109 |       /** Column heading sanatize
110 |         */
111 |         function sanitize (s) {
112 |           return s.replace('\n', ' ')
113 |         }
114 | 
115 |         /** Santize a string can be used as ID
116 |         */
117 |         function sanitizeID (s) {
118 |           const alpha = 'abcdefghijklmnopqrstuvwxyz' + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
119 |           const digits = '0123456789'
120 |           const alphnum = alpha + digits
121 |           let res = ''
122 |           for (let ch of s) {
123 |             if (alphnum.includes(ch)) {
124 |               res += ch
125 |             } else {
126 |                if (res.slice(-1) !== '_') {
127 |                  res += '_' // convert to underscore but not multiple
128 |                }
129 |             }
130 |           }
131 |           while (res.slice(-1) === '_') {
132 |             res = res.slice(0,-1)
133 |           }
134 |           return res
135 |         }
136 | 
137 |         /** Read a comma separated file (or tab sep) text
138 |         * @param {string} delim - typically comma or tab charcater
139 |         */
140 | 
141 |         function readOneRow() { // Was: readTabs
142 |           line = lines.shift() // take first
143 |           while (line !== "" && '\r\n'.includes(line.slice(-1))) {
144 |             line = line.slice(0, -1)
145 |           }
146 |           var fields = []
147 |           var field = ''
148 |           while (true) {
149 |             if (line[0] === '"') { // Quoted string
150 |                 line = line.slice(1) // chop quote
151 |                 while (1) { // Looking for unescaped terminator
152 |                   let j = line.indexOf('"')
153 |                   if (j >= 0) {
154 |                     if (line.slice(j+1, j+2) === '"') {
155 |                       field += line.slice(0,j +1)
156 |                       line = line.slice(j + 2)
157 |                     } else { //
158 |                       field += line.slice(0,j) // we are done
159 |                       line = line.slice(j + 1)
160 |                       break // next field
161 |                     }
162 |                   } else {
163 |                     field += line + '\n'
164 |                     line = lines.shift() // Get more lines as quoted strings can cross them
165 |                   }
166 |                 } // looking fior end of field
167 |             } else { // not quote
168 |               let j = line.indexOf(delim)
169 |               if (j < 0) { // No more fields
170 |                 field = line
171 |               } else {
172 |                 field = line.slice(0, j)
173 |                 line = line.slice(j+1)
174 |               }
175 |             }
176 |             fields.append(field)
177 |           } // feields
178 |           return fields
179 |         }
180 |         delim = delim || ','
181 |         var lines = text.split('\n')
182 |         const headers = readOneRow()
183 |         headers = headers.map(h => {
184 |           return h.trim()
185 |         })
186 |         if (headers.slice(-1)[0] === '') {
187 |           headers = headers.slice(0, -1) // remove empty field from paypal trailing comma
188 |         }
189 |         var lineno = 2
190 |         var table = []
191 |         while (true) {
192 |           let values = readOneRow()
193 |           if (values.length === 0) break
194 |           if (values.length === 1) continue
195 |           if (values .length !== headers.length) {
196 |             console.log(` Line ${lineno}: Warning: ${headers.length} headings but ${values.length} values`)
197 |           }
198 |           table.append(values)
199 |         }
200 | 
201 |         return [headers, values]
202 | 
203 |       } // parse CSV
204 | 
205 |       function loadCSV (text, store, doc, namespace) {
206 |         namespace = namespace || 'http://www.w3.org/ns/pim/csv' // @@@
207 |         var headers, values
208 |         const base = doc.uri
209 |         [ headers, values ] = parseCSV(text, ',')
210 | 
211 |         let labels = headers.map(h => {
212 |           let h = sanitizeID(h)
213 |           return h.slice(0,1).toLowerCase() + h.slice(1) // Initial lower case for predicate
214 |         })
215 | 
216 |         for (let r=0; r < values.length; r++) {
217 |           row = values[r]
218 |           for (let col=0; col < row.length; col++) {
219 |             let subject = store.sym(base + 'n' + r)
220 |             let predicate = store.sym(namespace + label[col])
221 |             let object = row[col] // @@@ guess data types
222 |             store.add(subject, predicate, object, doc)
223 |           }
224 |         }
225 |       }
226 | 
227 |       //  @@@ end parse CSV////////////////////////////////////////
228 | 
229 | 
230 |     async function importGroups (folder) {
231 |       log('  Import Groups:')
232 |       let resp = await fetcher.load(folder)
233 |       if (happy(resp)) {
234 |         store.each(folder,contains).forEach(groupFolder => {
235 |           if (groupFolder.uri.endsWith('/'))  {
236 |             log('  Group folder ' + groupFolder)
237 |             importCSV(fileIn(groupFolder, 'members.csv'))
238 |           }// ignore .DS_Store
239 |         })
240 |       }
241 |     }
242 | 
243 |     function debugTurtle (x) {
244 |       let sts = store.connectedStatements(x)
245 |       let sz = new $rdf.Serializer(store)
246 |       return sz.statementsToN3(sts)
247 |     }
248 | 
249 |     async function importPhotos (folder) {
250 |       log('  Import Photos:')
251 | 
252 |       let resp = await fetcher.load(folder)
253 |       if (!happy(resp)) return resp.error
254 |       let albums = store.each(folder,contains)
255 |       log('albums.length: ' + albums.length)
256 |       for (let i=0; i< albums.length; i++) {
257 |         let photoFolder = albums[i]
258 |         log('  Photo Album ' + photoFolder)
259 |         if (!photoFolder.uri.endsWith('/')) continue
260 |         let resp = await fetcher.load(photoFolder)
261 |         if (!happy(resp)) return response.error
262 | 
263 |         log('    Importing the metadata.json:')
264 |         let newAlbum = $rdf.sym(newPhotoLibrary.dir().uri + 'Albums/' + tail(photoFolder) + '/index.ttl#this')
265 |         let newAlbumDoc = newAlbum.doc()
266 |         let status = await importJSON(fileIn(photoFolder, 'metadata.json'), newAlbumDoc)
267 | 
268 |         store.each(photoFolder,contains).forEach(photo => {
269 |           log('Photo data ' + photo)
270 |           log(debugTurtle(photo))
271 |           let classURIs = store.each(photo, type).map(x => x.uri).join(',')
272 |           if (classURIs.includes('iana/media-types/video') ||
273 |              classURIs.includes('iana/media-types/image')) {
274 |                let newPhotoDoc = $rdf.sym(newAlbumDoc.uri + tail(photo))
275 |                let newPhotoMetaDoc = $rdf.sym(newAlbumDoc.uri + tail(photo) + .ttl)
276 |                log('      Media: '+ photo)
277 |                let mediaMetaDoc = $rdf.sym(photo.uri + '.json')
278 |                let mediaMetaDoc = $rdf.sym(photo.uri + '.json')
279 |                log('      Media: '+ mediaMetaDoc)
280 |                await importJSON(mediaMetaDoc, newPhotoMetaDoc)
281 | 
282 |           } else {
283 |             log('      Non-media: ' + photo)
284 |           }
285 | 
286 |         })
287 |       }
288 |     }
289 | 
290 |     var ABContext = {dom, div, statusArea}
291 |     if (live) {
292 |       ABContext = await UI.authn.findAppInstances(ABContext, ns.vcard('AddressBook'))
293 |     } else {
294 |       ABContext.instances = [ $rdf.sym('http://localhost:3080/timbl/Data/Contacts/index.ttl#this')]
295 |       ABContext.me =  $rdf.sym('https://www.w3.org/People/Berners-Lee/card#i')
296 |     }
297 |     var addressBook
298 |     if (!ABContext.instances.length) {
299 |       log("@@@ Wot no address book")
300 |     } else {
301 |       addressBook = ABContext.instances[0]
302 |       log("Address book " + addressBook)
303 |     }
304 | 
305 |     var photoLibraryContext = {dom, div, statusArea}
306 |     if (live) {
307 |       photoLibraryContext = await UI.authn.findAppInstances(photoLibraryContext, ns.solid('PhotoLibrary'))
308 |     } else {
309 |       photoLibraryContext.instances = [ $rdf.sym('http://localhost:3080/timbl/Photos/index.ttl#this')]
310 |       photoLibraryContext.me =  $rdf.sym('https://www.w3.org/People/Berners-Lee/card#i')
311 |     }
312 |     var newPhotoLibrary = photoLibraryContext.instances[0]
313 | 
314 |     log('loading ' + root)
315 |     let resp = await fetcher.load(root)
316 |     log('loaded ' + root)
317 |     let folders = store.each(root, contains)
318 |     for (let i=0; i < folders.length; i++) {
319 |       let folder = folders[i]
320 |       log(' Folder: ' + folder)
321 |       let name = tail(folder)// @@ unencode
322 |       log(' Looking at ' + name)
323 |       if (name.startsWith('.')) {
324 |         log('  ignoring hidden ' + name)
325 | 
326 |       } else {
327 |         switch(name) {
328 |           case '+1s':
329 |             log(' Import Plus-ones (Bookmarks):')
330 |             // Here need to parse a HTML Netscape bookmark file
331 |             break
332 |           case 'Contacts':
333 |             log('  Import Contacts:')
334 |             break
335 |           case 'Contacts':
336 |             log('  Import Contacts:')
337 |             break
338 |           case 'Google Photos':
339 |             log('  Import Google Photos:')
340 |             await importPhotos(folder)
341 |             break
342 |           case 'Google+ Circles':
343 |             log('  Import Google Photos:')
344 |             break
345 |           case 'Groups':
346 |             let status = await importGroups(folder)
347 |             break
348 | 
349 |           case 'index.html':
350 |             log(' skipping ' + name)
351 |             break
352 |           default:
353 |             log(' @@@@@@@ Unknown Top level Takeout: ' + name)
354 |         }
355 |       }
356 |     }
357 |   }
358 |   const here = $rdf.sym(document.location.href)
359 |   const root = $rdf.sym(here.dir().uri + 'Takeout/')
360 |   importTakeout(root)
361 | })
362 | </script>
363 | </head>
364 | <body>
365 | <h1>Import Takeout</h1>
366 | <p>This HTL file, if you want to uyse it as a tool,
367 |   should be put in the server as your google takeout data.
368 | <div>
369 | 
370 | <pre id='log'>
371 | </pre>
372 | 
373 | <div id='work' style='border-radius: 0.5em; border: solid gray 0.1em; padding: 1em;margin: 0.5em;'>
374 | </div>
375 | 
376 | <div id='statusArea' style='border-radius: 0.5em; border: solid gray 0.1em; padding: 1em;margin: 0.5em;'>
377 | </div>
378 | 
379 | </div>
380 | </body>
381 | </html>
382 | 


--------------------------------------------------------------------------------