├── .gitignore ├── helpers.js ├── index.js ├── package.json ├── rawgit2jsdelivr.js ├── readme.md └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | *.log 4 | cache 5 | -------------------------------------------------------------------------------- /helpers.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const CACHE = process.argv[3] 4 | 5 | const getTTL = (s = 3600) => (typeof CACHE !== 'undefined' ? s : undefined) 6 | 7 | module.exports = { 8 | getTTL 9 | } 10 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict' 4 | 5 | const { request, dom } = require('fs-request-cache') 6 | const cheerio = require('cheerio') 7 | const Parallel = require('async-parallel') 8 | const getUrls = require('get-urls') 9 | 10 | const rawGit2jsDelivr = require('./rawgit2jsdelivr') 11 | const { getTTL } = require('./helpers') 12 | 13 | const [, , USER] = process.argv 14 | if (!USER) throw 'username is required! try `cdp-rawgit-fix username`' 15 | 16 | const HOST = 'rawgit.com' 17 | 18 | const getAllPens = async (page = 1, list = []) => { 19 | console.log('Getting pens list: page', page) 20 | 21 | const data = await request( 22 | `https://codepen.io/${USER}/pens/popular/grid/${page}/?grid_type=list`, 23 | { json: true, ttl: getTTL() } 24 | ) 25 | 26 | const $ = cheerio.load(data.page.html) 27 | const items = $('.item-in-list-view') 28 | const pens = items 29 | .map((index, item) => { 30 | const link = $(item).find('.title a') 31 | const likes = $(item) 32 | .find('.stat-value:last-of-type') 33 | .text() 34 | .trim() 35 | .replace(/[\.\,]/g, '') 36 | 37 | return { 38 | href: link.attr('href'), 39 | title: link.text().trim(), 40 | likes: Number(likes) 41 | } 42 | }) 43 | .get() 44 | 45 | const all = [...list, ...(pens || [])] 46 | 47 | if (items.length) { 48 | return getAllPens(page + 1, all) 49 | } 50 | 51 | return all 52 | } 53 | 54 | // run 55 | ;(async () => { 56 | const allPens = await getAllPens() 57 | let curr = 0 58 | 59 | try { 60 | const filtered = (await Parallel.map( 61 | allPens, 62 | async pen => { 63 | const $ = await dom(pen.href, { ttl: getTTL() }) 64 | console.log(`📝 Got pen (${++curr} of ${allPens.length}):`, pen.title) 65 | 66 | const enc = $('#init-data').val() 67 | const json = JSON.parse(enc) 68 | const { html, css, js, head, resources } = JSON.parse(json.__item) 69 | 70 | const allUrls = [ 71 | ...resources.map(r => r.url.toLowerCase()), 72 | ...getUrls(html || ''), 73 | ...getUrls(css || ''), 74 | ...getUrls(js || ''), 75 | ...getUrls(head || '') 76 | ] 77 | 78 | return { ...pen, allUrls } 79 | }, 80 | 5 81 | )).filter(pen => pen.allUrls.some(url => url.toLowerCase().includes(HOST))) 82 | 83 | console.log('✅ Got all pens!') 84 | 85 | console.log(filtered.length, 'pens found with rawgit links') 86 | console.log(filtered.map(pen => pen.href).join('\n')) 87 | console.log('==') 88 | console.log('getting working links from jsdelivr...') 89 | 90 | await Parallel.map( 91 | filtered, 92 | async pen => { 93 | let newUrls = [] 94 | const rawgitUrls = pen.allUrls.filter(url => 95 | url.toLowerCase().includes(HOST) 96 | ) 97 | 98 | for (const url of rawgitUrls) { 99 | try { 100 | const newUrl = await rawGit2jsDelivr(url) 101 | 102 | console.log('✅ URL fix found for', pen.title) 103 | console.log('\tPen url: ', pen.href) 104 | console.log('\tOld lib url: ', url) 105 | console.log('\tNew lib url: ', newUrl) 106 | newUrls.push(newUrl) 107 | } catch (err) { 108 | console.error(err) 109 | } 110 | } 111 | 112 | return { ...pen, newUrls } 113 | }, 114 | 1 115 | ) 116 | } catch (err) { 117 | console.log(err.message) 118 | for (const item of err.list) { 119 | console.log(item.message) 120 | } 121 | } 122 | })() 123 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdp-rawgit-fix", 3 | "version": "1.0.5", 4 | "main": "index.js", 5 | "license": "MIT", 6 | "dependencies": { 7 | "async-parallel": "^1.2.3", 8 | "cheerio": "^1.0.0-rc.2", 9 | "fs-request-cache": "^1.0.2", 10 | "get-urls": "^8.0.0" 11 | }, 12 | "repository": "mallendeo/cdp-rawgit-fix", 13 | "author": { 14 | "name": "Mauricio Allende", 15 | "email": "mallendeo@outlook.com", 16 | "url": "https://mallendeo.com" 17 | }, 18 | "engines": { 19 | "node": ">=9" 20 | }, 21 | "files": [ 22 | "index.js", 23 | "helpers.js", 24 | "rawgit2jsdelivr.js" 25 | ], 26 | "bin": { 27 | "cdp-rawgit-fix": "./index.js" 28 | }, 29 | "keywords": [ 30 | "codepen", 31 | "cdn", 32 | "fix", 33 | "rawgit", 34 | "jsdelivr" 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /rawgit2jsdelivr.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { request } = require('fs-request-cache') 4 | const { getTTL } = require('./helpers') 5 | 6 | // from https://www.jsdelivr.com/js/app.js 7 | 8 | const buildJsDelivrLink = (user, repo, version, path) => 9 | `https://cdn.jsdelivr.net/gh/${user}/${repo}@${version}/${path}` 10 | 11 | const isBranch = async (user, repo, branch) => { 12 | try { 13 | const res = await request( 14 | `https://api.github.com/repos/${user}/${repo}/branches/${branch}`, 15 | { json: true, ttl: getTTL() } 16 | ) 17 | 18 | return res.commit.sha 19 | } catch (e) { 20 | return false 21 | } 22 | } 23 | 24 | const isCommitPrefixOrTag = async (user, repo, sha) => { 25 | try { 26 | const res = await request( 27 | `https://api.github.com/repos/${user}/${repo}/commits/${sha}`, 28 | { json: true, ttl: getTTL() } 29 | ).catch(() => false) 30 | return res.sha.indexOf(sha) === 0 ? res.sha : sha 31 | } catch (e) { 32 | return false 33 | } 34 | } 35 | 36 | module.exports = async rawGit => { 37 | const pattern = /^https?:\/\/(?:cdn\.)?rawgit\.com\/([^/@]+)\/([^/@]+)\/([^/@]+)\/(.*)$/i 38 | const commitPattern = /^[0-9a-f]{40}$/ 39 | const match = pattern.exec(rawGit) 40 | 41 | if (match) { 42 | const [, user, repo, version, file] = [...match] 43 | let sha = void 0 44 | 45 | // gist 46 | if (version === 'raw') { 47 | throw 'Sorry, Github Gists are not supported.' 48 | } 49 | 50 | // full commit hash 51 | if (commitPattern.test(version)) { 52 | return buildJsDelivrLink(user, repo, version, file) 53 | } 54 | 55 | // branch -> convert to commit sha 56 | if ((sha = await isBranch(user, repo, version))) { 57 | return buildJsDelivrLink(user, repo, sha, file) 58 | } 59 | 60 | // tag/commit prefix 61 | if ((sha = await isCommitPrefixOrTag(user, repo, version))) { 62 | return buildJsDelivrLink(user, repo, sha, file) 63 | } 64 | } 65 | 66 | throw `Sorry, this doesn't look like a valid RawGit link :(` 67 | } 68 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # CodePen rawgit fix 2 | 3 | This scans all your pens and tells you if any is using a rawgit asset or library. 4 | Then it tries to find a jsdelivr alternative link. 5 | 6 | *From https://rawgit.com/* 7 | 8 | > GitHub repositories that served content through RawGit within the last month will continue to be served until at least October of 2019. URLs for other repositories are no longer being served. If you're currently using RawGit, please stop using it as soon as you can. 9 | 10 | ## Usage 11 | 12 | Install with `npm i -g cdp-rawgit-fix` or `yarn global add cdp-rawgit-fix` 13 | 14 | ```bash 15 | $ cdp-rawgit-fix your_username 16 | ``` 17 | 18 | e.g 19 | 20 | ```bash 21 | $ cdp-rawgit-fix mallendeo 22 | ``` 23 | 24 | ## License 25 | 26 | MIT -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@types/node@*": 6 | version "10.11.7" 7 | resolved "https://registry.yarnpkg.com/@types/node/-/node-10.11.7.tgz#0e75ca9357d646ca754016ca1d68a127ad7e7300" 8 | 9 | agent-base@^4.1.0: 10 | version "4.2.1" 11 | resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.1.tgz#d89e5999f797875674c07d87f260fc41e83e8ca9" 12 | dependencies: 13 | es6-promisify "^5.0.0" 14 | 15 | app-root-path@^2.1.0: 16 | version "2.1.0" 17 | resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-2.1.0.tgz#98bf6599327ecea199309866e8140368fd2e646a" 18 | 19 | async-parallel@^1.2.3: 20 | version "1.2.3" 21 | resolved "https://registry.yarnpkg.com/async-parallel/-/async-parallel-1.2.3.tgz#0b90550aeffb7a365d8cee881eb0618f656a3450" 22 | 23 | axios-https-proxy-fix@^0.17.1: 24 | version "0.17.1" 25 | resolved "https://registry.yarnpkg.com/axios-https-proxy-fix/-/axios-https-proxy-fix-0.17.1.tgz#0214d065a4b8996d8d9ad3be8800f4b705e729ae" 26 | dependencies: 27 | follow-redirects "^1.2.5" 28 | https-proxy-agent "^2.1.1" 29 | is-buffer "^1.1.5" 30 | 31 | boolbase@~1.0.0: 32 | version "1.0.0" 33 | resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" 34 | 35 | cheerio@^1.0.0-rc.2: 36 | version "1.0.0-rc.2" 37 | resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.2.tgz#4b9f53a81b27e4d5dac31c0ffd0cfa03cc6830db" 38 | dependencies: 39 | css-select "~1.2.0" 40 | dom-serializer "~0.1.0" 41 | entities "~1.1.1" 42 | htmlparser2 "^3.9.1" 43 | lodash "^4.15.0" 44 | parse5 "^3.0.1" 45 | 46 | core-util-is@~1.0.0: 47 | version "1.0.2" 48 | resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" 49 | 50 | css-select@~1.2.0: 51 | version "1.2.0" 52 | resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858" 53 | dependencies: 54 | boolbase "~1.0.0" 55 | css-what "2.1" 56 | domutils "1.5.1" 57 | nth-check "~1.0.1" 58 | 59 | css-what@2.1: 60 | version "2.1.0" 61 | resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.0.tgz#9467d032c38cfaefb9f2d79501253062f87fa1bd" 62 | 63 | debug@=3.1.0: 64 | version "3.1.0" 65 | resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" 66 | dependencies: 67 | ms "2.0.0" 68 | 69 | debug@^3.1.0: 70 | version "3.2.6" 71 | resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" 72 | dependencies: 73 | ms "^2.1.1" 74 | 75 | dom-serializer@0, dom-serializer@~0.1.0: 76 | version "0.1.0" 77 | resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82" 78 | dependencies: 79 | domelementtype "~1.1.1" 80 | entities "~1.1.1" 81 | 82 | domelementtype@1, domelementtype@^1.3.0: 83 | version "1.3.0" 84 | resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.0.tgz#b17aed82e8ab59e52dd9c19b1756e0fc187204c2" 85 | 86 | domelementtype@~1.1.1: 87 | version "1.1.3" 88 | resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.1.3.tgz#bd28773e2642881aec51544924299c5cd822185b" 89 | 90 | domhandler@^2.3.0: 91 | version "2.4.2" 92 | resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.2.tgz#8805097e933d65e85546f726d60f5eb88b44f803" 93 | dependencies: 94 | domelementtype "1" 95 | 96 | domutils@1.5.1: 97 | version "1.5.1" 98 | resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" 99 | dependencies: 100 | dom-serializer "0" 101 | domelementtype "1" 102 | 103 | domutils@^1.5.1: 104 | version "1.7.0" 105 | resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a" 106 | dependencies: 107 | dom-serializer "0" 108 | domelementtype "1" 109 | 110 | entities@^1.1.1, entities@~1.1.1: 111 | version "1.1.1" 112 | resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0" 113 | 114 | es6-promise@^4.0.3: 115 | version "4.2.5" 116 | resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.5.tgz#da6d0d5692efb461e082c14817fe2427d8f5d054" 117 | 118 | es6-promisify@^5.0.0: 119 | version "5.0.0" 120 | resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203" 121 | dependencies: 122 | es6-promise "^4.0.3" 123 | 124 | follow-redirects@^1.2.5: 125 | version "1.5.9" 126 | resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.9.tgz#c9ed9d748b814a39535716e531b9196a845d89c6" 127 | dependencies: 128 | debug "=3.1.0" 129 | 130 | fs-extra@^7.0.0: 131 | version "7.0.0" 132 | resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.0.tgz#8cc3f47ce07ef7b3593a11b9fb245f7e34c041d6" 133 | dependencies: 134 | graceful-fs "^4.1.2" 135 | jsonfile "^4.0.0" 136 | universalify "^0.1.0" 137 | 138 | fs-request-cache@^1.0.2: 139 | version "1.0.2" 140 | resolved "https://registry.yarnpkg.com/fs-request-cache/-/fs-request-cache-1.0.2.tgz#a7d57fd418c9a2f1cbd9df7127a61bfec51c6fcc" 141 | dependencies: 142 | app-root-path "^2.1.0" 143 | axios-https-proxy-fix "^0.17.1" 144 | cheerio "^1.0.0-rc.2" 145 | fs-extra "^7.0.0" 146 | 147 | get-urls@^8.0.0: 148 | version "8.0.0" 149 | resolved "https://registry.yarnpkg.com/get-urls/-/get-urls-8.0.0.tgz#62a0225cf96e2336b57e5041781f015141f81511" 150 | dependencies: 151 | normalize-url "^3.3.0" 152 | url-regex "^4.0.0" 153 | 154 | graceful-fs@^4.1.2, graceful-fs@^4.1.6: 155 | version "4.1.11" 156 | resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" 157 | 158 | htmlparser2@^3.9.1: 159 | version "3.9.2" 160 | resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.9.2.tgz#1bdf87acca0f3f9e53fa4fcceb0f4b4cbb00b338" 161 | dependencies: 162 | domelementtype "^1.3.0" 163 | domhandler "^2.3.0" 164 | domutils "^1.5.1" 165 | entities "^1.1.1" 166 | inherits "^2.0.1" 167 | readable-stream "^2.0.2" 168 | 169 | https-proxy-agent@^2.1.1: 170 | version "2.2.1" 171 | resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0" 172 | dependencies: 173 | agent-base "^4.1.0" 174 | debug "^3.1.0" 175 | 176 | inherits@^2.0.1, inherits@~2.0.3: 177 | version "2.0.3" 178 | resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" 179 | 180 | ip-regex@^1.0.1: 181 | version "1.0.3" 182 | resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-1.0.3.tgz#dc589076f659f419c222039a33316f1c7387effd" 183 | 184 | is-buffer@^1.1.5: 185 | version "1.1.6" 186 | resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" 187 | 188 | isarray@~1.0.0: 189 | version "1.0.0" 190 | resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" 191 | 192 | jsonfile@^4.0.0: 193 | version "4.0.0" 194 | resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" 195 | optionalDependencies: 196 | graceful-fs "^4.1.6" 197 | 198 | lodash@^4.15.0: 199 | version "4.17.11" 200 | resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" 201 | 202 | ms@2.0.0: 203 | version "2.0.0" 204 | resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" 205 | 206 | ms@^2.1.1: 207 | version "2.1.1" 208 | resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" 209 | 210 | normalize-url@^3.3.0: 211 | version "3.3.0" 212 | resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-3.3.0.tgz#b2e1c4dc4f7c6d57743df733a4f5978d18650559" 213 | 214 | nth-check@~1.0.1: 215 | version "1.0.1" 216 | resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.1.tgz#9929acdf628fc2c41098deab82ac580cf149aae4" 217 | dependencies: 218 | boolbase "~1.0.0" 219 | 220 | parse5@^3.0.1: 221 | version "3.0.3" 222 | resolved "https://registry.yarnpkg.com/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c" 223 | dependencies: 224 | "@types/node" "*" 225 | 226 | process-nextick-args@~2.0.0: 227 | version "2.0.0" 228 | resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" 229 | 230 | readable-stream@^2.0.2: 231 | version "2.3.6" 232 | resolved "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" 233 | dependencies: 234 | core-util-is "~1.0.0" 235 | inherits "~2.0.3" 236 | isarray "~1.0.0" 237 | process-nextick-args "~2.0.0" 238 | safe-buffer "~5.1.1" 239 | string_decoder "~1.1.1" 240 | util-deprecate "~1.0.1" 241 | 242 | safe-buffer@~5.1.0, safe-buffer@~5.1.1: 243 | version "5.1.2" 244 | resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" 245 | 246 | string_decoder@~1.1.1: 247 | version "1.1.1" 248 | resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" 249 | dependencies: 250 | safe-buffer "~5.1.0" 251 | 252 | tlds@^1.187.0: 253 | version "1.203.1" 254 | resolved "https://registry.yarnpkg.com/tlds/-/tlds-1.203.1.tgz#4dc9b02f53de3315bc98b80665e13de3edfc1dfc" 255 | 256 | universalify@^0.1.0: 257 | version "0.1.2" 258 | resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" 259 | 260 | url-regex@^4.0.0: 261 | version "4.1.1" 262 | resolved "https://registry.yarnpkg.com/url-regex/-/url-regex-4.1.1.tgz#a5617b22e15e26dac57ce74c3f52088bcdfec995" 263 | dependencies: 264 | ip-regex "^1.0.1" 265 | tlds "^1.187.0" 266 | 267 | util-deprecate@~1.0.1: 268 | version "1.0.2" 269 | resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" 270 | --------------------------------------------------------------------------------