├── .gitignore ├── screenshot.png ├── .vscodeignore ├── .vscode ├── extensions.json ├── tasks.json ├── settings.json └── launch.json ├── .eslintrc.json ├── src ├── test │ ├── suite │ │ ├── extension.test.ts │ │ └── index.ts │ └── runTest.ts └── extension.ts ├── CHANGELOG.md ├── tsconfig.json ├── LICENSE ├── README.md ├── vsc-extension-quickstart.md └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | node_modules 3 | .vscode-test/ 4 | *.vsix 5 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsartelle/vscode-web-clipper/HEAD/screenshot.png -------------------------------------------------------------------------------- /.vscodeignore: -------------------------------------------------------------------------------- 1 | .vscode/** 2 | .vscode-test/** 3 | out/test/** 4 | src/** 5 | .gitignore 6 | vsc-extension-quickstart.md 7 | **/tsconfig.json 8 | **/.eslintrc.json 9 | **/*.map 10 | **/*.ts 11 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | // See http://go.microsoft.com/fwlink/?LinkId=827846 3 | // for the documentation about the extensions.json format 4 | "recommendations": [ 5 | "dbaeumer.vscode-eslint" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | // See https://go.microsoft.com/fwlink/?LinkId=733558 2 | // for the documentation about the tasks.json format 3 | { 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "type": "npm", 8 | "script": "watch", 9 | "problemMatcher": "$tsc-watch", 10 | "isBackground": true, 11 | "presentation": { 12 | "reveal": "never" 13 | }, 14 | "group": { 15 | "kind": "build", 16 | "isDefault": true 17 | } 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Place your settings in this file to overwrite default and user settings. 2 | { 3 | "files.exclude": { 4 | "out": false // set this to true to hide the "out" folder with the compiled JS files 5 | }, 6 | "search.exclude": { 7 | "out": true // set this to false to include "out" folder in search results 8 | }, 9 | // Turn off tsc task auto detection since we have the necessary tasks as npm scripts 10 | "typescript.tsc.autoDetect": "off" 11 | } -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "parser": "@typescript-eslint/parser", 4 | "parserOptions": { 5 | "ecmaVersion": 6, 6 | "sourceType": "module" 7 | }, 8 | "plugins": [ 9 | "@typescript-eslint" 10 | ], 11 | "rules": { 12 | "@typescript-eslint/class-name-casing": "warn", 13 | "@typescript-eslint/semi": "warn", 14 | "curly": "warn", 15 | "eqeqeq": "warn", 16 | "no-throw-literal": "warn", 17 | "semi": "off" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/test/suite/extension.test.ts: -------------------------------------------------------------------------------- 1 | import * as assert from 'assert'; 2 | 3 | // You can import and use all API from the 'vscode' module 4 | // as well as import your extension to test it 5 | import * as vscode from 'vscode'; 6 | // import * as myExtension from '../extension'; 7 | 8 | suite('Extension Test Suite', () => { 9 | vscode.window.showInformationMessage('Start all tests.'); 10 | 11 | test('Sample test', () => { 12 | assert.equal(-1, [1, 2, 3].indexOf(5)); 13 | assert.equal(-1, [1, 2, 3].indexOf(0)); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## 1.0.1 - 2019-06-08 8 | - Updated NPM packages to fix a vulnerability in **querystringify** 9 | 10 | ## 1.0.0 - 2019-04-20 11 | ### Added 12 | - Command: `Web Clipper: Clip Web Page` 13 | - URI handler: `vscode://jsartelle.web-clipper/clip?url={URL}` 14 | - Setting: `webClipper.autoShowPreviewToSide` 15 | - Setting: `webClipper.outputTemplate` 16 | - Setting: `webClipper.turndownOptions` -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es6", 5 | "outDir": "out", 6 | "lib": [ 7 | "es6" 8 | ], 9 | "sourceMap": true, 10 | "rootDir": "src", 11 | "strict": true /* enable all strict type-checking options */ 12 | /* Additional Checks */ 13 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 14 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 15 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 16 | }, 17 | "exclude": [ 18 | "node_modules", 19 | ".vscode-test" 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /src/test/runTest.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path'; 2 | 3 | import { runTests } from 'vscode-test'; 4 | 5 | async function main() { 6 | try { 7 | // The folder containing the Extension Manifest package.json 8 | // Passed to `--extensionDevelopmentPath` 9 | const extensionDevelopmentPath = path.resolve(__dirname, '../../'); 10 | 11 | // The path to test runner 12 | // Passed to --extensionTestsPath 13 | const extensionTestsPath = path.resolve(__dirname, './suite/index'); 14 | 15 | // Download VS Code, unzip it and run the integration test 16 | await runTests({ extensionDevelopmentPath, extensionTestsPath }); 17 | } catch (err) { 18 | console.error('Failed to run tests'); 19 | process.exit(1); 20 | } 21 | } 22 | 23 | main(); 24 | -------------------------------------------------------------------------------- /src/test/suite/index.ts: -------------------------------------------------------------------------------- 1 | import * as path from 'path'; 2 | import * as Mocha from 'mocha'; 3 | import * as glob from 'glob'; 4 | 5 | export function run(): Promise { 6 | // Create the mocha test 7 | const mocha = new Mocha({ 8 | ui: 'tdd', 9 | color: true 10 | }); 11 | 12 | const testsRoot = path.resolve(__dirname, '..'); 13 | 14 | return new Promise((c, e) => { 15 | glob('**/**.test.js', { cwd: testsRoot }, (err, files) => { 16 | if (err) { 17 | return e(err); 18 | } 19 | 20 | // Add files to the test suite 21 | files.forEach(f => mocha.addFile(path.resolve(testsRoot, f))); 22 | 23 | try { 24 | // Run the mocha test 25 | mocha.run(failures => { 26 | if (failures > 0) { 27 | e(new Error(`${failures} tests failed.`)); 28 | } else { 29 | c(); 30 | } 31 | }); 32 | } catch (err) { 33 | console.error(err); 34 | e(err); 35 | } 36 | }); 37 | }); 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 James Sartelle 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | // A launch configuration that compiles the extension and then opens it inside a new window 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | { 6 | "version": "0.2.0", 7 | "configurations": [ 8 | { 9 | "name": "Run Extension", 10 | "type": "extensionHost", 11 | "request": "launch", 12 | "runtimeExecutable": "${execPath}", 13 | "args": [ 14 | "--extensionDevelopmentPath=${workspaceFolder}" 15 | ], 16 | "outFiles": [ 17 | "${workspaceFolder}/out/**/*.js" 18 | ], 19 | "preLaunchTask": "${defaultBuildTask}" 20 | }, 21 | { 22 | "name": "Extension Tests", 23 | "type": "extensionHost", 24 | "request": "launch", 25 | "runtimeExecutable": "${execPath}", 26 | "args": [ 27 | "--extensionDevelopmentPath=${workspaceFolder}", 28 | "--extensionTestsPath=${workspaceFolder}/out/test/suite/index" 29 | ], 30 | "outFiles": [ 31 | "${workspaceFolder}/out/test/**/*.js" 32 | ], 33 | "preLaunchTask": "${defaultBuildTask}" 34 | } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Web Clipper 2 | 3 | Clip web pages into Markdown, similar to Evernote or OneNote. 4 | 5 | ## Features 6 | 7 | Clip the contents of a web page into a Markdown document using the `Web Clipper: Clip Web Page` command. 8 | 9 | ![Screenshot](./screenshot.png) 10 | 11 | You can also open VS Code and clip a page using the following URI format: `vscode://jsartelle.web-clipper/clip?url={URL}` 12 | 13 | The extension uses the [Mercury parser](https://github.com/postlight/mercury-parser) to extract the main content from a page. The HTML to Markdown conversion is handled by [Turndown](https://github.com/domchristie/turndown). 14 | 15 | ## Requirements 16 | 17 | None. 18 | 19 | ## Extension Settings 20 | 21 | This extension contributes the following settings: 22 | 23 | * `webClipper.autoShowPreviewToSide`: Automatically open a preview to the side after a page is clipped. 24 | * `webClipper.outputTemplate`: A custom template that the results will be interpolated into. For example, you can add the title and publication date to your clips. See the settings editor for more information. 25 | * `webClipper.turndownOptions`: Options to pass to Turndown. A list of valid options can be found [here](https://github.com/domchristie/turndown#options). Advanced options are not supported. 26 | 27 | ## Known Issues 28 | 29 | None! -------------------------------------------------------------------------------- /vsc-extension-quickstart.md: -------------------------------------------------------------------------------- 1 | # Welcome to your VS Code Extension 2 | 3 | ## What's in the folder 4 | 5 | * This folder contains all of the files necessary for your extension. 6 | * `package.json` - this is the manifest file in which you declare your extension and command. 7 | * The sample plugin registers a command and defines its title and command name. With this information VS Code can show the command in the command palette. It doesn’t yet need to load the plugin. 8 | * `src/extension.ts` - this is the main file where you will provide the implementation of your command. 9 | * The file exports one function, `activate`, which is called the very first time your extension is activated (in this case by executing the command). Inside the `activate` function we call `registerCommand`. 10 | * We pass the function containing the implementation of the command as the second parameter to `registerCommand`. 11 | 12 | ## Get up and running straight away 13 | 14 | * Press `F5` to open a new window with your extension loaded. 15 | * Run your command from the command palette by pressing (`Ctrl+Shift+P` or `Cmd+Shift+P` on Mac) and typing `Hello World`. 16 | * Set breakpoints in your code inside `src/extension.ts` to debug your extension. 17 | * Find output from your extension in the debug console. 18 | 19 | ## Make changes 20 | 21 | * You can relaunch the extension from the debug toolbar after changing code in `src/extension.ts`. 22 | * You can also reload (`Ctrl+R` or `Cmd+R` on Mac) the VS Code window with your extension to load your changes. 23 | 24 | 25 | ## Explore the API 26 | 27 | * You can open the full set of our API when you open the file `node_modules/@types/vscode/index.d.ts`. 28 | 29 | ## Run tests 30 | 31 | * Open the debug viewlet (`Ctrl+Shift+D` or `Cmd+Shift+D` on Mac) and from the launch configuration dropdown pick `Extension Tests`. 32 | * Press `F5` to run the tests in a new window with your extension loaded. 33 | * See the output of the test result in the debug console. 34 | * Make changes to `src/test/suite/extension.test.ts` or create new test files inside the `test/suite` folder. 35 | * The provided test runner will only consider files matching the name pattern `**.test.ts`. 36 | * You can create folders inside the `test` folder to structure your tests any way you want. 37 | 38 | ## Go further 39 | 40 | * Reduce the extension size and improve the startup time by [bundling your extension](https://code.visualstudio.com/api/working-with-extensions/bundling-extension). 41 | * [Publish your extension](https://code.visualstudio.com/api/working-with-extensions/publishing-extension) on the VSCode extension marketplace. 42 | * Automate builds by setting up [Continuous Integration](https://code.visualstudio.com/api/working-with-extensions/continuous-integration). 43 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web-clipper", 3 | "displayName": "Web Clipper", 4 | "description": "Clip web pages into Markdown", 5 | "version": "1.1.0", 6 | "publisher": "jsartelle", 7 | "repository": { 8 | "type": "git", 9 | "url": "https://github.com/jsartelle/vscode-web-clipper" 10 | }, 11 | "engines": { 12 | "vscode": "^1.44.0" 13 | }, 14 | "categories": [ 15 | "Other" 16 | ], 17 | "activationEvents": [ 18 | "onCommand:webClipper.clipWebPage", 19 | "onUri" 20 | ], 21 | "main": "./out/extension.js", 22 | "contributes": { 23 | "commands": [ 24 | { 25 | "command": "webClipper.clipWebPage", 26 | "title": "Web Clipper: Clip Web Page" 27 | } 28 | ], 29 | "configuration": { 30 | "type": "object", 31 | "title": "Web Clipper", 32 | "properties": { 33 | "webClipper.autoShowPreviewToSide": { 34 | "type": "boolean", 35 | "default": true, 36 | "description": "Automatically open a preview to the side after a page is clipped." 37 | }, 38 | "webClipper.outputTemplate": { 39 | "type": "string", 40 | "default": "# ${result.title}\n#### ${result.author ? `by ${result.author}, ` : ''}[${result.domain}](${result.url})${result.date_published ? ` ▪ ${new Date(result.date_published).toLocaleString(undefined, {weekday: 'long', year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: '2-digit'})}` : ''}\n\n${markdown}\n\n---\n\nClipped on ${new Date().toLocaleString(undefined, {weekday: 'long', year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: '2-digit'})}", 41 | "markdownDescription": "A custom template that the results will be interpolated into. Use ${result} for the Mercury result object and ${markdown} for the converted Markdown. \nA list of Mercury result properties can be found [here](https://github.com/postlight/mercury-parser#usage). \nEdit this setting in the JSON settings editor to show line breaks properly." 42 | }, 43 | "webClipper.turndownOptions": { 44 | "type": "object", 45 | "default": {}, 46 | "markdownDescription": "Options to pass to Turndown. A list of valid options can be found [here](https://github.com/domchristie/turndown#options). Advanced options are not supported." 47 | } 48 | } 49 | } 50 | }, 51 | "scripts": { 52 | "vscode:prepublish": "npm run compile", 53 | "compile": "tsc -p ./", 54 | "lint": "eslint src --ext ts", 55 | "watch": "tsc -watch -p ./", 56 | "pretest": "npm run compile && npm run lint", 57 | "test": "node ./out/test/runTest.js" 58 | }, 59 | "devDependencies": { 60 | "@types/vscode": "^1.44.0", 61 | "@types/glob": "^7.1.1", 62 | "@types/mocha": "^7.0.2", 63 | "@types/node": "^13.11.0", 64 | "eslint": "^6.8.0", 65 | "@typescript-eslint/parser": "^2.26.0", 66 | "@typescript-eslint/eslint-plugin": "^2.26.0", 67 | "glob": "^7.1.6", 68 | "mocha": "^7.1.1", 69 | "typescript": "^3.8.3", 70 | "vscode-test": "^1.3.0" 71 | }, 72 | "dependencies": { 73 | "@postlight/mercury-parser": "^2.2.0", 74 | "@types/postlight__mercury-parser": "^2.2.2", 75 | "@types/turndown": "^5.0.0", 76 | "turndown": "^6.0.0" 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/extension.ts: -------------------------------------------------------------------------------- 1 | // The module 'vscode' contains the VS Code extensibility API 2 | // Import the module and reference it with the alias vscode in your code below 3 | import * as Mercury from '@postlight/mercury-parser'; 4 | import TurndownService = require('turndown'); 5 | import { URL, URLSearchParams } from 'url'; 6 | import * as vscode from 'vscode'; 7 | 8 | // this method is called when your extension is activated 9 | // your extension is activated the very first time the command is executed 10 | export function activate(context: vscode.ExtensionContext) { 11 | 12 | // Use the console to output diagnostic information (console.log) and errors (console.error) 13 | // This line of code will only be executed once when your extension is activated 14 | // console.log('Congratulations, your extension "web-clipper" is now active!'); 15 | 16 | // The command has been defined in the package.json file 17 | // Now provide the implementation of the command with registerCommand 18 | // The commandId parameter must match the command field in package.json 19 | let disposable = vscode.commands.registerCommand('webClipper.clipWebPage', async () => { 20 | // Prompt for a URL 21 | const url = await vscode.window.showInputBox({ 22 | prompt: 'Enter a URL to clip', 23 | placeHolder: 'https://en.wikipedia.org/wiki/URL', 24 | validateInput: input => { 25 | try { 26 | new URL(input); 27 | return undefined; 28 | } catch (err) { 29 | return 'Input must be a valid URL (including the scheme and trailing slashes, such as https://)'; 30 | } 31 | } 32 | }); 33 | 34 | if (!url) return; 35 | 36 | clipPageAtUrl(url); 37 | }); 38 | 39 | vscode.window.registerUriHandler({ 40 | handleUri: async uri => { 41 | if (uri.path === '/clip') { 42 | const params = new URLSearchParams(uri.query); 43 | if (params.has('url')) { 44 | clipPageAtUrl(params.get('url') as string); 45 | } else { 46 | vscode.window.showErrorMessage('Please provide a URL in the query string (ex. vscode://jsartelle.web-clipper/clip?url=URL).'); 47 | } 48 | } 49 | } 50 | }); 51 | 52 | context.subscriptions.push(disposable); 53 | } 54 | 55 | // this method is called when your extension is deactivated 56 | export function deactivate() { } 57 | 58 | async function clipPageAtUrl(url: string) { 59 | // The code you place here will be executed every time your command is executed 60 | 61 | const configuration = vscode.workspace.getConfiguration(); 62 | 63 | // Test if the URL is valid 64 | try { 65 | new URL(url); 66 | } catch (err) { 67 | vscode.window.showErrorMessage(`Invalid URL: ${url}`); 68 | return; 69 | } 70 | 71 | vscode.window.withProgress({ 72 | location: vscode.ProgressLocation.Notification, 73 | title: 'Clipping web page...' 74 | }, () => { 75 | return new Promise((resolve, reject) => { 76 | // Use Mercury to get the page and extract the main content 77 | Mercury.parse(url) 78 | .then(async (result: any) => { 79 | // Render article content to Markdown 80 | let markdown = new TurndownService(configuration.get('webClipper.turndownOptions')) 81 | .turndown(result.content); 82 | 83 | // Interpolate the results into the template string 84 | let output; 85 | try { 86 | output = eval('`' + configuration.get('webClipper.outputTemplate') + '`'); 87 | } catch (err) { 88 | vscode.window.showErrorMessage( 89 | 'The webClipper.outputTemplate string appears to be invalid.', 90 | 'Open Settings' 91 | ).then(item => { 92 | if (item === 'Open Settings') { 93 | vscode.commands.executeCommand('workbench.action.openSettingsJson'); 94 | } 95 | }); 96 | reject(); 97 | return; 98 | } 99 | 100 | // Get rid of the progress notification 101 | resolve(); 102 | 103 | // Create and show a new Markdown editor with the article 104 | const document = await vscode.workspace.openTextDocument({ 105 | content: output, 106 | language: 'markdown' 107 | }); 108 | 109 | await vscode.window.showTextDocument(document, { 110 | preview: false 111 | }); 112 | 113 | // Open locked preview to the side automatically 114 | if (configuration.get('webClipper.autoShowPreviewToSide')) { 115 | vscode.commands.executeCommand('markdown.showLockedPreviewToSide'); 116 | } 117 | }) 118 | .catch((err: any) => { 119 | vscode.window.showErrorMessage('Error getting the page.'); 120 | console.error(err); 121 | reject(); 122 | }); 123 | }); 124 | }); 125 | } --------------------------------------------------------------------------------