├── .codeclimate.yml ├── .editorconfig ├── .gitignore ├── .npmignore ├── .travis.yml ├── .vscode ├── extensions.json ├── launch.json └── settings.json ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.md ├── README.md ├── _config.yml ├── commitlint.config.js ├── examples ├── mongoose-demo.js └── util-demo.js ├── index.js ├── index.spec.js ├── markFieldsAsPII.js ├── markFieldsAsPII.spec.js ├── package-lock.json ├── package.json └── util ├── ciphers.js ├── ciphers.spec.js ├── convert.js ├── convert.spec.js ├── passwords.js └── passwords.spec.js /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | checks: 3 | method-complexity: 4 | config: 5 | threshold: 15 6 | exclude_patterns: 7 | # Raising similar-code check thresholds does not seem to work, 8 | # making spec files an enormous source of issues despite their 9 | # specific nature. As we can't tweak checks by file pattern, 10 | # we have no choice but to disable tests :-( 11 | # 12 | # Oddly enough, CodeClimate does auto-exclude tests, but hasn't 13 | # got the right pattern for JS to cover our `*.spec.js` files… 14 | - '**/*.spec.js' 15 | - '*.spec.js' 16 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # https://EditorConfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | 10 | [*.{js,md}] 11 | charset = utf-8 12 | indent_style = space 13 | indent_size = 2 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cc-test-reporter 2 | coverage 3 | node_modules 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .codeclimate.yml 2 | .editorconfig 3 | .travis.yml 4 | .vscode 5 | _config.yml 6 | cc-test-reporter 7 | commitlint.config.js 8 | coverage 9 | examples 10 | TODO.md 11 | *.spec.js 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 8 4 | - 9 5 | - 10 6 | before_install: 7 | - npm install --global npm@6 8 | - npm install -g codecov 9 | before_script: 10 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 11 | - chmod +x ./cc-test-reporter 12 | - ./cc-test-reporter before-build 13 | after_script: 14 | - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT 15 | - codecov 16 | cache: 17 | directories: 18 | - node_modules 19 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | // See http://go.microsoft.com/fwlink/?LinkId=827846 3 | // for the documentation about the extensions.json format 4 | "recommendations": [ 5 | // advanced-new-file 6 | "patbenatar.advanced-new-file", 7 | // ECMAScript Quotes Transformer 8 | "vilicvane.es-quotes", 9 | // ESLint 10 | "dbaeumer.vscode-eslint", 11 | // JavaScript (ES6) code snippets 12 | "xabikos.javascriptsnippets", 13 | // markdownlint 14 | "DavidAnson.vscode-markdownlint", 15 | // nbsp-vscode 16 | "possan.nbsp-vscode", 17 | // npm 18 | "eg2.vscode-npm-script", 19 | // Path Intellisense 20 | "christian-kohler.path-intellisense", 21 | // Prettier - JavaScript formatter 22 | "esbenp.prettier-vscode", 23 | // Search node_modules 24 | "jasonnutter.search-node-modules", 25 | // Subword navigation 26 | "ow.vscode-subword-navigation" 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Debug Jest Tests", 9 | "type": "node", 10 | "request": "launch", 11 | "runtimeArgs": [ 12 | "--inspect-brk", 13 | "${workspaceFolder}/node_modules/.bin/jest", 14 | "--runInBand", 15 | "--watch" 16 | ], 17 | "console": "integratedTerminal", 18 | "internalConsoleOptions": "neverOpen", 19 | "port": 9229, 20 | "skipFiles": ["/**"] 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.insertSpaces": true, 3 | "editor.tabSize": 2, 4 | "files.insertFinalNewline": true, 5 | "files.trimTrailingWhitespace": true, 6 | "files.exclude": { 7 | "**/.git": true, 8 | "**/.DS_Store": true, 9 | "**/node_modules": true 10 | }, 11 | "prettier.arrowParens": "always", 12 | "prettier.jsxSingleQuote": true, 13 | "prettier.singleQuote": true, 14 | "prettier.trailingComma": "es5", 15 | "prettier.semi": false, 16 | "editor.formatOnSave": true, 17 | "javascript.format.enable": false 18 | } 19 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | See the GitHub repo’s [Releases page](https://github.com/deliciousinsights/mongoose-pii/releases) for full details. 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | - Using welcoming and inclusive language 18 | - Being respectful of differing viewpoints and experiences 19 | - Gracefully accepting constructive criticism 20 | - Focusing on what is best for the community 21 | - Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | - The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | - Trolling, insulting/derogatory comments, and personal or political attacks 28 | - Public or private harassment 29 | - Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | - Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at contact@delicious-insights.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Mongoose-PII 2 | 3 | We’re thrilled that you want to help! 🎉😍 4 | 5 | Here are a few guidelines to help you get on the right tracks. 6 | 7 | ## A word about our Code of Conduct 8 | 9 | This project uses the [Contributor Covenant](./CODE_OF_CONDUCT.md) code of conduct. Please read it carefully if you’re not already familiar with it. We expect you to behave in accordance with it on all online spaces related to this project. 10 | 11 | ## Check existing issues and discuss 12 | 13 | Before you start spending time on this, it’s best to check that you’re not going to _waste_ your time re-inventing something that is already being worked on (an effort which you could then join), or putting together something that we would end up refusing for various reasons. 14 | 15 | Check out [existing issues](https://github.com/deliciousinsights/mongoose-pii/issues?utf8=%E2%9C%93&q=is%3Aissue) (including closed ones, as such discussions might be resolved already) to verify that your intended contribution is both new and relevant. 16 | 17 | If you can't find anything related, please open an issue and discuss your proposal, so we can help you figure out the best way to go about it for a smooth, successful merge in the project. Feel free to at-mention @tdd in there for extra confidence in our getting promptly notified. 18 | 19 | Issues are the best place to hash out the details and approaches early, before too much code has been committed to it. You can then proceed to coding (or writing docs, or improving other aspects)! 20 | 21 | ## Working on the code 22 | 23 | Getting started is the usual stuff: 24 | 25 | 1. [Fork the project](https://github.com/deliciousinsights/mongoose-pii/fork) 26 | 2. Clone your forked repo on your local machine 27 | 3. `npm install` 28 | 4. Create a well-named branch for your work (`git checkout -b your-branch-name`) 29 | 30 | ## Coding style 31 | 32 | In accordance with the usual “Your House, Your Rules” approach, contributions are expected to honor the codebase’s coding style. To facilitate this, the repo comes with: 33 | 34 | - Detailed VS Code workspace settings (in `.vscode/settings.json`), including whitespace management and [ESLint](https://eslint.org/) / [Prettier](https://prettier.io/) extension settings. 35 | - Recommended VS Code extensions (in `.vscode/extensions.json`), including ESLint and Prettier. **If you use other JS linters / beautifiers, make sure you disable them for this workspace** to avoid conflicts. 36 | - [EditorConfig](https://EditorConfig.org) settings for the larger use-case (in `.editorconfig`) 37 | - A `npm run lint` script you can use to ensure consistency. It’s part of the `npm test` script, which in turn is run through the `pre-commit` and `pre-push` Git hooks. 38 | 39 | ## Testing 40 | 41 | The project files all have full tests. We use [Jest](https://jestjs.io/) for this, and have two npm scripts ready: 42 | 43 | - `npm test` runs both the linter and the tests (`test:core`). Run at commit and push time through Git hooks installed by [Husky](https://github.com/typicode/husky). 44 | - `npm run test:core` does a one-pass; it includes full test coverage reporting. 45 | - `npm run test:watch` runs a developer test watch; this is what you should use when working on your code and tests. 46 | 47 | The test suite uses the amazing [MongoDB Memory Server](https://www.npmjs.com/package/mongodb-memory-server) package, which means you won't need to have a running MongoDB server with a test database to run our tests! How cool is that! 48 | 49 | We use Jest’s built-in `expect()` and matchers, along with its native mocking abilities. If you’re not familiar with them, here are the docs for [matchers](https://jestjs.io/docs/en/expect), [function mocking](https://jestjs.io/docs/en/mock-function-api) and [module mocking](https://jestjs.io/docs/en/manual-mocks). Look at the existing tests for inspiration and guidance. 50 | 51 | Make sure **any new code you send has matching tests**, and that the test suite passes. 52 | 53 | ## Sending a Pull Request 54 | 55 | Once you have something ready to contribute, or if you’re too stuck and need help, send a Pull Request from your fork’s code to our main repository. GitHub makes that easy for you in multiple ways. If you’re new to Pull Requests, check out [their neat docs about it](https://help.github.com/articles/proposing-changes-to-your-work-with-pull-requests/). 56 | 57 | Any Pull Request submitted to main repository will trigger various Checks, including Continuous Integration on Travis and quality auditing on CodeClimate. These may result in failed checks that would prevent regular merging, but don't worry, we'll work on this together. 58 | 59 | We currently achieve [100% test coverage](https://codeclimate.com/github/deliciousinsights/mongoose-pii), and we’d love to keep it that way. If you’re not clear on why your tests do not cover 100% of your code in some ways, ask us about it in your Pull Request’s conversation. 60 | 61 | We currently achieve a [zero-issue, A-level quality grade](https://codeclimate.com/github/deliciousinsights/mongoose-pii) on CodeClimate; it will be tested on your Pull Requests as well, and we'd love for you to address any issues that may be surfaced. If you’re unsure how, ask us and we'll review it with you. 62 | 63 | ## Thanks again! 64 | 65 | 🙏🏻 Thanks a ton for helping out! That's what Open-Source is all about! 🙏🏻 66 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Christophe Porteneuve & Delicious Insights 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mongoose PII Plugin 2 | 3 | [![npm version](https://badge.fury.io/js/mongoose-pii.svg)](https://npmjs.com/package/mongoose-pii) 4 | [![MIT license](https://img.shields.io/github/license/deliciousinsights/mongoose-pii.svg)](https://en.wikipedia.org/wiki/MIT_License) 5 | [![Travis build](https://img.shields.io/travis/deliciousinsights/mongoose-pii.svg)](https://travis-ci.org/deliciousinsights/mongoose-pii) 6 | [![CodeCov Code Coverage score](https://img.shields.io/codecov/c/github/deliciousinsights/mongoose-pii.svg)](https://codecov.io/gh/deliciousinsights/mongoose-pii) 7 | 8 | 9 | ![Dependencies freshness](https://img.shields.io/david/deliciousinsights/mongoose-pii.svg) 10 | [![Greenkeeper badge](https://badges.greenkeeper.io/deliciousinsights/mongoose-pii.svg)](https://greenkeeper.io/) 11 | 12 | [![CodeClimate maintainability score](https://img.shields.io/codeclimate/maintainability/deliciousinsights/mongoose-pii.svg)](https://codeclimate.com/github/deliciousinsights/mongoose-pii) 13 | [![Coding style is StandardJS-based](https://img.shields.io/badge/style-standard-brightgreen.svg)](https://standardjs.com/) 14 | [![Code of Conduct is Contributor Covenant](https://img.shields.io/badge/code%20of%20conduct-contributor%20covenant-brightgreen.svg)](http://contributor-covenant.org/version/1/4/) 15 | 16 | ## TL;DR 17 | 18 | Store your data like your MongoDB database is getting stolen tomorrow, without sacrificing Mongoose comfort. 19 | 20 | ## The slightly longer intro 21 | 22 | Best practices for data storage dictate that: 23 | 24 | 1. **Passwords should be securely hashed**; the typical state of the art right now being BCrypt with a securely-random IV and 10+ rounds (e.g. 210+ iterations) in production. 25 | 2. **PII should be securely ciphered**; typically we'd use AES256. 26 | 27 | These help avoid access to cleartext passwords and compromission of PII (_Personally Identifiable Information_, such as e-mails, names, Social Security Numbers, Driver’s License information, Passport numbers…) by database theft or unauthorized direct access. 28 | 29 | This is all good, but we want to retain the comfort of authenticating, in our code, with cleartext password values that were typed in a form or sent in the API call; we also want to be able to query based on PII fields using cleartext values, or to update them with cleartext values. 30 | 31 | In short, we want secure storage without having to worry about it. 32 | 33 | This plugin does exactly that. 34 | 35 | ## In this document 36 | 37 | 1. [Installing](#installing) 38 | 2. [API](#api) 39 | 3. [Caveats](#caveats) 40 | 4. [Contributing](#contributing) 41 | 5. [License and copyright](#license-and-copyright) 42 | 43 | ## Installing 44 | 45 | If you’re using npm: 46 | 47 | ```bash 48 | npm install mongoose-pii 49 | # or npm install --save mongoose-pii if you're running npm < 5.x 50 | ``` 51 | 52 | With yarn: 53 | 54 | ```bash 55 | yarn add mongoose-pii 56 | ``` 57 | 58 | ## Quick start 59 | 60 | ### First, prep your schemas 61 | 62 | For every schema that has PII, passwords, or both: 63 | 64 | 1. open the file that define your schema 65 | 2. Require the plugin 66 | 3. Register it as a schema plugin, providing relevant field lists and, for ciphering PII, the ciphering key. 67 | 68 | Here’s what it could look like: 69 | 70 | ```js 71 | // 2. Require the plugin 72 | const { markFieldsAsPII } = require('mongoose-pii') 73 | 74 | const userSchema = new Schema({ 75 | address: String, 76 | email: { type: String, required: true, index: true }, 77 | firstName: String, 78 | lastName: String, 79 | password: { type: String, required: true }, 80 | role: String, 81 | }) 82 | 83 | // 3. Register the plugin 84 | userSchema.plugin(markFieldsAsPII, { 85 | fields: ['address', 'email', 'firstName', 'lastName'], 86 | key: process.env.MONGOOSE_PII_KEY, 87 | passwordFields: 'password', 88 | }) 89 | 90 | const User = mongoose.model('User', userSchema) 91 | ``` 92 | 93 | That’s it! Now… 94 | 95 | - **Your PII fields will be automatically ciphered at save and deciphered at load** (so in-memory, they’re cleartext), and you can use cleartext values for queries and updates on them. 96 | - **Your PII fields will be automatically ciphered in query arguments** for finders (e.g. `findOne()`) and updaters (e.g. `updateMany()`, `findOneAndUpdate()`). 97 | - **Your password fields will be automatically hashed** in a secure manner at save. This is a one-way hash, so you’ll never have access to the cleartext again, which is as it should be. To authenticate, use the plugin-provided `authenticate()` static method: 98 | 99 | ```js 100 | const user = await User.authenticate({ 101 | email: 'foo@bar.com', 102 | password: 'secret', 103 | }) 104 | ``` 105 | 106 | In its default mode, this resolves to either `null`, or the first matching `User` document. 107 | 108 | ### Second, convert your existing data 109 | 110 | You’re likely to have a ton of existing, unprotected data in your collections already. However, the moment you register the plugin with your Mongoose schemas, loading data starts to break down because it expected hashed passwords for authentication and ciphered PII in the database! 111 | 112 | It would be way too detrimental to loading performance to check for the ciphered state of data in the raw loaded document (not to mention heuristics are not universal there), so instead, we provide a helper API for you to convert your existing collections once you registered the plugin with the proper options. 113 | 114 | See the `convertDataForModel()` API below for details. 115 | 116 | ### Check out our examples! 117 | 118 | Find more usage examples in the [`examples`](https://github.com/deliciousinsights/mongoose-pii/tree/master/examples) directory. 119 | 120 | ---- 121 | 122 | ## API 123 | 124 | Click on the API names (or press Return when they have focus) to toggle API documentation for them. 125 | 126 |
127 | markFieldsAsPII 128 | 129 | ### `markFieldsAsPII` (the plugin itself) 130 | 131 | This is the core plugin, that you register with any schema you need it for through Mongoose’s `schema.plugin()` API. 132 | 133 | If you want PII ciphering, you’ll need to pass the `fields` and `key` options. If you want password hashing, you’ll need to pass the `passwordsFields` option. You can mix both, naturally. 134 | 135 | Passing no option is an invalid use and will trigger the appropriate exception. 136 | 137 | **Options** 138 | 139 | | Name | Default | Description | 140 | | ---------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 141 | | `fields` | `[]` | A list of PII fields to be ciphered. Can be provided either as an array of field names, or as a `String` listing fields separated by commas and/or whitespace, depending on your personal style and convenience. | 142 | | `key` | none | **Required for PII ciphering**. This is either a `String` or a `Buffer` that contains the ciphering key. The value should **never be stored in code**, especially it should **not be versioned**, and is expected to come from an environment variable. Because we’re using AES-256 for ciphering, **the key needs to be 32-byte long**, hence a 32-character `String` (regardless of its contents, hex or otherwise), or a `Buffer` with 32 bytes. | 143 | | `passwordFields` | `[]` | **Required for password hashing**. A list of password fields to hash; 99% of the time we expect this to just be `'password'` or some such (that is, single-field). The format is identical to `fields`. | 144 | 145 | **Example calls** 146 | 147 | These all assume a required plugin and a Mongoose Schema stored as `schema`, something like: 148 | 149 | ```js 150 | const { markFieldsAsPII } = require('mongoose-pii') 151 | const { Schema } = require('mongoose') 152 | 153 | const schema = new Schema({ 154 | // … 155 | }) 156 | ``` 157 | 158 | Based on this, let’s start with PII only: 159 | 160 | ```js 161 | schema.plugin(markFieldsAsPII, { 162 | fields: ['address', 'city', 'email', 'ssn', 'lastName'], 163 | key: process.env.MONGOOSE_PII_KEY, 164 | }) 165 | ``` 166 | 167 | Password hashing only, using the `String` form for field lists: 168 | 169 | ```js 170 | schema.plugin(markFieldsAsPII, { passwordFields: 'password' }) 171 | ``` 172 | 173 | Mixed use, using only `String` forms: 174 | 175 | ```js 176 | schema.plugin(markFieldsAsPII, { 177 | fields: 'address city email ssn lastName', 178 | key: process.env.MONGOOSE_PII_KEY, 179 | passwordFields: 'password', 180 | }) 181 | ``` 182 |
183 | 184 |
185 | authenticate(query[, options]) 186 | 187 | ### `authenticate(query[, options])` 188 | 189 | > **Important note about password hashing:** When you use password hashing, authenticating cannot be done at the MongoDB query level, because password hashes are intentionally unstable: hashing the same clear-text password multiple times will yield different values every time. 190 | > 191 | > Unlike PII ciphering, that we made intentionally stable, allowing for query-based filtering, we thus need to grab all documents matching the parts of `query` that do not relate to password fields, then check each matching document for password fields match using secure (fixed-time) Bcrypt-aware comparison methods. 192 | 193 | In order to make the API as unobtrusive as possible, we require a single query field; the plugin will distinguish between parts of the query that match your `passwordFields` settings, and the remainder, that will be used as a regular query (possibly ciphered for PII fields it may contain). 194 | 195 | **Beware: this method is asynchronous** and returns a Promise. You can use a `.then()` chain or, better yet, make the call site an `async` function if it isn’t yet, and use a simple `await` on the call. Asynchrony is a given considering this does a database fetch, anyway, but password checking is asynchronous too, FWIW. 196 | 197 | **Options** 198 | 199 | |Name|Default|Description| 200 | |-|-|-| 201 | |`single`|`true`|Whether to return a single Document (or `null` if none is found), or *all matching documents* (with an empty Array if none is found). Defaults to single-document mode, which is expected to be the vast majority of use cases, and makes for convenient truthiness of the result value.| 202 | 203 | **Example call** 204 | 205 | Say `User` is a Mongoose model built based on a schema with a `password` hashed field: 206 | 207 | ```js 208 | async function logIn(req, res) { 209 | try { 210 | const { email, password } = req.body 211 | const user = await User.authenticate({ email, password }) 212 | if (!user) { 213 | req.flash('warning', 'No user matches these credentials') 214 | res.render('sessions/new') 215 | return 216 | } 217 | 218 | req.logIn(user) 219 | req.flash('success', `Welcome back, ${user.firstName}!`) 220 | res.redirect(paths.userDashboard) 221 | } catch (err) { 222 | req.flash('error', `Authentication failed: ${err.message}`) 223 | res.redirect(paths.logIn) 224 | } 225 | } 226 | ``` 227 |
228 | 229 | ### Helper functions 230 | 231 | These functions are used internally by the plugin but we thought you’d like to have them around. They’re accessible as named exports from the module, just like the plugin. 232 | 233 |
234 | checkPassword(clearText, hashed) 235 | 236 | ### `checkPassword(clearText, hashed)` 237 | 238 | Asynchronously checks that a given cleartext matches the provided hash. This is asynchronous because depending on the amount of rounds used for the hash, computing a matching hash from cleartext could take nontrivial time and should therefore be nonblocking. 239 | 240 | This returns a Promise that resolves to a Boolean, indicating whether there is a match or not. 241 | 242 | **Example call** 243 | 244 | ```js 245 | if (await checkPassword('secret', user.password)) { 246 | req.flash('warning', 'Your password is a disgrace to privacy') 247 | } 248 | ``` 249 | 250 |
251 | 252 |
253 | cipher(key, clearText[, options]) 254 | 255 | ### `cipher(key, clearText[, options])` 256 | 257 | Ciphers a clear-text value using the AES-256-CBC algorithm, with the provided key. By default, ciphering will derive its IV (*Initialization Vector*) from the cleartext, ensuring stable ciphers, thereby opening the way for query-level ciphered field filtering. 258 | 259 | Both `key` and `clearText` can be either a `String` or `Buffer`. 260 | 261 | This returns the ciphered value as a Base64-encoded `String`, that includes the IV used. Base64 was preferred over hex-encoding as it is 33% more compact, resulting in less data storage requirements. 262 | 263 | Because we expect only short values to be ciphered (PII data are usually small bits of discrete information, such as address lines, names, e-mails or identification numbers), and because AES-256 remains a pretty fast algorithm, this function remains synchronous. 264 | 265 | **Options** 266 | 267 | |Name|Default|Description| 268 | |-|-|-| 269 | |`deriveIV`|`true`|Whether to produce stable ciphers for a given clear-text value (by deriving the IV off it in a secure way), or to use random IVs, which are slightly more secure but prevent querying ciphered fields. Defaults to stable ciphers.| 270 | 271 | **Example call** 272 | 273 | ```js 274 | const key = 'I say: kickass keys rule supreme' 275 | cipher(key, 'I wish all APIs were this nice') 276 | // => 'urWDOjnc6EeMv3ASdrerGAn9YIZw3gjO7lve2EzBQ7Qz7uq4b8UsEBRsOCUPfHitA=' 277 | ``` 278 |
279 | 280 |
281 | decipher(key, cipherText) 282 | 283 | ### `decipher(key, cipherText)` 284 | 285 | Deciphers a ciphered value using the AES-256-CBC algorithm, with the provided 286 | key. The ciphered text is assumed to have been ciphered with the sister `cipher()` function, hence to contain the IV. 287 | 288 | Both `key` and `cipherText` can be either a `String` or `Buffer`. 289 | 290 | This returns the clear-text value, unless the ciphered text is invalid, which results in an exception being thrown. 291 | 292 | Because we expect only short values to be ciphered (PII data are usually small bits of discrete information, such as address lines, names, e-mails or identification numbers), and because AES-256 remains a pretty fast algorithm, this function remains synchronous. 293 | 294 | **Example call** 295 | 296 | ```js 297 | const key = 'I say: kickass keys rule supreme' 298 | decipher(key, 'urWDOjnc6EeMv3ASdrerGAn9YIZw3gjO7lve2EzBQ7Qz7uq4b8UsEBRsOCUPfHitA=') 299 | // => 'I wish all APIs were this nice' 300 | decipher(key, 'ZdZK5sk5P6BGfQJX9qqvFgBUFhR/OXZtv27LaPeCk7kuGrglgq2BS+jSZU1H34GJs=') 301 | // => 'I wish all APIs were this nice' -- this used a non-derived IV 302 | ``` 303 |
304 | 305 |
306 | hashPassword(clearText[, options]) 307 | 308 | ### `hashPassword(clearText[, options])` 309 | 310 | Hashes a clear-text password using Bcrypt, with an amount of rounds depending on the current environment (production or otherwise). 311 | 312 | > Note: Bcrypt has a rather low (72 bytes) limit on the size of the input it can hash, so this function transparently handles longer inputs for you by turning them into their SHA512 hashes (to preserve entropy as best it can) and using the resulting value as input internally. 313 | 314 | Depending on the `sync` option, synchronously returns the hashed value, or returns a Promise resolving to it, to accomodate all use-cases. 315 | 316 | **Options** 317 | 318 | |Name|Default|Description| 319 | |-|-|-| 320 | |`rounds`|2 or 10|How many Bcrypt rounds (powers of 2 for iteration, so 10 rounds is actually 210 iterations) to use for hashing. We use recommended defaults for production (10) or test/development (2). Still, you can customize it by passing the option.| 321 | |`sync`|`false`|Whether to synchronously or asynchronously do the hashing. Synchronous returns the hash, asynchronous returns a Promise resolving to the hash. Defaults to asynchronous.| 322 | 323 | **Example calls** 324 | 325 | Asynchronously, here in the context of caller code that remains old-school Node callback-based: 326 | 327 | ```js 328 | async function demo(newPass, cb) { 329 | try { 330 | cb(null, await hashPassword(newPass)) 331 | } catch (err) { 332 | cb(err) 333 | } 334 | } 335 | ``` 336 | 337 | Synchronously, in the same context as above, but blocking instead of nonblocking: 338 | 339 | ```js 340 | function demo(newPass, cb) { 341 | try { 342 | cb(null, hashPassword(newPass, { sync: true })) 343 | } catch(err) { 344 | cb(err) 345 | } 346 | } 347 | ``` 348 |
349 | 350 | ### Data migration utility 351 | 352 |
353 | convertDataForModel(Model[, emitter]) 354 | 355 | ### `convertDataForModel(Model[, emitter])` 356 | 357 | In order to facilitate the initial migration of your collections’ raw data, we provide a helper API for you to convert your existing collections once you registered the plugin with the proper options. 358 | 359 | Here’s how to go about it, for a given schema: 360 | 361 | 1. Register the plugin, with all relevant options, on the schema 362 | 2. Write a small script that will establish the underlying connection (if your code doesn't do that automatically on model loading, for instance). 363 | 364 | This returns a promise, so if you’re into `async` / `await` (and you should!), go right ahead. 365 | 366 | As this is likely to be run just once in the terminal, it outputs by default, on `process.stderr`, a simple progress bar (that tops at 100 chars wide but can be narrower if your terminal mandates it). 367 | 368 | If you prefer to control the output, you can pass your own event emitter, as shown in the second example below. 369 | 370 | **Example uses** 371 | 372 | Interactively in the terminal, with a dynamic progress bar: 373 | 374 | ```js 375 | const YourModel = require('./path-to-your-model') 376 | const { convertDataForModel } = require('mongoose-pii/convert') 377 | 378 | convertDataForModel(YourModel) 379 | .then((convertedCount) => console.log(`Converted ${convertedCount} documents`)) 380 | .catch((error) => console.error('Failed during the conversion:', error)) 381 | ``` 382 | 383 | Using our own custom event emitter for reporting: 384 | 385 | ```js 386 | const EventEmitter = require('events') 387 | const YourModel = require('./path-to-your-model') 388 | const { convertDataForModel } = require('mongoose-pii/convert') 389 | 390 | const emitter = new EventEmitter() 391 | // This is fired for every successfully-converted Document (1-n) 392 | emitter.on('docs', (convertedCount) => { /* … */ }) 393 | // This is fired every time the (rounded-down) process completion percentage changes (1-100) 394 | emitter.on('progress', (updatedPercentage) => { /* … */ }) 395 | 396 | convertDataForModel(YourModel, emitter) 397 | .then((convertedCount) => console.log(`Converted ${convertedCount} documents`)) 398 | .catch((error) => console.error('Failed during the conversion:', error)) 399 | ``` 400 | 401 |
402 | 403 | ---- 404 | 405 | ## Caveats 406 | 407 | There are a few things to keep in mind when using this plugin. 408 | 409 | ### You need to cipher `deleteMany()` queries yourself 410 | 411 | Mongoose does not yet provide a `deleteMany` hook, which means we’re not auto-ciphering queries used with `deleteMany()`. If you’re using queries on ciphered fields with it, you currently need to cipher values yourself, using your ciphering key and the helper `cipher()` function we provide (see above), staying in `deriveIV` mode. 412 | 413 | ### We have to mutate many objects you pass Mongoose methods 414 | 415 | Mongoose’s plugin API does not let us return updated objects for documents, queries or update descriptors: all we have to work with are the “original” objects, and we have to mutate these. 416 | 417 | This means you should be super-careful to not inadvertently reuse an object you pass Mongoose that contains ciphered or hashed-password fields, as such objects will likely be mangled by the plugin; you’d end up double-ciphering stuff, possibly yell at double-deciphering attempts, too. 418 | 419 | ### Rotating keys isn’t easy right now 420 | 421 | Security best practices would mandate that you rotate ciphering keys as time passes, to further reduce the risk of compromission. However, using a new key would: 422 | 423 | - invalidate deciphering of existing PII in the database 424 | - incorrectly cipher fields in queries, causing empty results or mismatches 425 | 426 | The usual workaround for this is to work with a *keyring*: a small array of keys, most-recent first, where we use the most-recent for writes and all keys for queries. This is fine for cookie signature scenarios, but it seems to us that this could quickly aggravate queries on the database, and presents challenges in query descriptor mutations or composition to turn otherwise single-value matches into working OR clauses. If anyone can whip up a good benchmark on this, perf-wise, and a working PR with tests, we’d love it! 427 | 428 | ### Be wary of your maximum field sizes 429 | 430 | In MongoDB, maximum field sizes aren’t very useful… Still, sometimes you put some maximum length in there, usually based on well-known data formats, such as SSN’s, driver’s license numbers, phone numbers, etc. Many such fields are PII indeed. 431 | 432 | Do remember that ciphering these fields results in 22 characters of IV prefix plus at least 33% more characters than the original data, due to ciphering and Base64 encoding. Adjust your maximum lengths accordingly, if any. 433 | 434 | ### Avoid case transforms 435 | 436 | We store all our ciphered and hashed data in Base64 format, which is case-sensitive. It's nice to normalize such data as e-mail addresses to lowercase, but this will break deciphering in a very big way, as this essentially corrupts the ciphertext. Make sure you don't have such transforms set on your ciphered or hashed fields. 437 | 438 | ### We need Node 8.6+, unless you Babelize us 439 | 440 | We use modern ECMAScript, including REST/Spread properties (“Object spread”). Although it became an official part of the language in ES2018, it’s been available in Node since v8.6.0. Node 8 is currently (October 2018) the Maintenance LTS version, with Node 10 being the Active LTS, and Node 11 out already. Our `package.json` contains an `engines.node` field requiring `>= 8.6`, in order for npm to display a warning should you install it on a lower version. 441 | 442 | Still, if you absolutely must use a version below it (which means you’re on a version that was, or is imminently going to be, End-Of-Lifed: not a wise choice), you can configure Babel to transpile our source, too. 443 | 444 | We’re soon going to dual-publish (using both our native and transpiled source in the module’s package), but still, you should keep your Node runtimes up-to-date, at least with the latest LTS. There’s a lot to gain with this approach. 445 | 446 | ---- 447 | 448 | ## Contributing 449 | 450 | You want to help? That’s awesome! Check out the details of our [contribution process](./CONTRIBUTING.md) (it’s fairly standard). 451 | 452 | This project is run under the [Contributor Covenant](./CODE_OF_CONDUCT.md): make sure you read its dispositions and agree with it before you start contributing. 453 | 454 | ## License and copyright 455 | 456 | This library is © 2018 Delicious Insights and is MIT licensed. See [LICENSE.md](./LICENSE.md) for details. 457 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-leap-day -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { extends: ['@commitlint/config-conventional'] } 2 | -------------------------------------------------------------------------------- /examples/mongoose-demo.js: -------------------------------------------------------------------------------- 1 | const mongoose = require('mongoose') 2 | const { Schema } = require('mongoose') 3 | const { markFieldsAsPII } = require('../index') 4 | 5 | mongoose.connect('mongodb://localhost:27017/demos', { 6 | connectTimeoutMS: 1000, 7 | useNewUrlParser: true, 8 | }) 9 | 10 | demoScenario() 11 | 12 | const email = 'christophe@delicious-insights.com' 13 | 14 | async function demoFinders(User) { 15 | const id = (await User.find().limit(1))[0].id 16 | 17 | // 7. findById / findOne, including a ciphered query 18 | let fetchedUser = await User.findById(id) 19 | console.log('Fetched user (findById):', fetchedUser.toJSON()) 20 | fetchedUser = await User.where({ firstName: 'Chris' }).findOne() 21 | console.log('Fetched user (findOne, ciphered query):', fetchedUser.toJSON()) 22 | 23 | // 8. multi-fetch using find() 24 | const fetchedUsers = await User.where({ lastName: 'Roberts', email }) 25 | .limit(2) 26 | .find() 27 | console.log( 28 | '2 fetched users (find, ciphered query):', 29 | fetchedUsers.map((u) => u.toJSON()) 30 | ) 31 | // 9. findOneAndDelete, with a ciphered query 32 | const deletedUser = await User.where({ email }).findOneAndDelete() 33 | console.log('Deleted user (ciphered query):', deletedUser) 34 | } 35 | 36 | async function demoInsertions(User) { 37 | const attrs = { 38 | address: '83 av. Philippe-Auguste 75011 Paris', 39 | email, 40 | firstName: 'Christophe', 41 | lastName: 'Porteneuve', 42 | password: 'foobar42', 43 | } 44 | 45 | // 1. Single-step create 46 | const user = await User.create(attrs) 47 | console.log('Created user:', user.toJSON()) 48 | // 2. Two-step init-and-save 49 | const user2 = new User(attrs) 50 | await user2.save() 51 | console.log('Saved user:', user2.toJSON()) 52 | // 3. insertMany 53 | const users = await User.insertMany([attrs, attrs]) 54 | console.log( 55 | 'insertMany users:', 56 | users.map((u) => u.toJSON()) 57 | ) 58 | } 59 | 60 | async function demoScenario() { 61 | const User = await setup() 62 | 63 | try { 64 | await demoInsertions(User) 65 | await demoUpdates(User) 66 | await demoFinders(User) 67 | } finally { 68 | mongoose.disconnect() 69 | } 70 | } 71 | 72 | async function demoUpdates(User) { 73 | const user = await User.findOne() 74 | 75 | // 4. updateOne (update would work too but is deprecated) 76 | await user.updateOne({ firstName: 'Chris' }) 77 | // 5. updateMany with both query and update ciphering 78 | const { nModified, n: nMatched } = await User.updateMany( 79 | { email: user.email }, 80 | { address: '19 rue François Mauriac 92700 Colombes', lastName: 'Roberts' } 81 | ) 82 | console.log( 83 | `updateMany with ciphered queries and updates: ${nMatched} matched, ${nModified} updated` 84 | ) 85 | } 86 | 87 | function prepareModel() { 88 | const schema = new Schema({ 89 | address: String, 90 | email: String, 91 | firstName: String, 92 | lastName: String, 93 | password: String, 94 | }) 95 | schema.plugin(markFieldsAsPII, { 96 | fields: ['email', 'firstName', 'lastName'], 97 | key: '59aad44db330ad2bf34f6730e50c0058', 98 | }) 99 | 100 | return mongoose.model('User', schema) 101 | } 102 | 103 | async function setup() { 104 | const User = prepareModel() 105 | 106 | if (process.stdout.isTTY) { 107 | process.stdout.write('\x1b[2J\x1b[H') 108 | } 109 | await User.deleteMany({}) 110 | 111 | return User 112 | } 113 | -------------------------------------------------------------------------------- /examples/util-demo.js: -------------------------------------------------------------------------------- 1 | const { checkPassword, hashPassword } = require('../util/passwords') 2 | const { cipher, decipher } = require('../util/ciphers') 3 | 4 | console.log('-- PII CIPHERING ------------------------------') 5 | 6 | const key = '59aad44db330ad2bf34f6730e50c0058' 7 | 8 | const clearText = 'hello world this is nice' 9 | const obscured = cipher(key, clearText) 10 | console.log(clearText, '->', obscured) 11 | 12 | const deciphered = decipher(key, obscured) 13 | console.log(obscured, '->', deciphered) 14 | 15 | console.log('-- PASSWORDS ----------------------------------') 16 | 17 | passDemo() 18 | 19 | async function passDemo() { 20 | const hash = await hashPassword(clearText) 21 | console.log(clearText, '->', hash) 22 | 23 | console.log(clearText, '<>', hash, '->', await checkPassword(clearText, hash)) 24 | } 25 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const { checkPassword, hashPassword } = require('./util/passwords') 2 | const { cipher, decipher } = require('./util/ciphers') 3 | const { markFieldsAsPII } = require('./markFieldsAsPII') 4 | 5 | module.exports = { 6 | checkPassword, 7 | cipher, 8 | decipher, 9 | hashPassword, 10 | markFieldsAsPII, 11 | } 12 | -------------------------------------------------------------------------------- /index.spec.js: -------------------------------------------------------------------------------- 1 | describe('Top-level module', () => { 2 | it('should re-export all the useful methods', () => { 3 | const { 4 | checkPassword, 5 | cipher, 6 | decipher, 7 | hashPassword, 8 | markFieldsAsPII, 9 | } = require('./index') 10 | 11 | expect(checkPassword).toBeInstanceOf(Function) 12 | expect(cipher).toBeInstanceOf(Function) 13 | expect(decipher).toBeInstanceOf(Function) 14 | expect(hashPassword).toBeInstanceOf(Function) 15 | expect(markFieldsAsPII).toBeInstanceOf(Function) 16 | }) 17 | }) 18 | -------------------------------------------------------------------------------- /markFieldsAsPII.js: -------------------------------------------------------------------------------- 1 | const { cipher, decipher } = require('./util/ciphers') 2 | const { checkPassword, hashPassword } = require('./util/passwords') 3 | 4 | const settings = new WeakMap() 5 | 6 | const QUERY_METHODS = [ 7 | 'count', 8 | 'countDocuments', 9 | // Mongoose has no deleteMany hooks?! 10 | // estimatedDocumentCount does not accept a filter, so no need… 11 | 'find', 12 | 'findOne', 13 | 'findOneAndDelete', 14 | 'findOneAndRemove', 15 | 'findOneAndUpdate', 16 | 'replaceOne', 17 | 'update', 18 | 'updateOne', 19 | 'updateMany', 20 | ] 21 | 22 | function markFieldsAsPII(schema, { fields, key, passwordFields } = {}) { 23 | fields = normalizeFieldList('fields', fields) 24 | passwordFields = normalizeFieldList('passwordFields', passwordFields) 25 | 26 | if (fields.length === 0 && passwordFields.length === 0) { 27 | throw new Error( 28 | 'Using markFieldsAsPII assumes at least one of `fields` or `passwordFields`' 29 | ) 30 | } 31 | 32 | if (fields.length > 0 && !key) { 33 | throw new Error( 34 | 'Missing required `key` option for ciphering `fields` in markFieldsAsPII' 35 | ) 36 | } 37 | 38 | settings.set(schema, { fields, key, passwordFields }) 39 | 40 | if (fields.length > 0) { 41 | schema.pre('insertMany', cipherDocumentFields) 42 | schema.pre('save', cipherDocumentFields) 43 | schema.post('insertMany', decipherDocumentFields) 44 | schema.post('save', decipherDocumentFields) 45 | schema.post('init', decipherDocumentFields) 46 | 47 | for (const method of QUERY_METHODS) { 48 | schema.pre(method, cipherQueryFields) 49 | } 50 | } 51 | 52 | if (passwordFields.length > 0) { 53 | schema.pre('insertMany', hashDocumentPasswords) 54 | schema.pre('save', hashDocumentPasswords) 55 | schema.statics.authenticate = authenticate 56 | } 57 | } 58 | 59 | // 1. Hook functions 60 | // ----------------- 61 | 62 | // Ciphers document fields pre-insert and pre-save, so they're stored 63 | // ciphered in the database. 64 | function cipherDocumentFields(next, docs) { 65 | const { fields, key } = settings.get(this.schema) 66 | 67 | // If we're on `Model.insertMany`, `this` is a Model and `docs` is an Array. 68 | // Otherwise we're on `Document#save/Model.create`, `docs` is missing and 69 | // `this` is a Document. 70 | if (!Array.isArray(docs)) { 71 | docs = [this] 72 | } 73 | 74 | // Just in case we have the same original descriptor object 75 | // multiple times: only cipher once per instance! 76 | docs = [...new Set(docs)] 77 | 78 | processDocs(docs, { fields, key, mode: 'cipher' }) 79 | next() 80 | } 81 | 82 | // Ciphers query, and possibly update, fields for any 83 | // finder/updater/replacer/counter method that does provide 84 | // a hook (not all of them so far, check out `QUERY_METHODS` 85 | // further above). 86 | // 87 | // Ciphering the query ensures we do a proper match on what is 88 | // actually stored in the database. This is mostly useful for 89 | // equality/inclusion operations, but loses meaning for matching, 90 | // starting/ending and other partial ops. 91 | // 92 | // Ciphering the update ensures that updated/replaced data is 93 | // indeed stored ciphered in the database, like we did 94 | // at first save through the `cipherDocumentFields` hook above. 95 | function cipherQueryFields(next) { 96 | // this is the Query -- we're on finder methods 97 | const { fields, key } = settings.get(this.schema) 98 | 99 | const query = this.getQuery() 100 | processObject(query, { fields, key, mode: 'cipher' }) 101 | 102 | const update = this.getUpdate() 103 | if (update) { 104 | processObject(update, { fields, key, mode: 'cipher' }) 105 | } 106 | 107 | next() 108 | } 109 | 110 | // This third and final hook deciphers document fields post-load, 111 | // so we get cleartext data for fetched documents (through the *post* `init` 112 | // hook), and also for just-created documents that were ciphered pre-save 113 | // (through `save` and `insertMany` *post* hooks). 114 | function decipherDocumentFields(docs) { 115 | // If we're on `Model.insertMany`, `this` is a Model and `docs` is an Array. 116 | // Otherwise we're on `Document#save/Model.create`, `docs` is a single 117 | // Document and is `this` as well. 118 | const { fields, key } = settings.get(this.schema) 119 | 120 | if (!Array.isArray(docs)) { 121 | docs = [docs] 122 | } 123 | 124 | processDocs(docs, { fields, key, mode: 'decipher' }) 125 | } 126 | 127 | // Hashes document password fields pre-insert and pre-save, 128 | // so they're stored hashed in the database. 129 | function hashDocumentPasswords(next, docs) { 130 | const { passwordFields } = settings.get(this.schema) 131 | 132 | // If we're on `Model.insertMany`, `this` is a Model and `docs` is an Array. 133 | // Otherwise we're on `Document#save/Model.create`, `docs` is missing and 134 | // `this` is a Document. 135 | if (!Array.isArray(docs)) { 136 | docs = [this] 137 | } 138 | 139 | // Just in case we have the same original descriptor object 140 | // multiple times: only cipher once per instance! 141 | docs = [...new Set(docs)] 142 | 143 | processDocs(docs, { fields: passwordFields, mode: 'hash' }) 144 | next() 145 | } 146 | 147 | // Schema static methods 148 | // --------------------- 149 | 150 | // A static method added to schemas that define password fields. 151 | // Returns documents that match the query fields (that are not 152 | // password fields) and check out on *all* provided password 153 | // fields. It is expected that password field values be passed 154 | // as clear text; there will usually be just one password field, 155 | // and often just one query field (e-mail or other identifier), 156 | // but this allows any number of both query and password fields 157 | // for matching. 158 | // 159 | // @param `fields` a single descriptor that can mix query fields 160 | // (that will be ciphered if necessary) and password 161 | // fields (that will be securely compared). 162 | // @option `single` if true (default), the method will either 163 | // return the first matching document, or `null`. If 164 | // false, it will always return an array of matching 165 | // documents, potentially empty. 166 | async function authenticate(fields, { single = true } = {}) { 167 | const { passwordFields } = settings.get(this.schema) 168 | 169 | const { query, passwords } = splitAuthenticationFields({ 170 | fields, 171 | passwordFields, 172 | }) 173 | 174 | const result = [] 175 | for (const doc of await this.find(query)) { 176 | const passwordPairs = walkDocumentPasswordFields(doc, passwords) 177 | const allPasswordsChecks = await Promise.all( 178 | passwordPairs.map(([clearText, hashed]) => 179 | checkPassword(clearText, hashed) 180 | ) 181 | ) 182 | if (allPasswordsChecks.every((match) => match)) { 183 | if (single) { 184 | return doc 185 | } 186 | 187 | result.push(doc) 188 | } 189 | } 190 | 191 | return single ? null : result 192 | } 193 | 194 | // An internal-use, exported function that our convert utility 195 | // can use to ensure this plugin was registered on a given model or schema. 196 | function pluginWasUsedOn(modelOrSchema) { 197 | return settings.has(modelOrSchema.schema || modelOrSchema) 198 | } 199 | 200 | // Internal helper functions 201 | // ------------------------- 202 | 203 | // Ciphers a value in a consistent way (same cipher for the same value, which is 204 | // critical for enabling query ciphering). 205 | // 206 | // Buffers are left as-is, but anything other that is not a String is turned into 207 | // one (numbers, dates, regexes, etc.) as underlying crypto ciphering mandates 208 | // either a Buffer or a String. Note that deciphering will not restore the original 209 | // data type, but always yield a String; still, it is anticipated that non-String 210 | // values are less likely to be PII, as most sensitive information is usually strings 211 | // or “patterned numbers” (SSN, CC#, etc.) stored as strings. 212 | function cipherValue(key, value) { 213 | if (!(value instanceof Buffer)) { 214 | value = String(value) 215 | } 216 | return cipher(key, value, { deriveIV: true }) 217 | } 218 | 219 | // Tiny internal helper to escape a text to be inserted in a regexp. 220 | function escapeRegexp(text) { 221 | return text.replace(/[\](){}.?+*]/g, '\\$&') 222 | } 223 | 224 | const REGEX_BCRYPT_HASH = /^\$2a\$\d{2}\$[\w./]{53}$/ 225 | 226 | // Hashes a password value… unless it's a hash already! 227 | function hashValue(value) { 228 | return REGEX_BCRYPT_HASH.test(value) 229 | ? value 230 | : hashPassword(value, { sync: true }) 231 | } 232 | 233 | // Simple field-list option normalization. This way fields can be passed as 234 | // a whitespace- or comma-separated string, or as an Array. 235 | function normalizeFieldList(name, value) { 236 | if (typeof value === 'string') { 237 | value = value.trim().split(/[\s,]+/) 238 | } 239 | value = [...new Set(value || [])].sort() 240 | 241 | return value 242 | } 243 | 244 | // A quick helper to iterate over a series of documents for (de)ciphering. 245 | // All options are forwarded to `processObject`, the actual workhorse. 246 | function processDocs(docs, { fields, key, mode }) { 247 | for (const doc of docs) { 248 | processObject(doc, { fields, key, mode, isDocument: true }) 249 | } 250 | } 251 | 252 | // This is **the core function** for this entire plugin. It is used to cipher 253 | // and decipher, both queries/updates objects and actual documents (that are not 254 | // to be traversed in the same way). 255 | // 256 | // Due to Mongoose plugin limitations, this has to **modify the object in-place**, 257 | // which isn't ideal and yields several caveats, but can't be worked around. 258 | // Therefore this doesn't return anything, it just mutates its `obj` argument. 259 | // 260 | // @param obj (Object) The object or document to be processed. 261 | // @option fields (Array) The list of field paths provided to the plugin. 262 | // @option key (String|Buffer) The ciphering key. 263 | // @option isDocument (Boolean) Whether to traverse `obj` as a query/update object 264 | // (false) or as a Document (true). 265 | // @option mode ('cipher'|'decipher'|'hash') Whether to cipher, decipher or hash values. 266 | // @option prefix (String|null) A path prefix for the current level of recursive 267 | // object traversal. Top-level calls have it `null`, deeper levels 268 | // use the caller’s current path context. 269 | function processObject( 270 | obj, 271 | { fields, key, isDocument = false, mode, prefix = null } 272 | ) { 273 | if (mode !== 'cipher' && mode !== 'decipher' && mode !== 'hash') { 274 | throw new Error(`Unknown processObject mode: ${mode}`) 275 | } 276 | 277 | // Define what object keys to iterate over, depending on whether we’re 278 | // processing a Document or query/update object. 279 | const keyList = produceKeyList(obj, { fields, isDocument, prefix }) 280 | 281 | for (const objKey of keyList) { 282 | // Compute the current field path. Operators (that start with '$') 283 | // do not augment the path. 284 | const fieldPath = 285 | objKey[0] === '$' ? prefix : prefix ? `${prefix}.${objKey}` : objKey 286 | const value = obj[objKey] 287 | if (typeof value === 'object' && value != null) { 288 | // Dive into objects/arrays, recursively. 289 | processObject(value, { fields, key, isDocument, mode, prefix: fieldPath }) 290 | } else if (value != null) { 291 | // Null/undefined values need no processing, for the others, let's process 292 | processValue(obj, { fieldPath, fields, key, mode, objKey, prefix }) 293 | } 294 | } 295 | } 296 | 297 | // Just a split of a second-level nontrivial processing in `processObject`, 298 | // to keep it reasonably simple cognitively. 299 | // 300 | // Let’s see if the current field matches our path list. "Relative" paths 301 | // (simple field names) can be matched regardless of depth, hence the 302 | // two first condition elements. Paths that result in arrays mean all 303 | // items in the array are to be processed. 304 | // 305 | // @see `processObject()` 306 | function processValue(obj, { fieldPath, fields, key, mode, objKey, prefix }) { 307 | const value = obj[objKey] 308 | const parentFieldName = (prefix || '').split('.').slice(-1)[0] 309 | const fieldMatches = 310 | fields.includes(fieldPath) || 311 | fields.includes(objKey) || 312 | (Array.isArray(obj) && 313 | (fields.includes(prefix) || fields.includes(parentFieldName))) 314 | 315 | if (!fieldMatches) { 316 | return 317 | } 318 | 319 | if (mode === 'decipher') { 320 | obj[objKey] = decipher(key, value) 321 | } else if (mode === 'cipher') { 322 | obj[objKey] = cipherValue(key, value) 323 | } else { 324 | // Has to be `hash`, invalid modes filtered at `processObject()` level 325 | obj[objKey] = hashValue(value) 326 | } 327 | } 328 | 329 | // Produces a relevant object key list to be traversed for an object, 330 | // depending on whether we regard it as a Document or a query/update descriptor. 331 | // 332 | // - Documents should only have their current-level fields inspected, as it 333 | // is likely that `Object.keys()` would return waaaay too many technical 334 | // Mongoose fields on them, and not the synthetic document property accessors, 335 | // that are not enumerable. 336 | // - Query/Update object descriptors should be traversed by inspecting all their 337 | // keys, conversely. 338 | function produceKeyList(obj, { fields, isDocument, prefix }) { 339 | if (!isDocument) { 340 | return Object.keys(obj) 341 | } 342 | 343 | // Document mode: 344 | // 1. Filter field paths based on the current prefix, if any 345 | const baseList = prefix 346 | ? fields.filter((path) => path === prefix || path.startsWith(prefix + '.')) 347 | : fields 348 | // 2. Strip prefix and deeper path levels to retain only current-level fields 349 | const prefixRegex = prefix 350 | ? new RegExp('^' + escapeRegexp(prefix) + '.?') 351 | : '' 352 | const currentLevelFields = baseList 353 | .map((path) => path.replace(prefixRegex, '').split('.')[0]) 354 | .filter(Boolean) 355 | 356 | // 3. If there are no current-level fields and we're on an Array, this 357 | // means all the array items need processing, so `Object.keys()` is fine. 358 | if (currentLevelFields.length === 0 && Array.isArray(obj)) { 359 | return Object.keys(obj) 360 | } 361 | 362 | // 4. Otherwise, ensure uniqueness to avoid double processing 363 | return [...new Set(currentLevelFields)] 364 | } 365 | 366 | // Partitions a single descriptor `fields` into a query on the one hand 367 | // (fields that do not match `passwordFields` paths) and passwords on the 368 | // other hands (fields that do match). We need this in `authenticate()` in 369 | // order to first filter by query, then build a list of secure password 370 | // comparisons on the resulting docs, as hashes are intentionally unstable 371 | // (they vary from one hash to the other for the same cleartext), so we 372 | // can't just query on a hash we'd get this time around. 373 | // 374 | // @see `authenticate()` 375 | function splitAuthenticationFields({ 376 | fields, 377 | passwordFields, 378 | query = {}, 379 | passwords = {}, 380 | prefix = null, 381 | }) { 382 | for (const [field, value] of Object.entries(fields)) { 383 | if (typeof value === 'object' && value != null) { 384 | prefix = prefix ? `${prefix}.${field}` : field 385 | splitAuthenticationFields({ 386 | fields: value, 387 | passwordFields, 388 | query, 389 | passwords, 390 | prefix, 391 | }) 392 | } else { 393 | const fieldPath = prefix ? `${prefix}.${field}` : field 394 | const recipient = passwordFields.includes(fieldPath) ? passwords : query 395 | updateObject(recipient, fieldPath, value) 396 | } 397 | } 398 | 399 | if (prefix == null && Object.keys(passwords).length === 0) { 400 | const candidates = [...passwordFields].sort().join(', ') 401 | throw new Error( 402 | `No password field (${candidates}) found in \`authenticate\` call` 403 | ) 404 | } 405 | 406 | return { query, passwords } 407 | } 408 | 409 | // Updates a recipient object `obj` so the field at path `path` (which 410 | // potentially describes a nested field using dot separators) exists in 411 | // it with value `value`. Missing intermediary object properties are 412 | // created on-the-fly. Used to populate query/password field descriptors 413 | // in `splitAuthenticationFields()` above. 414 | // 415 | // @see `splitAuthenticationFields()`. 416 | function updateObject(obj, path, value) { 417 | const segments = path.split('.') 418 | let node 419 | while ((node = segments.shift())) { 420 | obj[node] = segments.length > 0 ? obj[node] || {} : value 421 | obj = obj[node] 422 | } 423 | } 424 | 425 | // Produces a list of cleartext/hashed password value pairs, so a promise list 426 | // of secure comparisons can be done based on it. This recursively walks the 427 | // potentially-nested password field/cleartext descriptor (`passwords`), matching 428 | // the traversal on document fields. If the document misses some of the relevant 429 | // fields, it will yield empty-string hashes for these, ensuring comparison failure. 430 | // 431 | // @see `authenticate()`. 432 | function walkDocumentPasswordFields(doc = {}, passwords, result = []) { 433 | for (const [field, value] of Object.entries(passwords)) { 434 | if (typeof value === 'object' && value != null) { 435 | walkDocumentPasswordFields(doc[field], value, result) 436 | } else { 437 | result.push([value, doc[field] || '']) 438 | } 439 | } 440 | return result 441 | } 442 | 443 | module.exports = { 444 | tests: { 445 | cipherValue, 446 | processObject, 447 | splitAuthenticationFields, 448 | updateObject, 449 | walkDocumentPasswordFields, 450 | }, 451 | markFieldsAsPII, 452 | pluginWasUsedOn, 453 | } 454 | -------------------------------------------------------------------------------- /markFieldsAsPII.spec.js: -------------------------------------------------------------------------------- 1 | const MongoDBMemoryServer = require('mongodb-memory-server').default 2 | const mongoose = require('mongoose') 3 | 4 | describe('markFieldsAsPII plugin', () => { 5 | let connection 6 | let server 7 | 8 | beforeAll(async () => { 9 | // DEV NOTE: do NOT resetModules() here, as it'll impact Mongoose’s 10 | // internal driver initialization, leading to weird "Decimal128 of null" 11 | // errors -- this one was tough to hunt down… 12 | jest.dontMock('./util/ciphers') 13 | 14 | server = new MongoDBMemoryServer() 15 | const url = await server.getConnectionString() 16 | connection = await mongoose.createConnection(url, { 17 | autoReconnect: true, 18 | connectTimeoutMS: 1000, 19 | reconnectInterval: 100, 20 | reconnectTries: Number.MAX_VALUE, 21 | useNewUrlParser: true, 22 | }) 23 | }) 24 | 25 | afterAll(() => { 26 | connection.close() 27 | server.stop() 28 | }) 29 | 30 | describe('when checking its options', () => { 31 | const { markFieldsAsPII } = require('./markFieldsAsPII') 32 | 33 | it('should mandate either a `fields` or `passwordFields` option', () => { 34 | expect(() => markFieldsAsPII({})).toThrowError( 35 | /at least one of.*fields.*passwordFields/ 36 | ) 37 | }) 38 | 39 | it('should mandate a `key` setting when `fields` is provided', () => { 40 | expect(() => markFieldsAsPII({}, { fields: ['email'] })).toThrowError( 41 | /Missing required.*key/ 42 | ) 43 | }) 44 | }) 45 | 46 | describe('when dealing with PII fields', () => { 47 | let User 48 | let userCollection 49 | 50 | beforeAll(() => { 51 | const { markFieldsAsPII } = require('./markFieldsAsPII') 52 | const schema = new mongoose.Schema({ 53 | email: String, 54 | firstName: String, 55 | historySize: Number, 56 | lastName: String, 57 | role: String, 58 | }) 59 | 60 | const key = '126d8cf92d95941e9907b0d9913ce00e' 61 | schema.plugin(markFieldsAsPII, { 62 | fields: ['email', 'firstName', 'lastName'], 63 | key, 64 | }) 65 | 66 | User = connection.model('PIIUser', schema) 67 | userCollection = new User().collection 68 | }) 69 | 70 | it('should cipher DB fields on create', async () => { 71 | const user = await User.create({ 72 | email: 'foo@bar.com', 73 | firstName: 'John', 74 | historySize: 100, 75 | lastName: 'Smith', 76 | role: 'guest', 77 | }) 78 | 79 | // Instance fields were deciphered back again 80 | expect(user.email).toEqual('foo@bar.com') 81 | expect(user.firstName).toEqual('John') 82 | expect(user.lastName).toEqual('Smith') 83 | 84 | // DB fields were ciphered 85 | const doc = await rawDocFor(user) 86 | expect(doc.email).toEqual( 87 | '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==' 88 | ) 89 | expect(doc.firstName).toEqual( 90 | 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg==' 91 | ) 92 | expect(doc.historySize).toEqual(100) 93 | expect(doc.lastName).toEqual( 94 | 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA==' 95 | ) 96 | expect(doc.role).toEqual('guest') 97 | }) 98 | 99 | it('should cipher DB fields on save', async () => { 100 | const user = new User({ 101 | email: 'foo@bar.com', 102 | firstName: 'John', 103 | historySize: 100, 104 | lastName: 'Smith', 105 | role: 'guest', 106 | }) 107 | await user.save() 108 | 109 | // Instance fields were deciphered back again 110 | expect(user.email).toEqual('foo@bar.com') 111 | expect(user.firstName).toEqual('John') 112 | expect(user.lastName).toEqual('Smith') 113 | 114 | // DB fields were ciphered 115 | const doc = await rawDocFor(user) 116 | expect(doc.email).toEqual( 117 | '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==' 118 | ) 119 | expect(doc.firstName).toEqual( 120 | 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg==' 121 | ) 122 | expect(doc.historySize).toEqual(100) 123 | expect(doc.lastName).toEqual( 124 | 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA==' 125 | ) 126 | expect(doc.role).toEqual('guest') 127 | }) 128 | 129 | it('should cipher DB fields on insertMany', async () => { 130 | const data = [ 131 | { 132 | email: [ 133 | 'foo@bar.com', 134 | '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==', 135 | ], 136 | firstName: ['John', 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg=='], 137 | lastName: ['Smith', 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA=='], 138 | }, 139 | { 140 | email: [ 141 | 'mark@example.com', 142 | 'qdxEGhjGiXX8dQvjEFp4yASg5f8rgyBzDF0X9l7T1jbhG+Dbajz5EENQ0TEpaxOlc=', 143 | ], 144 | firstName: ['Mark', 'Aj6ic0IkuWp3LV5P31i76Q1+eHIACe7wck3uM0vfBu1Q=='], 145 | lastName: [ 146 | 'Roberts', 147 | 'tmoDgrAWHw60SdZPl4pJLgfGmzIAYRuPiGCuK3JtgxTQ==', 148 | ], 149 | }, 150 | ] 151 | for (const datum of data) { 152 | datum.historySize = [100, 100] 153 | datum.role = ['guest', 'guest'] 154 | } 155 | 156 | // Insert data descriptors should not use [clearText, ciphered] pairs, but 157 | // just clearText values, so let's derive a proper descriptor array from above. 158 | const insertData = data.map((desc) => 159 | Object.entries(desc).reduce((obj, [field, [clearText]]) => { 160 | obj[field] = clearText 161 | return obj 162 | }, {}) 163 | ) 164 | const users = await User.insertMany(insertData) 165 | 166 | // And now let's check all the fields across both Mongoose Documents 167 | // and stored raw MongoDB documents. 168 | for (const [index, user] of users.entries()) { 169 | const doc = await rawDocFor(user) 170 | for (const [field, [clearText, ciphered]] of Object.entries( 171 | data[index] 172 | )) { 173 | // It stinks to high Heaven that Jasmine/Jest matchers do not allow 174 | // a custom failure message, like, say, Chai or RSpec would. Unbelievable. 175 | // jest-expect-message is a solution, but putting the message *inside expect* 176 | // instead of as a last arg in the matcher just looks fugly. 177 | expect(user[field]).toEqual(clearText) 178 | expect(doc[field]).toEqual(ciphered) 179 | } 180 | } 181 | }) 182 | 183 | it('should uncipher fields on finds', async () => { 184 | const { ops: docs } = await userCollection.insertMany([ 185 | { 186 | email: '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==', 187 | firstName: 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg==', 188 | historySize: 100, 189 | lastName: 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA==', 190 | role: 'guest', 191 | }, 192 | { 193 | email: 194 | 'qdxEGhjGiXX8dQvjEFp4yASg5f8rgyBzDF0X9l7T1jbhG+Dbajz5EENQ0TEpaxOlc=', 195 | firstName: 'Aj6ic0IkuWp3LV5P31i76Q1+eHIACe7wck3uM0vfBu1Q==', 196 | historySize: 100, 197 | lastName: 'tmoDgrAWHw60SdZPl4pJLgfGmzIAYRuPiGCuK3JtgxTQ==', 198 | role: 'guest', 199 | }, 200 | ]) 201 | 202 | const users = await User.find({ _id: docs.map(({ _id }) => _id) }) 203 | expect(users[0]).toMatchObject({ 204 | email: 'foo@bar.com', 205 | firstName: 'John', 206 | historySize: 100, 207 | lastName: 'Smith', 208 | role: 'guest', 209 | }) 210 | expect(users[1]).toMatchObject({ 211 | email: 'mark@example.com', 212 | firstName: 'Mark', 213 | historySize: 100, 214 | lastName: 'Roberts', 215 | role: 'guest', 216 | }) 217 | }) 218 | 219 | it('should cipher queries for finds', async () => { 220 | const { 221 | ops: [doc], 222 | } = await userCollection.insertOne({ 223 | email: '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==', 224 | firstName: 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg==', 225 | historySize: 100, 226 | lastName: 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA==', 227 | role: 'guest', 228 | }) 229 | 230 | // We serialize this to avoid deletion happening before another find 231 | // completes (due to the parallel execution of `Promise.all`), which 232 | // would cause random failures as we've routinely seen on Travis :-/ 233 | const results = [ 234 | ...(await User.find({ 235 | _id: doc._id, 236 | email: 'foo@bar.com', 237 | role: 'guest', 238 | })), 239 | await User.findOne({ 240 | _id: doc._id, 241 | email: 'foo@bar.com', 242 | role: 'guest', 243 | }), 244 | await User.findOneAndDelete({ 245 | _id: doc._id, 246 | email: 'foo@bar.com', 247 | role: 'guest', 248 | }), 249 | ] 250 | for (const user of results) { 251 | expect(user).toMatchObject({ 252 | email: 'foo@bar.com', 253 | firstName: 'John', 254 | historySize: 100, 255 | lastName: 'Smith', 256 | role: 'guest', 257 | }) 258 | } 259 | }) 260 | 261 | it('should cipher updates for findOneAndUpdate', async () => { 262 | const { 263 | ops: [doc], 264 | } = await userCollection.insertOne({ 265 | email: '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==', 266 | firstName: 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg==', 267 | historySize: 100, 268 | lastName: 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA==', 269 | role: 'guest', 270 | }) 271 | 272 | const user = await User.findOneAndUpdate( 273 | { _id: doc._id }, 274 | { 275 | email: 'foo@bar.net', 276 | role: 'admin', 277 | }, 278 | { new: true } 279 | ) 280 | expect(user).toMatchObject({ 281 | email: 'foo@bar.net', 282 | firstName: 'John', 283 | historySize: 100, 284 | lastName: 'Smith', 285 | role: 'admin', 286 | }) 287 | 288 | const updatedDoc = await userCollection.findOne({ _id: doc._id }) 289 | expect(updatedDoc).toMatchObject({ 290 | email: 'YQ3zjPwlyq6xP8+Aq4uSrwEmOooRV/uaPioRQog9zoBQ==', 291 | role: 'admin', 292 | }) 293 | }) 294 | 295 | it('should cipher queries for counts', async () => { 296 | const attrs = { 297 | email: '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==', 298 | firstName: 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg==', 299 | historySize: 150, 300 | lastName: 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA==', 301 | role: 'countable', 302 | } 303 | const descriptors = [1, 2, 3].map(() => ({ ...attrs })) 304 | await userCollection.insertMany(descriptors) 305 | 306 | expect( 307 | await User.count({ email: 'foo@bar.com', role: 'countable' }) 308 | ).toEqual(descriptors.length) 309 | expect( 310 | await User.countDocuments({ firstName: 'John', role: 'countable' }) 311 | ).toEqual(descriptors.length) 312 | }) 313 | 314 | it('should cipher queries for updates and replaces', async () => { 315 | const attrs = { 316 | email: '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==', 317 | firstName: 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg==', 318 | historySize: 100, 319 | lastName: 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA==', 320 | role: 'updatable', 321 | } 322 | const descriptors = [1, 2, 3, 4, 5].map(() => ({ ...attrs })) 323 | const { ops: docs } = await userCollection.insertMany(descriptors) 324 | 325 | const result = await Promise.all([ 326 | User.replaceOne( 327 | { _id: docs[0]._id, email: 'foo@bar.com', role: 'updatable' }, 328 | { 329 | ...attrs, 330 | role: 'updated', 331 | } 332 | ), 333 | User.update( 334 | { _id: docs[1]._id, email: 'foo@bar.com', role: 'updatable' }, 335 | { 336 | $set: { role: 'updated' }, 337 | } 338 | ), 339 | User.updateOne( 340 | { _id: docs[2]._id, email: 'foo@bar.com', role: 'updatable' }, 341 | { 342 | $set: { role: 'updated' }, 343 | } 344 | ), 345 | User.updateMany( 346 | { 347 | _id: docs.slice(3).map(({ _id }) => _id), 348 | email: 'foo@bar.com', 349 | role: 'updatable', 350 | }, 351 | { 352 | $set: { role: 'updated' }, 353 | } 354 | ), 355 | ]) 356 | 357 | expect(result).toMatchObject([ 358 | { n: 1, nModified: 1 }, 359 | { n: 1, nModified: 1 }, 360 | { n: 1, nModified: 1 }, 361 | { n: 2, nModified: 2 }, 362 | ]) 363 | }) 364 | 365 | it('should cipher updates on updates and replaces', async () => { 366 | const attrs = { 367 | email: '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==', 368 | firstName: 'QbbQzV3asVB0+Ivxw1bonAIjsLZzhRCjyMoCeHeAE8Lg==', 369 | historySize: 100, 370 | lastName: 'L0H0hF8b4HZSxYiKRbMntQsLUZRGr6OpauwAEWsAP4jA==', 371 | role: 'updateCipherable', 372 | } 373 | const descriptors = [1, 2, 3, 4, 5].map(() => ({ ...attrs })) 374 | const { ops: docs } = await userCollection.insertMany(descriptors) 375 | 376 | await Promise.all([ 377 | User.replaceOne( 378 | { _id: docs[0]._id }, 379 | { ...attrs, email: 'foo@bar.net' } 380 | ), 381 | User.update({ _id: docs[1]._id }, { $set: { firstName: 'Mark' } }), 382 | User.updateOne({ _id: docs[2]._id }, { $set: { lastName: 'Roberts' } }), 383 | User.updateMany( 384 | { _id: docs.slice(3).map(({ _id }) => _id) }, 385 | { $set: { email: 'foo@bar.net' } } 386 | ), 387 | ]) 388 | 389 | const updatedDocs = await userCollection 390 | .find({ role: 'updateCipherable' }) 391 | .toArray() 392 | expect(updatedDocs).toMatchObject([ 393 | { email: 'YQ3zjPwlyq6xP8+Aq4uSrwEmOooRV/uaPioRQog9zoBQ==' }, 394 | { firstName: 'Aj6ic0IkuWp3LV5P31i76Q1+eHIACe7wck3uM0vfBu1Q==' }, 395 | { lastName: 'tmoDgrAWHw60SdZPl4pJLgfGmzIAYRuPiGCuK3JtgxTQ==' }, 396 | { email: 'YQ3zjPwlyq6xP8+Aq4uSrwEmOooRV/uaPioRQog9zoBQ==' }, 397 | { email: 'YQ3zjPwlyq6xP8+Aq4uSrwEmOooRV/uaPioRQog9zoBQ==' }, 398 | ]) 399 | }) 400 | 401 | function rawDocFor(modelDoc) { 402 | return modelDoc.collection.findOne({ _id: modelDoc._id }) 403 | } 404 | }) 405 | 406 | describe('when dealing with password fields', () => { 407 | let User 408 | let userCollection 409 | 410 | beforeAll(() => { 411 | const { markFieldsAsPII } = require('./markFieldsAsPII') 412 | const schema = new mongoose.Schema({ 413 | admin: { password: String, role: String }, 414 | a: { b: { secret: String } }, 415 | email: String, 416 | password: String, 417 | }) 418 | 419 | schema.plugin(markFieldsAsPII, { 420 | passwordFields: ['password', 'admin.password', 'a.b.secret'], 421 | }) 422 | 423 | User = connection.model('PassUser', schema) 424 | userCollection = new User().collection 425 | }) 426 | 427 | it('should hash on save', async () => { 428 | const user = new User({ 429 | email: 'foo@bar.com', 430 | password: 'foobar', 431 | admin: { role: 'manager', password: 'foobar42' }, 432 | }) 433 | await user.save() 434 | const doc = await userCollection.findOne({ _id: user._id }) 435 | 436 | expect(user.email).toEqual('foo@bar.com') 437 | expect(user.admin.role).toEqual('manager') 438 | expect(user.password).toMatch(/^\$2a\$\d{2}\$.{53}$/) 439 | expect(doc.password).toEqual(user.password) 440 | expect(user.admin.password).toMatch(/^\$2a\$\d{2}\$.{53}$/) 441 | expect(doc.admin.password).toEqual(user.admin.password) 442 | }) 443 | 444 | it('should not double-hash', async () => { 445 | const user = await User.create({ password: 'foobar' }) 446 | const pwd = user.password 447 | 448 | await user.save() 449 | expect(user.password).toEqual(pwd) 450 | }) 451 | 452 | it('should hash on insertMany', async () => { 453 | const users = await User.insertMany([ 454 | { email: 'foo@bar.net', password: 'kapoué' }, 455 | { email: 'foo@bar.org', password: 'yolo' }, 456 | ]) 457 | const docs = await User.find({ email: users.map(({ email }) => email) }) 458 | 459 | for (const [index, user] of users.entries()) { 460 | expect(user.password).toMatch(/^\$2a\$\d{2}\$.{53}$/) 461 | expect(docs[index].password).toEqual(user.password) 462 | } 463 | }) 464 | 465 | it('should not use stable hashes', async () => { 466 | const descriptors = Array(10) 467 | .fill(true) 468 | .map(() => ({ password: 'foobar' })) 469 | const users = await User.insertMany(descriptors) 470 | const uniqueHashes = new Set(users.map(({ password }) => password)) 471 | 472 | expect(uniqueHashes.size).toEqual(descriptors.length) 473 | }) 474 | 475 | it('should provide a static authenticate method for models', () => { 476 | expect(User).toHaveProperty('authenticate') 477 | expect(User.authenticate).toBeInstanceOf(Function) 478 | }) 479 | 480 | it('should require at least one defined password field', () => { 481 | expect(User.authenticate({ email: 'foo@bar.com' })).rejects.toThrowError( 482 | /No password field.*found/ 483 | ) 484 | }) 485 | 486 | it('should authenticate honoring query fields', async () => { 487 | const users = await User.insertMany([ 488 | { email: 'query@example.com', password: 'query' }, 489 | { email: 'query@example.net', password: 'query' }, 490 | ]) 491 | 492 | expect(await User.authenticate({ password: 'query' })).toMatchObject( 493 | users[0].toJSON() 494 | ) 495 | expect( 496 | await User.authenticate({ password: 'query' }, { single: false }) 497 | ).toHaveLength(2) 498 | expect( 499 | await User.authenticate({ 500 | email: 'query@example.com', 501 | password: 'query', 502 | }) 503 | ).toMatchObject(users[0].toJSON()) 504 | expect( 505 | await User.authenticate({ 506 | email: 'query@example.org', 507 | password: 'query', 508 | }) 509 | ).toBeNull() 510 | }) 511 | 512 | it('should be able to authenticate across multiple password fields', async () => { 513 | const users = await User.insertMany([ 514 | { password: 'toplevel' }, 515 | { password: 'toplevel', a: { b: { secret: 'yo' } } }, 516 | ]) 517 | 518 | expect( 519 | await User.authenticate({ 520 | password: 'toplevel', 521 | a: { b: { secret: 'yo' } }, 522 | }) 523 | ).toMatchObject(users[1].toJSON()) 524 | }) 525 | }) 526 | 527 | describe('when dealing with both PII and password fields', () => { 528 | it('should be able to authenticate with queries that need ciphering', async () => { 529 | const { markFieldsAsPII } = require('./markFieldsAsPII') 530 | const schema = new mongoose.Schema({ 531 | email: String, 532 | password: String, 533 | kind: String, 534 | }) 535 | 536 | schema.plugin(markFieldsAsPII, { 537 | fields: 'email', 538 | key: '126d8cf92d95941e9907b0d9913ce00e', 539 | passwordFields: 'password', 540 | }) 541 | 542 | const User = connection.model('HardUser', schema) 543 | 544 | const user = await User.create({ 545 | email: 'foo@bar.com', 546 | kind: 'hard', 547 | password: 'foobar', 548 | }) 549 | 550 | const users = await User.authenticate({ 551 | kind: 'hard', 552 | email: 'foo@bar.com', 553 | password: 'foobar', 554 | }) 555 | expect(users).toMatchObject(user.toJSON()) 556 | }) 557 | }) 558 | }) 559 | 560 | describe('pluginWasUsedOn', () => { 561 | let markFieldsAsPII 562 | let pluginWasUsedOn 563 | 564 | beforeAll(() => { 565 | ;({ markFieldsAsPII, pluginWasUsedOn } = require('./markFieldsAsPII')) 566 | }) 567 | 568 | it('should return true on Schemas that use the plugin', () => { 569 | const schema = new mongoose.Schema({ name: String, password: String }) 570 | schema.plugin(markFieldsAsPII, { passwordFields: 'password' }) 571 | 572 | expect(pluginWasUsedOn(schema)).toBeTruthy() 573 | }) 574 | 575 | it('should return true on Models whose schemas use the plugin', () => { 576 | const schema = new mongoose.Schema({ name: String, password: String }) 577 | schema.plugin(markFieldsAsPII, { passwordFields: 'password' }) 578 | const Model = mongoose.model('UsingPlugin', schema) 579 | 580 | expect(pluginWasUsedOn(Model)).toBeTruthy() 581 | }) 582 | 583 | it('should return false on Schemas that don’t use the plugin', () => { 584 | const schema = new mongoose.Schema({ name: String }) 585 | 586 | expect(pluginWasUsedOn(schema)).toBeFalsy() 587 | }) 588 | 589 | it('should return false on Models whose schemas don’t use the plugin', () => { 590 | const schema = new mongoose.Schema({ name: String }) 591 | const Model = mongoose.model('NotUsingPlugin', schema) 592 | 593 | expect(pluginWasUsedOn(Model)).toBeFalsy() 594 | }) 595 | }) 596 | 597 | describe('Helper functions', () => { 598 | describe('cipherValue', () => { 599 | const { cipherValue } = require('./markFieldsAsPII').tests 600 | const key = '126d8cf92d95941e9907b0d9913ce00e' 601 | 602 | it('should handle Buffer values', () => { 603 | const actual = cipherValue(key, Buffer.from('yowza', 'utf8')) 604 | const expected = 'SaUyLkjzcKSwx9PY2c6A5geG7Ydb0nOAFiwQqJZweE+Q==' 605 | expect(actual).toEqual(expected) 606 | }) 607 | 608 | it('should handle String values', () => { 609 | const actual = cipherValue(key, 'yowza') 610 | const expected = 'SaUyLkjzcKSwx9PY2c6A5geG7Ydb0nOAFiwQqJZweE+Q==' 611 | expect(actual).toEqual(expected) 612 | }) 613 | 614 | it('should handle non-Buffer, non-String values', () => { 615 | expect(cipherValue(key, 42)).toEqual( 616 | 'Ocp86ezGn2lr99ILsj3RUgDAN6Jv5s8/5L00TeTW6Zmg==' 617 | ) 618 | expect(cipherValue(key, /foobar/)).toEqual( 619 | 'KS13lM9qZVEPZ9eVFTUisQXbLWlfJ394d0C+WgbYMe3w==' 620 | ) 621 | }) 622 | 623 | it('should be stable across calls for the same value', () => { 624 | const expected = 'SaUyLkjzcKSwx9PY2c6A5geG7Ydb0nOAFiwQqJZweE+Q==' 625 | for (let index = 0; index < 10; ++index) { 626 | const actual = cipherValue(key, 'yowza') 627 | expect(actual).toEqual(expected) 628 | } 629 | }) 630 | }) 631 | 632 | describe('processObject', () => { 633 | const key = '59aad44db330ad2bf34f6730e50c0058' 634 | let processObject 635 | 636 | beforeAll(() => { 637 | jest.resetModules() 638 | jest.doMock('./util/ciphers', () => ({ 639 | cipher(_, clearText) { 640 | return `CIPHERED:${clearText}` 641 | }, 642 | decipher(_, obscured) { 643 | return `DECIPHERED:${obscured}` 644 | }, 645 | })) 646 | 647 | processObject = require('./markFieldsAsPII').tests.processObject 648 | }) 649 | 650 | afterAll(() => { 651 | jest.dontMock('./util/ciphers') 652 | }) 653 | 654 | it('should refuse invalid modes', () => { 655 | expect(() => processObject({}, { mode: 'yolo' })).toThrowError( 656 | 'Unknown processObject mode: yolo' 657 | ) 658 | }) 659 | 660 | it('should only cipher specified fields', () => { 661 | const obj = { firstName: 'John', lastName: 'Smith', age: 42 } 662 | const expected = { 663 | firstName: 'CIPHERED:John', 664 | lastName: 'CIPHERED:Smith', 665 | age: 42, 666 | } 667 | processObject(obj, { 668 | fields: ['firstName', 'lastName'], 669 | key, 670 | mode: 'cipher', 671 | }) 672 | 673 | expect(obj).toEqual(expected) 674 | }) 675 | 676 | it('should dive into operator descriptors', () => { 677 | const update = { $set: { age: 18, firstName: 'Mark' } } 678 | const expected = { 679 | $set: { age: 18, firstName: 'CIPHERED:Mark' }, 680 | } 681 | processObject(update, { 682 | fields: ['firstName', 'lastName'], 683 | key, 684 | mode: 'cipher', 685 | }) 686 | 687 | expect(update).toEqual(expected) 688 | }) 689 | 690 | it('should dive into object values', () => { 691 | const obj = { 692 | identity: { firstName: 'John', lastName: 'Smith' }, 693 | firstName: 'Mark', 694 | } 695 | const expected = { 696 | identity: { 697 | firstName: 'CIPHERED:John', 698 | lastName: 'CIPHERED:Smith', 699 | }, 700 | firstName: 'CIPHERED:Mark', 701 | } 702 | processObject(obj, { 703 | fields: ['firstName', 'lastName'], 704 | key, 705 | mode: 'cipher', 706 | }) 707 | 708 | expect(obj).toEqual(expected) 709 | }) 710 | 711 | it('should dive into array values', () => { 712 | const obj = { 713 | aliases: ['Killer', 'Boss', 'Spy'], 714 | firstName: 'John', 715 | lastName: 'Smith', 716 | } 717 | const expected = { 718 | aliases: ['CIPHERED:Killer', 'CIPHERED:Boss', 'CIPHERED:Spy'], 719 | firstName: 'CIPHERED:John', 720 | lastName: 'CIPHERED:Smith', 721 | } 722 | processObject(obj, { 723 | fields: ['aliases', 'firstName', 'lastName'], 724 | key, 725 | mode: 'cipher', 726 | }) 727 | 728 | expect(obj).toEqual(expected) 729 | }) 730 | 731 | it('should handle nested field descriptors', () => { 732 | const obj = { 733 | identity: { firstName: 'John', lastName: 'Smith' }, 734 | age: 42, 735 | firstName: 'Mark', 736 | } 737 | const expected = { 738 | identity: { 739 | firstName: 'CIPHERED:John', 740 | lastName: 'CIPHERED:Smith', 741 | }, 742 | age: 42, 743 | firstName: 'Mark', 744 | } 745 | processObject(obj, { 746 | fields: ['identity.firstName', 'identity.lastName'], 747 | key, 748 | mode: 'cipher', 749 | }) 750 | 751 | expect(obj).toEqual(expected) 752 | }) 753 | 754 | it('should work in decipher mode', () => { 755 | const obj = { 756 | identity: { 757 | aliases: ['Killer', 'Boss', 'Spy'], 758 | firstName: 'John', 759 | lastName: 'Smith', 760 | }, 761 | firstName: 'Mark', 762 | } 763 | const expected = { 764 | identity: { 765 | aliases: ['DECIPHERED:Killer', 'DECIPHERED:Boss', 'DECIPHERED:Spy'], 766 | firstName: 'DECIPHERED:John', 767 | lastName: 'DECIPHERED:Smith', 768 | }, 769 | firstName: 'Mark', 770 | } 771 | 772 | processObject(obj, { 773 | fields: ['aliases', 'identity.firstName', 'lastName'], 774 | key, 775 | mode: 'decipher', 776 | }) 777 | 778 | expect(obj).toEqual(expected) 779 | }) 780 | 781 | it('should work in document mode', () => { 782 | const obj = { 783 | identity: { aliases: ['Foo', 'Bar'] }, 784 | trap: { firstName: 'Mark', lastName: 'Roberts' }, 785 | firstName: 'John', 786 | lastName: 'Smith', 787 | } 788 | const expected = { 789 | identity: { aliases: ['CIPHERED:Foo', 'CIPHERED:Bar'] }, 790 | trap: { firstName: 'Mark', lastName: 'Roberts' }, 791 | firstName: 'CIPHERED:John', 792 | lastName: 'CIPHERED:Smith', 793 | } 794 | processObject(obj, { 795 | fields: ['identity.aliases', 'firstName', 'lastName'], 796 | key, 797 | isDocument: true, 798 | mode: 'cipher', 799 | }) 800 | 801 | expect(obj).toEqual(expected) 802 | }) 803 | }) 804 | 805 | describe('splitAuthenticationFields', () => { 806 | const { splitAuthenticationFields } = require('./markFieldsAsPII').tests 807 | 808 | it('should split toplevel fields', () => { 809 | const fields = { email: 'foo@bar.com', password: 'foobar' } 810 | const passwordFields = ['password'] 811 | 812 | expect(splitAuthenticationFields({ fields, passwordFields })).toEqual({ 813 | query: { email: 'foo@bar.com' }, 814 | passwords: { password: 'foobar' }, 815 | }) 816 | }) 817 | 818 | it('should split nested fields', () => { 819 | const fields = { 820 | email: 'foo@bar.com', 821 | password: 'foobar', 822 | admin: { role: 'manager', password: 'quuxdoo' }, 823 | } 824 | const passwordFields = ['password', 'admin.password'] 825 | 826 | expect(splitAuthenticationFields({ fields, passwordFields })).toEqual({ 827 | query: { email: 'foo@bar.com', admin: { role: 'manager' } }, 828 | passwords: { password: 'foobar', admin: { password: 'quuxdoo' } }, 829 | }) 830 | }) 831 | }) 832 | 833 | describe('updateObject', () => { 834 | const { updateObject } = require('./markFieldsAsPII').tests 835 | 836 | it('should work on existing containers', () => { 837 | const obj = { foo: 'bar', xyz: 123 } 838 | 839 | updateObject(obj, 'foo', 'baz') 840 | expect(obj).toEqual({ foo: 'baz', xyz: 123 }) 841 | 842 | obj.nested = { abc: 'def', ghi: 'jkl' } 843 | updateObject(obj, 'nested.ghi', 'JKL') 844 | updateObject(obj, 'nested.mno', 'pqr') 845 | expect(obj.nested).toEqual({ abc: 'def', ghi: 'JKL', mno: 'pqr' }) 846 | }) 847 | 848 | it('should create missing containers on-the-fly', () => { 849 | const obj = { foo: 'bar', xyz: 123, nested: { abc: 'def' } } 850 | 851 | updateObject(obj, 'nested.deeper.wow', 'much win') 852 | updateObject(obj, 'parallel.yowza', 'such code') 853 | expect(obj).toEqual({ 854 | foo: 'bar', 855 | xyz: 123, 856 | nested: { abc: 'def', deeper: { wow: 'much win' } }, 857 | parallel: { yowza: 'such code' }, 858 | }) 859 | }) 860 | }) 861 | 862 | describe('walkDocumentPasswordFields', () => { 863 | const { walkDocumentPasswordFields } = require('./markFieldsAsPII').tests 864 | 865 | it('should produce pairs of cleartext/hash, including nested fields', () => { 866 | const doc = { 867 | password: 'hashedPassword', 868 | admin: { password: 'hashedAdminPassword' }, 869 | foo: 'bar', 870 | } 871 | const passwords = { 872 | password: 'password', 873 | admin: { password: 'adminPassword' }, 874 | } 875 | 876 | expect(walkDocumentPasswordFields(doc, passwords)).toEqual([ 877 | ['password', 'hashedPassword'], 878 | ['adminPassword', 'hashedAdminPassword'], 879 | ]) 880 | }) 881 | 882 | it('should default missing hashed values to an empty string', () => { 883 | const doc = { 884 | password: 'hashedPassword', 885 | foo: 'bar', 886 | } 887 | const passwords = { 888 | password: 'password', 889 | admin: { password: 'adminPassword' }, 890 | } 891 | 892 | expect(walkDocumentPasswordFields(doc, passwords)).toEqual([ 893 | ['password', 'hashedPassword'], 894 | ['adminPassword', ''], 895 | ]) 896 | }) 897 | }) 898 | }) 899 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongoose-pii", 3 | "version": "2.0.0", 4 | "description": "A Mongoose plugin that lets you transparently cipher stored PII and use securely-hashed passwords. Helps with security best practices for data storage.", 5 | "keywords": [ 6 | "mongodb", 7 | "mongoose", 8 | "security", 9 | "plugin", 10 | "pii", 11 | "password", 12 | "passwords", 13 | "bcrypt" 14 | ], 15 | "main": "index.js", 16 | "engines": { 17 | "node": ">= 8.6", 18 | "npm": ">= 5.2" 19 | }, 20 | "devInfo": { 21 | "why-no-deprecation-in-test": "Because we’re intentionally testing deprecated Mongo/Mongoose APIs are still hooked onto (count, findAndModify, update…)" 22 | }, 23 | "scripts": { 24 | "lint": "eslint *.js util/*.js examples/*.js", 25 | "test": "npm run lint && npm run test:core", 26 | "test:core": "npx --node-arg --no-deprecation jest", 27 | "test:watch": "npm run test:core -- --watch" 28 | }, 29 | "husky": { 30 | "hooks": { 31 | "commit-msg": "commitlint -E HUSKY_GIT_PARAMS", 32 | "pre-commit": "npm test", 33 | "pre-push": "npm test" 34 | } 35 | }, 36 | "directories": { 37 | "example": "examples" 38 | }, 39 | "repository": { 40 | "type": "git", 41 | "url": "git+https://github.com/deliciousinsights/mongoose-pii.git" 42 | }, 43 | "bugs": { 44 | "url": "https://github.com/deliciousinsights/mongoose-pii/issues" 45 | }, 46 | "author": "Christophe Porteneuve (https://delicious-insights.com/)", 47 | "homepage": "https://deliciousinsights.github.io/mongoose-pii", 48 | "license": "MIT", 49 | "eslintConfig": { 50 | "extends": [ 51 | "standard", 52 | "prettier" 53 | ], 54 | "plugins": [ 55 | "prettier" 56 | ], 57 | "rules": { 58 | "no-irregular-whitespace": 0 59 | }, 60 | "env": { 61 | "commonjs": true, 62 | "es6": true, 63 | "jest": true, 64 | "node": true 65 | } 66 | }, 67 | "prettier": { 68 | "arrowParens": "always", 69 | "jsxSingleQuote": true, 70 | "semi": false, 71 | "singleQuote": true, 72 | "trailingComma": "es5" 73 | }, 74 | "jest": { 75 | "collectCoverage": true, 76 | "collectCoverageFrom": [ 77 | "/*.js", 78 | "!commitlint.config.js", 79 | "/util/*.js" 80 | ], 81 | "coverageReporters": [ 82 | "lcov", 83 | "text", 84 | "html" 85 | ], 86 | "notify": true, 87 | "testEnvironment": "node" 88 | }, 89 | "dependencies": { 90 | "bcryptjs": "^2.4.3" 91 | }, 92 | "peerDependencies": { 93 | "mongoose": ">= 4" 94 | }, 95 | "devDependencies": { 96 | "@commitlint/cli": "^8.3.5", 97 | "@commitlint/config-conventional": "^8.3.4", 98 | "eslint": "^6.8.0", 99 | "eslint-config-prettier": "^6.9.0", 100 | "eslint-config-standard": "^14.1.0", 101 | "eslint-plugin-import": "^2.20.0", 102 | "eslint-plugin-node": "^11.0.0", 103 | "eslint-plugin-prettier": "^3.1.2", 104 | "eslint-plugin-promise": "^4.2.1", 105 | "eslint-plugin-standard": "^4.0.1", 106 | "husky": "^4.0.10", 107 | "jest": "^24.9.0", 108 | "mongodb-memory-server": "^6.2.3", 109 | "mongoose": "^5.8.9", 110 | "prettier": "^1.19.1" 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /util/ciphers.js: -------------------------------------------------------------------------------- 1 | const { 2 | createCipheriv, 3 | createDecipheriv, 4 | createHash, 5 | randomBytes, 6 | } = require('crypto') 7 | 8 | const ALGORITHM = 'aes-256-cbc' 9 | const IV_BYTES = 16 // 128 / 8 10 | const IV_BYTES_BASE64 = 22 // ceil(16 * 4 / 3), we strip the '==' suffix 11 | 12 | function cipher(key, clearText, { deriveIV = true } = {}) { 13 | let iv 14 | 15 | if (deriveIV) { 16 | const hasher = createHash('sha512') 17 | hasher.update(clearText) 18 | iv = Buffer.alloc(IV_BYTES) 19 | hasher.digest().copy(iv, 0, 0, IV_BYTES) 20 | } else { 21 | iv = randomBytes(IV_BYTES) 22 | } 23 | 24 | const processor = createCipheriv(ALGORITHM, key, iv) 25 | const data = Buffer.concat([processor.update(clearText), processor.final()]) 26 | const result = iv.toString('base64').slice(0, -2) + data.toString('base64') 27 | return result.toString('utf8') 28 | } 29 | 30 | function decipher(key, obscured) { 31 | const iv = Buffer.from(obscured.slice(0, IV_BYTES_BASE64) + '==', 'base64') 32 | const processor = createDecipheriv(ALGORITHM, key, iv) 33 | const data = Buffer.from(obscured.slice(IV_BYTES_BASE64), 'base64') 34 | const result = Buffer.concat([processor.update(data), processor.final()]) 35 | return result.toString('utf8') 36 | } 37 | 38 | module.exports = { cipher, decipher } 39 | -------------------------------------------------------------------------------- /util/ciphers.spec.js: -------------------------------------------------------------------------------- 1 | const { cipher, decipher } = require('./ciphers') 2 | 3 | const key = '126d8cf92d95941e9907b0d9913ce00e' 4 | 5 | describe('Cipher Utils', () => { 6 | const REGEX_OBSCURED = /^[\w/+]{38,}=*$/ 7 | 8 | describe('cipher', () => { 9 | it('should use stable, derived IVs by default', () => { 10 | const obscured1 = cipher(key, 'foo@bar.com') 11 | const obscured2 = cipher(key, 'foo@bar.com') 12 | expect(obscured1).toMatch(REGEX_OBSCURED) 13 | expect(obscured2).toMatch(REGEX_OBSCURED) 14 | expect(obscured1).toEqual(obscured2) 15 | }) 16 | 17 | it('should allow stable, derived IVs', () => { 18 | const obscured1 = cipher(key, 'foo@bar.com', { deriveIV: false }) 19 | const obscured2 = cipher(key, 'foo@bar.com', { deriveIV: false }) 20 | expect(obscured1).toMatch(REGEX_OBSCURED) 21 | expect(obscured2).toMatch(REGEX_OBSCURED) 22 | expect(obscured1).not.toEqual(obscured2) 23 | }) 24 | }) 25 | 26 | describe('decipher', () => { 27 | it('should work on both derived and random IV obscured texts', () => { 28 | expect( 29 | decipher(key, '2ZXfUUBPTPaETqXIA33bRwQNnif1/u/axrI84yQShR9Q==') 30 | ).toEqual('foo@bar.com') 31 | expect( 32 | decipher(key, 'Zj6jEHwYOGVDT92Dg9rKFw8DdfreEhm4pB4qtq6CdAFw==') 33 | ).toEqual('foo@bar.com') 34 | }) 35 | }) 36 | }) 37 | -------------------------------------------------------------------------------- /util/convert.js: -------------------------------------------------------------------------------- 1 | const { pluginWasUsedOn } = require('../markFieldsAsPII') 2 | 3 | function checkPluginWasUsed(Model) { 4 | if (!pluginWasUsedOn(Model)) { 5 | throw new Error( 6 | [ 7 | `${Model.modelName}’s schema did not register the markFieldsAsPII plugin.`, 8 | 'Make sure your model’s schema registers it, for instance:', 9 | 'mySchema.plugin(markFieldsAsPII, { /* your options here */ })', 10 | ].join('\n\n') 11 | ) 12 | } 13 | } 14 | 15 | async function convertDataForModel(Model, emitter = null) { 16 | checkPluginWasUsed(Model) 17 | 18 | const total = await Model.collection.estimatedDocumentCount() 19 | 20 | if (total === 0) { 21 | return total 22 | } 23 | 24 | let [converted, oldPercentage, oldBarWidth] = [0, 0, 0] 25 | let doc 26 | const scope = Model.collection.find().batchSize(10) 27 | 28 | while ((doc = await scope.next())) { 29 | await setupModelDoc(Model, doc).save() 30 | ++converted 31 | 32 | const percentage = Math.floor((converted * 100) / total) 33 | if (emitter) { 34 | notifyEmitter({ emitter, converted, percentage, oldPercentage }) 35 | } else { 36 | oldBarWidth = maintainProgressBar({ percentage, oldBarWidth }) 37 | } 38 | oldPercentage = percentage 39 | } 40 | 41 | return converted 42 | } 43 | 44 | function maintainProgressBar({ percentage, oldBarWidth }) { 45 | const output = process.stderr 46 | const barWidth = (output.columns ? Math.min(100, output.columns) : 80) - 2 47 | const newBarWidth = Math.round((percentage / 100) * barWidth) 48 | 49 | if (newBarWidth === oldBarWidth) { 50 | return oldBarWidth 51 | } 52 | 53 | if (oldBarWidth === 0) { 54 | output.write('\n[') 55 | } 56 | 57 | output.write('='.repeat(newBarWidth - oldBarWidth)) 58 | 59 | if (percentage === 100) { 60 | output.write(']\n') 61 | } 62 | 63 | return newBarWidth 64 | } 65 | 66 | function notifyEmitter({ emitter, converted, percentage, oldPercentage }) { 67 | emitter.emit('docs', converted) 68 | if (percentage > oldPercentage) { 69 | emitter.emit('progress', percentage) 70 | } 71 | } 72 | 73 | function setupModelDoc(Model, doc) { 74 | const result = Model.hydrate({}) 75 | for (const [key, value] of Object.entries(doc)) { 76 | result[key] = value 77 | } 78 | return result 79 | } 80 | 81 | module.exports = { convertDataForModel } 82 | -------------------------------------------------------------------------------- /util/convert.spec.js: -------------------------------------------------------------------------------- 1 | const MongoDBMemoryServer = require('mongodb-memory-server').default 2 | const mongoose = require('mongoose') 3 | 4 | const { checkPassword } = require('./passwords') 5 | const { convertDataForModel } = require('./convert') 6 | const { decipher } = require('./ciphers') 7 | const { markFieldsAsPII } = require('../markFieldsAsPII') 8 | 9 | describe('convert() utility', () => { 10 | let connection 11 | let pswMock 12 | let server 13 | 14 | beforeAll(async () => { 15 | server = new MongoDBMemoryServer() 16 | const url = await server.getConnectionString() 17 | connection = await mongoose.createConnection(url, { 18 | connectTimeoutMS: 1000, 19 | useNewUrlParser: true, 20 | useUnifiedTopology: true, 21 | }) 22 | }) 23 | 24 | afterAll(() => { 25 | connection.close() 26 | server.stop() 27 | }) 28 | 29 | beforeEach(() => { 30 | pswMock = jest 31 | .spyOn(process.stderr, 'write') 32 | .mockImplementation(() => {}) 33 | .mockName('process.stdout.write') 34 | }) 35 | 36 | afterEach(() => { 37 | pswMock.mockRestore() 38 | }) 39 | 40 | it('should detect that the plugin was not registered', async () => { 41 | const Model = connection.model( 42 | 'PluginLess', 43 | new mongoose.Schema({ name: String }) 44 | ) 45 | 46 | await expect(convertDataForModel(Model)).rejects.toThrow( 47 | /PluginLess’s schema did not register the markFieldsAsPII plugin/ 48 | ) 49 | }) 50 | 51 | it('should short-circuit on no-document situations', async () => { 52 | const schema = new mongoose.Schema({ name: String, password: String }) 53 | schema.plugin(markFieldsAsPII, { passwordFields: 'password' }) 54 | const Model = connection.model('WithPlugin', schema) 55 | 56 | await expect(convertDataForModel(Model)).resolves.toEqual(0) 57 | }) 58 | 59 | describe('along the way', () => { 60 | const DOCS = [ 61 | { email: 'john@example.com', name: 'John', password: 'secret' }, 62 | { email: 'mark@example.com', name: 'Mark', password: 'secret' }, 63 | { email: 'suzy@example.com', name: 'Suzy', password: 'secret' }, 64 | ] 65 | const KEY = 'I just luv mongodb-memory-server' 66 | 67 | it('should properly cipher documents', async () => { 68 | const schema = new mongoose.Schema({ name: String, email: String }) 69 | schema.plugin(markFieldsAsPII, { fields: 'email', key: KEY }) 70 | const Model = connection.model('Ciphered', schema) 71 | await Model.collection.insertMany(DOCS) 72 | 73 | await expect(convertDataForModel(Model)).resolves.toEqual(DOCS.length) 74 | 75 | for (const [index, { email }] of ( 76 | await Model.collection.find().toArray() 77 | ).entries()) { 78 | // It should look like a ciphertext -- this early check avoid cryptic errors 79 | // on deciphering later. 80 | expect(email).toMatch(/^[A-Za-z0-9+/]{25,}={0,2}$/) 81 | // Deciphering should work 82 | expect(decipher(KEY, email)).toEqual(DOCS[index].email) 83 | } 84 | }) 85 | 86 | it('should properly hash password fields', async () => { 87 | const schema = new mongoose.Schema({ name: String, password: String }) 88 | schema.plugin(markFieldsAsPII, { passwordFields: 'password' }) 89 | const Model = connection.model('Hashed', schema) 90 | await Model.collection.insertMany(DOCS) 91 | 92 | await expect(convertDataForModel(Model)).resolves.toEqual(DOCS.length) 93 | 94 | for (const [index, { password }] of ( 95 | await Model.collection.find().toArray() 96 | ).entries()) { 97 | // It should look like a hashed Bcrypt -- this early check avoid cryptic errors 98 | // on checkPassword later. 99 | expect(password).toMatch(/^\$2a\$\d{2}\$/) 100 | // It should be a valid hash of the source password 101 | await expect( 102 | checkPassword(DOCS[index].password, password) 103 | ).resolves.toEqual(true) 104 | } 105 | }) 106 | 107 | describe('when reporting progress', () => { 108 | describe('when reporting to the console because no emitter is passed', () => { 109 | const output = process.stderr 110 | let displayWidth 111 | let Model 112 | 113 | beforeAll(() => { 114 | const schema = new mongoose.Schema({ name: String, password: String }) 115 | schema.plugin(markFieldsAsPII, { passwordFields: 'password' }) 116 | Model = connection.model('Consoled', schema) 117 | displayWidth = output.columns 118 | }) 119 | 120 | afterAll(() => { 121 | output.columns = displayWidth 122 | }) 123 | 124 | beforeEach(async () => { 125 | await Model.collection.deleteMany({}) 126 | await Model.collection.insertMany(DOCS) 127 | }) 128 | 129 | it('should use a maximum of 100 chars on wider displays', async () => { 130 | output.columns = 160 131 | 132 | await convertDataForModel(Model) 133 | expect(pswMock.mock.calls).toEqual([ 134 | ['\n['], 135 | ['='.repeat(32)], 136 | ['='.repeat(33)], 137 | ['='.repeat(33)], 138 | [']\n'], 139 | ]) 140 | }) 141 | 142 | it('should ensure it doesn’t write too much on narrow displays', async () => { 143 | output.columns = 4 144 | 145 | await convertDataForModel(Model) 146 | expect(pswMock.mock.calls).toEqual([['\n['], ['='], ['='], [']\n']]) 147 | }) 148 | 149 | it('should default to 80 chars when display width is unknown', async () => { 150 | output.columns = undefined 151 | 152 | await convertDataForModel(Model) 153 | expect(pswMock.mock.calls).toEqual([ 154 | ['\n['], 155 | ['='.repeat(26)], 156 | ['='.repeat(25)], 157 | ['='.repeat(27)], 158 | [']\n'], 159 | ]) 160 | }) 161 | }) 162 | 163 | describe('when reporting as events because an emitter is passed', () => { 164 | let Model 165 | 166 | beforeAll(() => { 167 | const schema = new mongoose.Schema({ name: String, password: String }) 168 | schema.plugin(markFieldsAsPII, { passwordFields: 'password' }) 169 | Model = connection.model('Emitted', schema) 170 | }) 171 | 172 | beforeEach(async () => { 173 | await Model.collection.deleteMany({}) 174 | }) 175 | 176 | it('should emit both events on every doc for small-enough datasets', async () => { 177 | await Model.collection.insertMany(DOCS) 178 | const emitter = { emit: jest.fn().mockName('emit') } 179 | 180 | await convertDataForModel(Model, emitter) 181 | expect(emitter.emit.mock.calls).toEqual([ 182 | ['docs', 1], 183 | ['progress', 33], 184 | ['docs', 2], 185 | ['progress', 66], 186 | ['docs', 3], 187 | ['progress', 100], 188 | ]) 189 | }) 190 | 191 | it('should only emit progress events when the percentage changes', async () => { 192 | const dataset = new Array(110).fill(null).map((_, index) => ({ 193 | email: `john${index}@example.com`, 194 | name: `John ${index}`, 195 | password: 'secret', 196 | })) 197 | await Model.collection.insertMany(dataset) 198 | const emitter = { emit: jest.fn().mockName('emit') } 199 | 200 | await convertDataForModel(Model, emitter) 201 | const calls = emitter.emit.mock.calls 202 | 203 | // One doc call per document, with its one-based converted index 204 | const docCalls = calls.filter(([kind]) => kind === 'docs') 205 | expect(docCalls).toEqual( 206 | dataset.map((_, index) => ['docs', index + 1]) 207 | ) 208 | 209 | // One progress call per percentage change, with values from 1 to 100 210 | const progressCalls = calls.filter(([kind]) => kind === 'progress') 211 | expect(progressCalls).toEqual( 212 | dataset.slice(0, 100).map((_, index) => ['progress', index + 1]) 213 | ) 214 | }) 215 | }) 216 | }) 217 | }) 218 | }) 219 | -------------------------------------------------------------------------------- /util/passwords.js: -------------------------------------------------------------------------------- 1 | const bcrypt = require('bcryptjs') 2 | const { createHash } = require('crypto') 3 | 4 | const MAX_BCRYPT_USED_BYTES = 72 5 | const ROUNDS = process.env.NODE_ENV === 'production' ? 10 : 2 6 | 7 | async function checkPassword(clearText, hash) { 8 | return bcrypt.compare(clearText, hash) 9 | } 10 | 11 | function hashPassword(clearText, { rounds = ROUNDS, sync = false } = {}) { 12 | const buf = Buffer.from(clearText, 'utf8') 13 | if (buf.length > MAX_BCRYPT_USED_BYTES) { 14 | const processor = createHash('sha512') 15 | processor.update(clearText) 16 | clearText = processor.digest('base64') 17 | } 18 | 19 | return sync 20 | ? bcrypt.hashSync(clearText, rounds) 21 | : bcrypt.hash(clearText, rounds) 22 | } 23 | 24 | module.exports = { checkPassword, hashPassword } 25 | -------------------------------------------------------------------------------- /util/passwords.spec.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto') 2 | const spy = jest.spyOn(crypto, 'createHash') 3 | const { checkPassword, hashPassword } = require('./passwords') 4 | 5 | describe('Password Utils', () => { 6 | describe('checkPassword', () => { 7 | it('should resolve to true on a matching hash', () => { 8 | expect( 9 | checkPassword( 10 | 'secret', 11 | '$2a$04$VX4I2s9192QIuOLYdYw0aO.mc1GlnpgpLRzF8D7BpNxl/ficBIt4y' 12 | ) 13 | ).resolves.toBeTruthy() 14 | }) 15 | 16 | it('should resolve to false on a non-matching hash', () => { 17 | expect( 18 | checkPassword( 19 | 'secret', 20 | '$2a$04$VX4I2s9192QIuOLYdYw0aO.mc1GlnpgpLRzF8D7BpNxl/xxxxxxxx' 21 | ) 22 | ).resolves.toBeFalsy() 23 | 24 | expect(checkPassword('secret', '')).resolves.toBeFalsy() 25 | }) 26 | }) 27 | 28 | describe('hashPassword', () => { 29 | // function hashPassword(clearText, { rounds = ROUNDS, sync = false } = {}) 30 | it('should default to 4 rounds outside production', () => { 31 | expect(hashPassword('secret')).resolves.toMatch(/^\$2a\$04\$/) 32 | }) 33 | 34 | it('should default to 10 rounds in production', () => { 35 | jest.resetModules() 36 | const oldEnv = process.env.NODE_ENV 37 | process.env.NODE_ENV = 'production' 38 | try { 39 | const { hashPassword } = require('./passwords') 40 | expect(hashPassword('secret')).resolves.toMatch(/^\$2a\$10\$/) 41 | } finally { 42 | process.env.NODE_ENV = oldEnv 43 | } 44 | }) 45 | 46 | it('should accept custom rounds', () => { 47 | expect(hashPassword('secret', { rounds: 6 })).resolves.toMatch( 48 | /^\$2a\$06\$/ 49 | ) 50 | }) 51 | 52 | it('should allow synchronous usage', () => { 53 | expect(hashPassword('secret', { sync: true })).toMatch(/^\$2a\$04\$/) 54 | }) 55 | 56 | it('should SHA512 its input if it’s above maximum bcrypt input size, to preserve entropy', () => { 57 | spy.mockClear() 58 | const input = crypto.randomBytes(256).toString('utf8') 59 | hashPassword(input, { sync: true }) 60 | 61 | expect(spy).toHaveBeenCalledWith('sha512') 62 | }) 63 | }) 64 | }) 65 | --------------------------------------------------------------------------------