├── .babelrc ├── .editorconfig ├── .eslintignore ├── .eslintrc.js ├── .gitignore ├── .npmrc ├── .prettierrc.json ├── .vscode ├── extensions.json └── settings.json ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── scripts ├── assign ├── branches ├── delete ├── export ├── fe ├── force ├── rebase └── rename ├── src ├── .gitkeep ├── benchmarks.js └── key-value-store.js └── tests ├── .eslintrc.js ├── challenges └── key-value-store-simple.js └── key-value-store.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["@babel/env"] 3 | } 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | 11 | [Makefile] 12 | indent_style = tab 13 | indent_size = 2 14 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | node_modules/ 4 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { 4 | node: true, 5 | es6: true, 6 | }, 7 | extends: ['eslint:recommended', 'plugin:prettier/recommended', 'prettier/standard'], 8 | rules: { 9 | 'prettier/prettier': 'error', 10 | 'no-console': process.env.NODE_ENV === 'production' ? 'warn' : 'off', 11 | 'no-debugger': process.env.NODE_ENV === 'production' ? 'warn' : 'off', 12 | quotes: ['warn', 'single', { avoidEscape: true }], 13 | indent: ['warn', 2], 14 | semi: ['warn', 'never'], 15 | 'comma-dangle': [ 16 | 'warn', 17 | { 18 | arrays: 'always-multiline', 19 | objects: 'always-multiline', 20 | imports: 'never', 21 | exports: 'never', 22 | functions: 'ignore', 23 | }, 24 | ], 25 | }, 26 | parser: 'babel-eslint', 27 | parserOptions: { 28 | sourceType: 'module', 29 | ecmaVersion: 10, 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # node npm dependencies 7 | node_modules/ 8 | 9 | # dotenv environment variables file 10 | .env.local 11 | 12 | # BeyondCompare backups 13 | *.orig 14 | 15 | /db_temp/ 16 | /db_bench/ 17 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | save-exact=true 2 | engine-strict=true 3 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 120, 3 | "singleQuote": true, 4 | "semi": false, 5 | "trailingComma": "es5" 6 | } 7 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "coenraads.bracket-pair-colorizer", 4 | "dbaeumer.vscode-eslint", 5 | "drknoxy.eslint-disable-snippets", 6 | "editorconfig.editorconfig", 7 | "esbenp.prettier-vscode", 8 | "mgmcdermott.vscode-language-babel", 9 | "mikestead.dotenv", 10 | "ramyaraoa.show-offset", 11 | "tyriar.sort-lines" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "files.autoSave": "onFocusChange", 4 | "javascript.format.enable": false, 5 | "prettier.eslintIntegration": true, 6 | "prettier.requireConfig": true 7 | } 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 James Barton . All rights reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @learndb/learndb 2 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@learndb/learndb", 3 | "version": "0.1.1", 4 | "author": "James Barton ", 5 | "license": "ALL_RIGHTS_RESERVED", 6 | "private": true, 7 | "scripts": { 8 | "benchmarks": "node --require @babel/register src/benchmarks.js", 9 | "test": "run-s test:e2e", 10 | "test:e2e": "mocha --require @babel/register --full-trace tests/**/*.js", 11 | "test:challenges": "cross-env CHALLENGES=1 run-s test:e2e", 12 | "rebase": "node --require babel-polyfill --require @babel/register scripts/rebase.js" 13 | }, 14 | "dependencies": { 15 | "@babel/core": "7.4.5", 16 | "@babel/preset-env": "7.4.5", 17 | "@babel/register": "7.4.4", 18 | "babel-eslint": "10.0.2", 19 | "babel-polyfill": "6.26.0", 20 | "chai-string": "1.5.0", 21 | "cross-env": "5.2.0", 22 | "npm-run-all": "4.1.5", 23 | "string-hash": "1.1.3" 24 | }, 25 | "devDependencies": { 26 | "benchmark": "2.1.4", 27 | "chai": "4.2.0", 28 | "eslint": "5.16.0", 29 | "eslint-config-prettier": "5.0.0", 30 | "eslint-plugin-prettier": "3.1.0", 31 | "mocha": "6.1.4", 32 | "prettier": "1.18.2", 33 | "shelljs": "0.8.3" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /scripts/assign: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")/.." 4 | IFS=$'\t\n' 5 | 6 | tip='main' 7 | 8 | function main() { 9 | git switch -q "$tip" 10 | last_branch="$tip" 11 | 12 | git log --pretty=format:'%h %s' | while read -r log; do 13 | echo "log=$log" 14 | sha="$(cut -d' ' -f1 <<<"$log")" 15 | sub="$(cut -d' ' -f2 <<<"$log")" 16 | branch="${sub%:}" 17 | if [[ "$branch" != "$last_branch" ]]; then 18 | echo "Detected branch change for last_branch=$last_branch: Assigning branch=$branch to sha=$sha" 19 | git branch -f "$branch" "$sha" 20 | last_branch="$branch" 21 | fi 22 | done 23 | } 24 | 25 | main "$@" 26 | -------------------------------------------------------------------------------- /scripts/branches: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")/.." 4 | IFS=$'\t\n' 5 | 6 | for ref in $(git branch -r); do 7 | if ! [[ "$ref" =~ (kvs-.*) ]]; then 8 | continue 9 | fi 10 | 11 | branch="${BASH_REMATCH[1]}" 12 | 13 | echo "$branch" 14 | done 15 | -------------------------------------------------------------------------------- /scripts/delete: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")/.." 4 | IFS=$'\t\n' 5 | 6 | function nuke_branch() { 7 | usage='Usage: nuke_branch ' 8 | local remote="${1:?$usage}" branch="${2:?$usage}" 9 | 10 | current_branch="$(git rev-parse --abbrev-ref HEAD)" 11 | 12 | if [[ "$current_branch" == "$branch" ]]; then 13 | echo "You are currently on the branch you want to delete: $current_branch" 14 | echo "Git won't allow this, so you must change to another branch first" 15 | return 1 16 | fi 17 | 18 | if ! git remote | grep --color -Fx "$remote" &>/dev/null; then 19 | echo "The remote specified does not exist: $remote" 20 | echo "You can get a list of remotes and their URLs by running 'git remote -v'" 21 | return 1 22 | fi 23 | 24 | echo 25 | echo "Deleting local branch: $branch" 26 | if ! git branch --delete "$branch"; then 27 | echo "Failed to delete local branch -- it might not exist" 28 | fi 29 | 30 | echo 31 | echo "Deleting tracking branch: $remote/$branch" 32 | if ! git branch --delete --remotes "$remote/$branch"; then 33 | echo "Failed to delete tracking branch -- it might not exist" 34 | fi 35 | 36 | echo 37 | echo "Deleting remote branch: $branch on remote $remote" 38 | if ! git push "$remote" ":${branch}"; then 39 | echo "Failed to delete remote branch -- it might not exist" 40 | fi 41 | } 42 | 43 | function main() { 44 | for branch in "$@"; do 45 | nuke_branch origin "$branch" 46 | done 47 | } 48 | 49 | main "$@" 50 | -------------------------------------------------------------------------------- /scripts/export: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")/.." 4 | IFS=$'\t\n' 5 | 6 | function main() { 7 | for ref in $(git branch -r); do 8 | if ! [[ "$ref" =~ (kvs-.*) ]]; then 9 | continue 10 | fi 11 | 12 | branch="${BASH_REMATCH[1]}" 13 | 14 | echo "branch=$branch" 15 | 16 | git checkout "$branch" 17 | mkdir -p ../out/"$branch" 18 | rsync -a ./ ../out/"$branch"/ 19 | done 20 | 21 | git checkout main 22 | } 23 | 24 | main "$@" 25 | -------------------------------------------------------------------------------- /scripts/fe: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")/.." 4 | IFS=$'\t\n' 5 | 6 | function main() { 7 | last_branch='setup' 8 | 9 | for branch in $(scripts/branches); do 10 | git switch -q "$branch" 11 | echo "$branch" 12 | eval "$*" 13 | echo 14 | # shellcheck disable=2034 15 | last_branch="$branch" 16 | done 17 | } 18 | 19 | main "$@" 20 | -------------------------------------------------------------------------------- /scripts/force: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")/.." 4 | IFS=$'\t\n' 5 | 6 | if (($# > 0)); then 7 | echo "$0 does not take args, it runs on all branches except main" 8 | fi 9 | 10 | # shellcheck disable=SC2016 11 | scripts/fe 'git push --force-with-lease' 12 | -------------------------------------------------------------------------------- /scripts/rebase: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")/.." 4 | IFS=$'\t\n' 5 | 6 | git rebase --rerere-autoupdate -X ours "$@" 7 | -------------------------------------------------------------------------------- /scripts/rename: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | cd "$(dirname "$0")/.." 4 | IFS=$'\t\n' 5 | 6 | function nuke_branch() { 7 | usage='Usage: nuke_branch ' 8 | local remote="${1:?$usage}" branch="${2:?$usage}" 9 | 10 | current_branch="$(git rev-parse --abbrev-ref HEAD)" 11 | 12 | if [[ "$current_branch" == "$branch" ]]; then 13 | echo "You are currently on the branch you want to delete: $current_branch" 14 | echo "Git won't allow this, so you must change to another branch first" 15 | return 1 16 | fi 17 | 18 | if ! git remote | grep --color -Fx "$remote" &>/dev/null; then 19 | echo "The remote specified does not exist: $remote" 20 | echo "You can get a list of remotes and their URLs by running 'git remote -v'" 21 | return 1 22 | fi 23 | 24 | echo 25 | echo "Deleting local branch: $branch" 26 | if ! git branch --delete "$branch"; then 27 | echo "Failed to delete local branch -- it might not exist" 28 | fi 29 | 30 | echo 31 | echo "Deleting tracking branch: $remote/$branch" 32 | if ! git branch --delete --remotes "$remote/$branch"; then 33 | echo "Failed to delete tracking branch -- it might not exist" 34 | fi 35 | 36 | echo 37 | echo "Deleting remote branch: $branch on remote $remote" 38 | if ! git push "$remote" ":${branch}"; then 39 | echo "Failed to delete remote branch -- it might not exist" 40 | fi 41 | } 42 | 43 | function main() { 44 | local branch="${1:?}" 45 | 46 | local new_branch="${branch/key-value-store/ksv}" 47 | echo "branch=$branch new_branch=$new_branch" 48 | git checkout -b "$new_branch" 49 | git push -u origin HEAD 50 | nuke_branch origin "$branch" 51 | } 52 | 53 | main "$@" 54 | -------------------------------------------------------------------------------- /src/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/neodon/learndb2018/293bd31f61ebb45a04d5f8b7b9342c4ca9b43d6e/src/.gitkeep -------------------------------------------------------------------------------- /src/benchmarks.js: -------------------------------------------------------------------------------- 1 | import Benchmark from 'benchmark' 2 | import path from 'path' 3 | import shell from 'shelljs' 4 | import { KeyValueStore } from './key-value-store' 5 | 6 | const bytesPerMB = 1024 * 1024 7 | const maxItems = 5000 8 | 9 | function run() { 10 | const dbPath = path.resolve(__dirname, '../db_bench') 11 | 12 | if (!dbPath.includes('db_bench')) { 13 | throw new Error(`Refusing to run benchmarks for db path not containing 'db_bench': ${dbPath}`) 14 | } 15 | 16 | shell.rm('-rf', dbPath) 17 | shell.mkdir('-p', dbPath) 18 | 19 | const keyValueStore = new KeyValueStore({ dbPath, maxBufferLength: 10000 }) 20 | const suite = new Benchmark.Suite() 21 | 22 | const startingRss = process.memoryUsage().rss 23 | console.log(`Starting RSS memory usage: ${bytesToMB(startingRss)} MB`) 24 | 25 | keyValueStore.init() 26 | 27 | suite 28 | .add('KeyValueStore#set', function() { 29 | keyValueStore.set(Math.floor(Math.random() * maxItems).toString(), Math.random().toString()) 30 | }) 31 | .add('KeyValueStore#get', function() { 32 | keyValueStore.get(Math.floor(Math.random() * maxItems).toString()) 33 | }) 34 | .add('KeyValueStore#delete', function() { 35 | keyValueStore.delete(Math.floor(Math.random() * maxItems).toString()) 36 | }) 37 | .on('cycle', function(event) { 38 | console.log(String(event.target)) 39 | }) 40 | .on('complete', function() { 41 | keyValueStore.flush() 42 | const endingRss = process.memoryUsage().rss 43 | console.log(`Ending RSS memory usage: ${bytesToMB(endingRss)} MB`) 44 | console.log(`Difference: ${bytesToMB(endingRss - startingRss)} MB`) 45 | }) 46 | .on('error', function(err) { 47 | console.error(err) 48 | }) 49 | .run({ maxTime: 10, async: true }) 50 | } 51 | 52 | function bytesToMB(bytes) { 53 | return (bytes / bytesPerMB).toFixed(3) 54 | } 55 | 56 | export { run } 57 | 58 | if (!module.parent) { 59 | run() 60 | } 61 | -------------------------------------------------------------------------------- /src/key-value-store.js: -------------------------------------------------------------------------------- 1 | import path from 'path' 2 | import fs from 'fs' 3 | import shell from 'shelljs' 4 | 5 | const KEY_INDEX = 0 6 | const VALUE_INDEX = 1 7 | const IS_DELETED_INDEX = 2 8 | 9 | const SST_FILE_NAME_REGEXP = /^sorted_string_table_(\d+)[.]json$/ 10 | 11 | export class KeyValueStore { 12 | constructor({ dbPath, maxBufferLength }) { 13 | this.dbPath = dbPath 14 | this.maxBufferLength = maxBufferLength 15 | this.buffer = [] 16 | } 17 | 18 | init() { 19 | shell.mkdir('-p', this.dbPath) 20 | } 21 | 22 | set(key, value, isDeleted = false) { 23 | this.buffer.push([key, value, isDeleted]) 24 | 25 | if (this.buffer.length < this.maxBufferLength) { 26 | // The buffer isn't full yet, so we're done. 27 | return 28 | } 29 | 30 | this.flush() 31 | } 32 | 33 | flush() { 34 | if (this.buffer.length === 0) { 35 | return 36 | } 37 | 38 | // De-duplicate buffer entries for the same key, preserving only the last entry. 39 | const bufferObject = {} 40 | for (const entry of this.buffer) { 41 | bufferObject[entry[KEY_INDEX]] = entry 42 | } 43 | 44 | const sstFileName = this._generateNextSstFileName() 45 | 46 | // Flush the buffer to disk. 47 | fs.writeFileSync( 48 | path.resolve(this.dbPath, sstFileName), 49 | // Stringify the buffer entries, reverse sort them, and then join them into one string separated by newlines. 50 | Object.keys(bufferObject) 51 | .map(key => JSON.stringify(bufferObject[key])) 52 | .sort() 53 | .join('\n') 54 | ) 55 | 56 | this.buffer = [] 57 | } 58 | 59 | get(key) { 60 | // First, check the buffer for the newest entry with the key. 61 | const latestBufferEntryValue = this._findLatestBufferEntryValue(key) 62 | 63 | if (latestBufferEntryValue !== undefined) { 64 | // The key was found in an entry in the buffer, so we're done. 65 | return latestBufferEntryValue 66 | } 67 | 68 | // The key wasn't found in the buffer, so now we search the SST files. 69 | const sstFileNames = shell.ls(this.dbPath).filter(fileName => SST_FILE_NAME_REGEXP.test(fileName)) 70 | 71 | if (sstFileNames.length === 0) { 72 | // If there are no SST files, the key can't exist. 73 | return undefined 74 | } 75 | 76 | // We want to search the newest SSTs first so that we get the newest entry for the key. 77 | sstFileNames.reverse() 78 | 79 | // Search through the SST files, newest to oldest. 80 | for (const sstFileName of sstFileNames) { 81 | // Parse the SST file into an array of entries. It's the same structure as the buffer, but it's sorted. 82 | const entries = this._loadEntriesFromSstFile(sstFileName) 83 | const entryValue = this._findEntryValue(key, entries) 84 | 85 | if (entryValue !== undefined) { 86 | // The key was found in an entry in the current SST file, so we're done. 87 | return entryValue 88 | } 89 | } 90 | 91 | // The key was not found. 92 | return undefined 93 | } 94 | 95 | delete(key) { 96 | this.set(key, null, true) 97 | } 98 | 99 | checkAndSet({ key, expectedValue, newValue }) { 100 | if (this.get(key) === expectedValue) { 101 | this.set(key, newValue) 102 | return true 103 | } 104 | 105 | return false 106 | } 107 | 108 | _generateNextSstFileName() { 109 | const existingSstFileNames = shell.ls(this.dbPath).filter(fileName => SST_FILE_NAME_REGEXP.test(fileName)) 110 | 111 | if (existingSstFileNames.length === 0) { 112 | return 'sorted_string_table_0001.json' 113 | } 114 | 115 | // By default, ls returns a file list sorted by name. So we can use pop() to get the filename for the newest SST 116 | // file, which will also have the highest index. 117 | const lastSstFileName = existingSstFileNames.pop() 118 | 119 | // The regex matches the format of SST file names and extracts the index. 120 | const lastSstIndexString = SST_FILE_NAME_REGEXP.exec(lastSstFileName)[1] 121 | 122 | // We need to explicitly parse it to an Int before we can increment it. 123 | const lastSstIndex = parseInt(lastSstIndexString) 124 | const nextSstIndex = lastSstIndex + 1 125 | 126 | // E.g. 1 becomes '0001' and 123 becomes '0123'. 127 | const nextSstIndexPaddedString = nextSstIndex.toString().padStart(4, '0') 128 | 129 | const nextSstFileName = `sorted_string_table_${nextSstIndexPaddedString}.json` 130 | return nextSstFileName 131 | } 132 | 133 | _findLatestBufferEntryValue(key) { 134 | // Search the entries from most recent to oldest. 135 | for (let i = this.buffer.length - 1; i >= 0; i--) { 136 | if (this.buffer[i][KEY_INDEX] === key) { 137 | // We found the entry with the key in the buffer, so we're done. 138 | return this.buffer[i][IS_DELETED_INDEX] 139 | ? undefined // The isDeleted flag is set to true. 140 | : this.buffer[i][VALUE_INDEX] // Return the value for the key. 141 | } 142 | } 143 | 144 | // The key was not found in the buffer. 145 | return undefined 146 | } 147 | 148 | _loadEntriesFromSstFile(sstFileName) { 149 | // readFileSync returns a Buffer object that represents binary data. 150 | const buffer = fs.readFileSync(path.resolve(this.dbPath, sstFileName)) 151 | 152 | // Stringify the buffer so we can split it into lines. 153 | const bufferString = buffer.toString() 154 | 155 | // Split the buffer into lines, each representing an entry in JSON format. 156 | const lines = bufferString.trim().split('\n') 157 | 158 | // Parse the JSON in each line into an array representing an entry. 159 | const entries = lines.map(jsonLine => JSON.parse(jsonLine)) 160 | return entries 161 | } 162 | 163 | _findEntryValue(key, entries) { 164 | let entry = undefined 165 | let left = 0 166 | let right = entries.length - 1 167 | 168 | while (left <= right) { 169 | const mid = left + Math.floor((right - left) / 2) 170 | 171 | // We found the key. 172 | if (entries[mid][KEY_INDEX] === key) { 173 | entry = entries[mid] 174 | break 175 | } 176 | 177 | if (entries[mid][KEY_INDEX] > key) { 178 | // The key might exist in an entry before this entry. 179 | right = mid - 1 180 | } else { 181 | // The key might exist in an entry after this entry. 182 | left = mid + 1 183 | } 184 | } 185 | 186 | if (entry) { 187 | // We found the entry with the key in the sst file, so we're done. 188 | return entry[IS_DELETED_INDEX] 189 | ? undefined // The isDeleted flag is set to true. 190 | : entry[VALUE_INDEX] // Return the value for the key. 191 | } 192 | 193 | // The key was not found in the given entries. 194 | return undefined 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /tests/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: '../.eslintrc.js', 3 | env: { 4 | mocha: true, 5 | }, 6 | rules: { 7 | 'import/no-extraneous-dependencies': 'off', 8 | }, 9 | } 10 | -------------------------------------------------------------------------------- /tests/challenges/key-value-store-simple.js: -------------------------------------------------------------------------------- 1 | import chai, { assert } from 'chai' 2 | import chaiString from 'chai-string' 3 | import path from 'path' 4 | import shell from 'shelljs' 5 | import { KeyValueStore } from '../../src/key-value-store' 6 | 7 | chai.use(chaiString) 8 | 9 | const tests = () => { 10 | describe('Challenge', () => { 11 | // Test keys and values we'll use in the new tests 12 | const testKey1 = 'test-key-1' 13 | const testValue1 = 'test-value-1' 14 | const testValue2 = 'test-value-2' 15 | 16 | // The path where temporary db files will be written, in the project root. 17 | const dbTempPath = path.resolve(__dirname, '../../db_temp') 18 | 19 | // Contains a fresh instance of the key-value store for each test. 20 | let keyValueStore = null 21 | 22 | // Contains the path for the db files for the currently executing test. 23 | let dbPath = null 24 | 25 | // Allows us to give each test a unique directory. 26 | let testId = 1 27 | 28 | before(() => { 29 | // Safety check so we don't delete the wrong files 30 | assert.endsWith(dbTempPath, 'db_temp') 31 | shell.rm('-rf', dbTempPath) 32 | }) 33 | 34 | beforeEach(() => { 35 | // Generate a unique path in the project root to hold the db files for this test. 36 | dbPath = path.resolve(dbTempPath, process.pid.toString() + '_' + (testId++).toString()) 37 | shell.mkdir('-p', dbPath) 38 | 39 | // Before each test, create a new instance of the key-value store. 40 | keyValueStore = new KeyValueStore({ dbPath }) 41 | keyValueStore.init() 42 | }) 43 | 44 | // The point of checkAndSet() is to let us avoid accidentally clobbering data 45 | // from another client. Let's say client1 needs to add to a balance and client2 46 | // needs to substract from the balance. We don't want a race condition where 47 | // client1 retrieves the current balance, then client 2 retrieves the current balance, 48 | // then they both change the balance and write it back. Only the latest write 49 | // will "win" and the value of balance will be wrong. 50 | 51 | it('checkAndSet() returns true when expectedValue matches and value was set', () => { 52 | let newValueWasSet = keyValueStore.checkAndSet({ 53 | key: testKey1, 54 | expectedValue: undefined, 55 | newValue: testValue1, 56 | }) 57 | assert.isTrue(newValueWasSet) 58 | assert.equal(keyValueStore.get(testKey1), testValue1) 59 | 60 | newValueWasSet = keyValueStore.checkAndSet({ 61 | key: testKey1, 62 | expectedValue: testValue1, 63 | newValue: testValue2, 64 | }) 65 | assert.isTrue(newValueWasSet) 66 | assert.equal(keyValueStore.get(testKey1), testValue2) 67 | }) 68 | 69 | it('checkAndSet() returns false when expectedValue does not match and value was not set', () => { 70 | keyValueStore.set(testKey1, testValue1) 71 | 72 | let newValueWasSet = keyValueStore.checkAndSet({ 73 | key: testKey1, 74 | expectedValue: undefined, 75 | newValue: testValue2, 76 | }) 77 | assert.isFalse(newValueWasSet) 78 | assert.equal(keyValueStore.get(testKey1), testValue1) 79 | 80 | newValueWasSet = keyValueStore.checkAndSet({ key: testKey1, expectedValue: testValue2, newValue: 'foo' }) 81 | assert.isFalse(newValueWasSet) 82 | assert.equal(keyValueStore.get(testKey1), testValue1) 83 | }) 84 | 85 | // NOTE: If you get the 2 tests above passing, the test below should also pass. 86 | // It's not necessary to understand all of how it works right now. 87 | 88 | const expectedCounterValue = 10 89 | 90 | // This function attempts to increment a counter using checkAndSet(). It adds 91 | // a delay between reading and writing the counter in order to expose issues 92 | // with concurrency where another operation might try to increment the counter 93 | // at the same time. 94 | async function atomicIncrement() { 95 | let newValueWasSet = false 96 | do { 97 | const counterValue = keyValueStore.get('counter') 98 | const newCounterValue = counterValue + 1 99 | await new Promise(resolve => { 100 | setTimeout(() => { 101 | newValueWasSet = keyValueStore.checkAndSet({ 102 | key: 'counter', 103 | expectedValue: counterValue, 104 | newValue: newCounterValue, 105 | }) 106 | resolve() 107 | }, Math.random() * 100) 108 | }) 109 | } while (!newValueWasSet) 110 | } 111 | 112 | it('checkAndSet() enables concurrent atomic increment', async () => { 113 | keyValueStore.set('counter', 0) 114 | 115 | // Run expectedCounterValue (10) atomicIncrement() operations concurrently 116 | await Promise.all([...Array(expectedCounterValue)].map(atomicIncrement)) 117 | assert.equal( 118 | keyValueStore.get('counter'), 119 | expectedCounterValue, 120 | `expected counter value to reflect ${expectedCounterValue} atomic increments` 121 | ) 122 | }) 123 | }) 124 | } 125 | 126 | describe('KeyValueStore', process.env.CHALLENGES ? tests : () => {}) 127 | -------------------------------------------------------------------------------- /tests/key-value-store.js: -------------------------------------------------------------------------------- 1 | import chai, { assert } from 'chai' 2 | import chaiString from 'chai-string' 3 | import path from 'path' 4 | import shell from 'shelljs' 5 | import { KeyValueStore } from '../src/key-value-store' 6 | 7 | chai.use(chaiString) 8 | 9 | describe('KeyValueStore', () => { 10 | // Test keys and values we'll use in the new tests 11 | const testKey1 = 'test-key-1' 12 | const testValue1 = 'test-value-1' 13 | const testValue2 = 'test-value-2' 14 | 15 | // The path where temporary db files will be written, in the project root. 16 | const dbTempPath = path.resolve(__dirname, '../db_temp') 17 | 18 | // Contains a fresh instance of the key-value store for each test. 19 | let keyValueStore = null 20 | 21 | // Contains the path for the db files for the currently executing test. 22 | let dbPath = null 23 | 24 | // How many entries the buffer can store before they get flushed to disk 25 | let maxBufferLength = 3 26 | 27 | // Allows us to give each test a unique directory. 28 | let testId = 1 29 | 30 | // Functions passed to before() run only once, before any of the tests run. 31 | before(() => { 32 | // Safety check so we don't delete the wrong files 33 | assert.endsWith(dbTempPath, 'db_temp') 34 | shell.rm('-rf', dbTempPath) 35 | }) 36 | 37 | // Functions passed to beforeEach will run before every test. 38 | beforeEach(() => { 39 | // Generate a unique path in the project root to hold the db files for this test. 40 | dbPath = path.resolve(dbTempPath, process.pid.toString() + '_' + (testId++).toString()) 41 | shell.mkdir('-p', dbPath) 42 | 43 | // Before each test, create a new instance of the key-value store. 44 | keyValueStore = new KeyValueStore({ dbPath, maxBufferLength }) 45 | keyValueStore.init() 46 | }) 47 | 48 | it('get() returns value that was set()', () => { 49 | keyValueStore.set(testKey1, testValue1) 50 | assert.equal(keyValueStore.get(testKey1), testValue1) 51 | }) 52 | 53 | it('get() returns last value that was set()', () => { 54 | keyValueStore.set(testKey1, testValue1) 55 | keyValueStore.set(testKey1, testValue2) 56 | assert.equal(keyValueStore.get(testKey1), testValue2) 57 | }) 58 | 59 | it('get() for non-existent key returns undefined', () => { 60 | assert.equal(keyValueStore.get(testKey1), undefined) 61 | }) 62 | 63 | it('set() and get() support null value', () => { 64 | keyValueStore.set(testKey1, null) 65 | assert.equal(keyValueStore.get(testKey1), null) 66 | }) 67 | 68 | it('delete() for key causes get() to return undefined', () => { 69 | keyValueStore.set(testKey1, testValue1) 70 | keyValueStore.delete(testKey1) 71 | assert.equal(keyValueStore.get(testKey1), undefined) 72 | }) 73 | 74 | it('flush() flushes buffer to disk', () => { 75 | assert.equal(shell.ls(dbPath).length, 0, 'no SST files should exist in dbPath yet') 76 | keyValueStore.set('test-key-3', 'test-value-3') 77 | 78 | assert.equal(shell.ls(dbPath).length, 0, 'no SST files should exist in dbPath yet') 79 | keyValueStore.flush() 80 | 81 | assert.equal(shell.ls(dbPath).length, 1, 'buffer should be flushed to disk as sorted_string_table_0001.json') 82 | assert.lengthOf(keyValueStore.buffer, 0, 'the buffer should be emptied after flushing to disk') 83 | }) 84 | 85 | it('set() flushes buffer to disk after maxBufferLength (3) entries', () => { 86 | assert.equal(shell.ls(dbPath).length, 0, 'no SST files should exist in dbPath yet') 87 | keyValueStore.set('test-key-2', 'test-value-2') 88 | 89 | assert.equal(shell.ls(dbPath).length, 0, 'no SST files should exist in dbPath yet') 90 | keyValueStore.set('test-key-1', 'test-value-1') 91 | 92 | assert.equal(shell.ls(dbPath).length, 0, 'no SST files should exist in dbPath yet') 93 | keyValueStore.set('test-key-3', 'test-value-3') 94 | 95 | assert.equal(shell.ls(dbPath).length, 1, 'buffer should be flushed to disk as sorted_string_table_0001.json') 96 | assert.lengthOf(keyValueStore.buffer, 0, 'the buffer should be emptied after flushing to disk') 97 | 98 | assert.equal(shell.ls(dbPath).length, 1, 'first SST file should exist in dbPath') 99 | keyValueStore.set('test-key-2', 'test-value-2') 100 | 101 | assert.equal(shell.ls(dbPath).length, 1, 'first SST file should exist in dbPath') 102 | keyValueStore.set('test-key-1', 'test-value-1') 103 | 104 | assert.equal(shell.ls(dbPath).length, 1, 'first SST file should exist in dbPath') 105 | keyValueStore.set('test-key-3', 'test-value-3') 106 | 107 | assert.equal(shell.ls(dbPath).length, 2, 'buffer should be flushed to disk as sorted_string_table_0002.json') 108 | assert.lengthOf(keyValueStore.buffer, 0, 'the buffer should be emptied after flushing to disk') 109 | 110 | const expectedEntries = [ 111 | ['test-key-1', 'test-value-1', false], 112 | ['test-key-2', 'test-value-2', false], 113 | ['test-key-3', 'test-value-3', false], 114 | ] 115 | 116 | const expectedSortedStringTableContent = expectedEntries.map(JSON.stringify).join('\n') 117 | 118 | const actualSortedStringTableContent1 = shell.cat(path.resolve(dbPath, 'sorted_string_table_0001.json')).stdout 119 | assert.equal(actualSortedStringTableContent1, expectedSortedStringTableContent) 120 | 121 | const actualSortedStringTableContent2 = shell.cat(path.resolve(dbPath, 'sorted_string_table_0002.json')).stdout 122 | assert.equal(actualSortedStringTableContent2, expectedSortedStringTableContent) 123 | }) 124 | }) 125 | --------------------------------------------------------------------------------