├── .dockerignore ├── .eslintrc ├── .github ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── dependabot.yml ├── .gitignore ├── .mocharc.json ├── .nvmrc ├── .prettierrc ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── app.js ├── app └── js │ ├── DeleteQueueManager.js │ ├── DiffCodec.js │ ├── DispatchManager.js │ ├── DocumentManager.js │ ├── Errors.js │ ├── HistoryManager.js │ ├── HistoryRedisManager.js │ ├── HttpController.js │ ├── LockManager.js │ ├── LoggerSerializers.js │ ├── Metrics.js │ ├── PersistenceManager.js │ ├── Profiler.js │ ├── ProjectFlusher.js │ ├── ProjectHistoryRedisManager.js │ ├── ProjectManager.js │ ├── RangesManager.js │ ├── RangesTracker.js │ ├── RateLimitManager.js │ ├── RealTimeRedisManager.js │ ├── RedisManager.js │ ├── ShareJsDB.js │ ├── ShareJsUpdateManager.js │ ├── SnapshotManager.js │ ├── UpdateKeys.js │ ├── UpdateManager.js │ ├── mongodb.js │ └── sharejs │ ├── README.md │ ├── count.js │ ├── helpers.js │ ├── index.js │ ├── json-api.js │ ├── json.js │ ├── model.js │ ├── server │ ├── model.js │ └── syncqueue.js │ ├── simple.js │ ├── syncqueue.js │ ├── text-api.js │ ├── text-composable-api.js │ ├── text-composable.js │ ├── text-tp2-api.js │ ├── text-tp2.js │ ├── text.js │ ├── types │ ├── count.js │ ├── helpers.js │ ├── index.js │ ├── json-api.js │ ├── json.js │ ├── model.js │ ├── simple.js │ ├── syncqueue.js │ ├── text-api.js │ ├── text-composable-api.js │ ├── text-composable.js │ ├── text-tp2-api.js │ ├── text-tp2.js │ ├── text.js │ └── web-prelude.js │ └── web-prelude.js ├── benchmarks └── multi_vs_mget_mset.rb ├── buildscript.txt ├── config └── settings.defaults.js ├── docker-compose.ci.yml ├── docker-compose.yml ├── expire_docops.js ├── nodemon.json ├── package-lock.json ├── package.json ├── redis_cluster ├── 7000 │ └── redis.conf ├── 7001 │ └── redis.conf ├── 7002 │ └── redis.conf ├── 7003 │ └── redis.conf ├── 7004 │ └── redis.conf ├── 7005 │ └── redis.conf ├── redis-cluster.sh └── redis-trib.rb └── test ├── acceptance └── js │ ├── ApplyingUpdatesToADocTests.js │ ├── ApplyingUpdatesToProjectStructureTests.js │ ├── DeletingADocumentTests.js │ ├── DeletingAProjectTests.js │ ├── FlushingAProjectTests.js │ ├── FlushingDocsTests.js │ ├── GettingADocumentTests.js │ ├── GettingProjectDocsTests.js │ ├── PeekingADoc.js │ ├── RangesTests.js │ ├── SettingADocumentTests.js │ ├── SizeCheckTests.js │ └── helpers │ ├── DocUpdaterApp.js │ ├── DocUpdaterClient.js │ ├── MockProjectHistoryApi.js │ ├── MockTrackChangesApi.js │ └── MockWebApi.js ├── cluster_failover └── js │ ├── test_blpop_failover.js │ └── test_pubsub_failover.js ├── setup.js ├── stress └── js │ └── run.js └── unit └── js ├── DiffCodec └── DiffCodecTests.js ├── DispatchManager └── DispatchManagerTests.js ├── DocumentManager └── DocumentManagerTests.js ├── HistoryManager └── HistoryManagerTests.js ├── HistoryRedisManager └── HistoryRedisManagerTests.js ├── HttpController └── HttpControllerTests.js ├── LockManager ├── CheckingTheLock.js ├── ReleasingTheLock.js ├── getLockTests.js └── tryLockTests.js ├── PersistenceManager └── PersistenceManagerTests.js ├── ProjectHistoryRedisManager └── ProjectHistoryRedisManagerTests.js ├── ProjectManager ├── flushAndDeleteProjectTests.js ├── flushProjectTests.js ├── getProjectDocsTests.js └── updateProjectTests.js ├── RangesManager └── RangesManagerTests.js ├── RateLimitManager └── RateLimitManager.js ├── RealTimeRedisManager └── RealTimeRedisManagerTests.js ├── RedisManager └── RedisManagerTests.js ├── ShareJS └── TextTransformTests.js ├── ShareJsDB └── ShareJsDBTests.js ├── ShareJsUpdateManager └── ShareJsUpdateManagerTests.js └── UpdateManager └── UpdateManagerTests.js /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/* 2 | gitrev 3 | .git 4 | .gitignore 5 | .npm 6 | .nvmrc 7 | nodemon.json 8 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | // this file was auto-generated, do not edit it directly. 2 | // instead run bin/update_build_scripts from 3 | // https://github.com/sharelatex/sharelatex-dev-environment 4 | { 5 | "extends": [ 6 | "eslint:recommended", 7 | "standard", 8 | "prettier" 9 | ], 10 | "parserOptions": { 11 | "ecmaVersion": 2018 12 | }, 13 | "plugins": [ 14 | "mocha", 15 | "chai-expect", 16 | "chai-friendly" 17 | ], 18 | "env": { 19 | "node": true, 20 | "mocha": true 21 | }, 22 | "rules": { 23 | // TODO(das7pad): remove overrides after fixing all the violations manually (https://github.com/overleaf/issues/issues/3882#issuecomment-878999671) 24 | // START of temporary overrides 25 | "array-callback-return": "off", 26 | "no-dupe-else-if": "off", 27 | "no-var": "off", 28 | "no-empty": "off", 29 | "node/handle-callback-err": "off", 30 | "no-loss-of-precision": "off", 31 | "node/no-callback-literal": "off", 32 | "node/no-path-concat": "off", 33 | "prefer-regex-literals": "off", 34 | // END of temporary overrides 35 | 36 | // Swap the no-unused-expressions rule with a more chai-friendly one 37 | "no-unused-expressions": 0, 38 | "chai-friendly/no-unused-expressions": "error", 39 | 40 | // Do not allow importing of implicit dependencies. 41 | "import/no-extraneous-dependencies": "error" 42 | }, 43 | "overrides": [ 44 | { 45 | // Test specific rules 46 | "files": ["test/**/*.js"], 47 | "globals": { 48 | "expect": true 49 | }, 50 | "rules": { 51 | // mocha-specific rules 52 | "mocha/handle-done-callback": "error", 53 | "mocha/no-exclusive-tests": "error", 54 | "mocha/no-global-tests": "error", 55 | "mocha/no-identical-title": "error", 56 | "mocha/no-nested-tests": "error", 57 | "mocha/no-pending-tests": "error", 58 | "mocha/no-skipped-tests": "error", 59 | "mocha/no-mocha-arrows": "error", 60 | 61 | // chai-specific rules 62 | "chai-expect/missing-assertion": "error", 63 | "chai-expect/terminating-properties": "error", 64 | 65 | // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests. 66 | // we don't enforce this at the top-level - just in tests to manage `this` scope 67 | // based on mocha's context mechanism 68 | "mocha/prefer-arrow-callback": "error" 69 | } 70 | }, 71 | { 72 | // Backend specific rules 73 | "files": ["app/**/*.js", "app.js", "index.js"], 74 | "rules": { 75 | // don't allow console.log in backend code 76 | "no-console": "error", 77 | 78 | // Do not allow importing of implicit dependencies. 79 | "import/no-extraneous-dependencies": ["error", { 80 | // Do not allow importing of devDependencies. 81 | "devDependencies": false 82 | }] 83 | } 84 | } 85 | ] 86 | } 87 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Steps to Reproduce 4 | 5 | 6 | 7 | 1. 8 | 2. 9 | 3. 10 | 11 | ## Expected Behaviour 12 | 13 | 14 | ## Observed Behaviour 15 | 16 | 17 | 18 | ## Context 19 | 20 | 21 | ## Technical Info 22 | 23 | 24 | * URL: 25 | * Browser Name and version: 26 | * Operating System and version (desktop or mobile): 27 | * Signed in as: 28 | * Project and/or file: 29 | 30 | ## Analysis 31 | 32 | 33 | ## Who Needs to Know? 34 | 35 | 36 | 37 | - 38 | - 39 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ### Description 7 | 8 | 9 | 10 | #### Screenshots 11 | 12 | 13 | 14 | #### Related Issues / PRs 15 | 16 | 17 | 18 | ### Review 19 | 20 | 21 | 22 | #### Potential Impact 23 | 24 | 25 | 26 | #### Manual Testing Performed 27 | 28 | - [ ] 29 | - [ ] 30 | 31 | #### Accessibility 32 | 33 | 34 | 35 | ### Deployment 36 | 37 | 38 | 39 | #### Deployment Checklist 40 | 41 | - [ ] Update documentation not included in the PR (if any) 42 | - [ ] 43 | 44 | #### Metrics and Monitoring 45 | 46 | 47 | 48 | #### Who Needs to Know? 49 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "npm" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | 8 | pull-request-branch-name: 9 | # Separate sections of the branch name with a hyphen 10 | # Docker images use the branch name and do not support slashes in tags 11 | # https://github.com/overleaf/google-ops/issues/822 12 | # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#pull-request-branch-nameseparator 13 | separator: "-" 14 | 15 | # Block informal upgrades -- security upgrades use a separate queue. 16 | # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates#open-pull-requests-limit 17 | open-pull-requests-limit: 0 18 | 19 | # currently assign team-magma to all dependabot PRs - this may change in 20 | # future if we reorganise teams 21 | labels: 22 | - "dependencies" 23 | - "type:maintenance" 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | compileFolder 2 | 3 | Compiled source # 4 | ################### 5 | *.com 6 | *.class 7 | *.dll 8 | *.exe 9 | *.o 10 | *.so 11 | 12 | # Packages # 13 | ############ 14 | # it's better to unpack these files and commit the raw source 15 | # git has its own built in compression methods 16 | *.7z 17 | *.dmg 18 | *.gz 19 | *.iso 20 | *.jar 21 | *.rar 22 | *.tar 23 | *.zip 24 | 25 | # Logs and databases # 26 | ###################### 27 | *.log 28 | *.sql 29 | *.sqlite 30 | 31 | # OS generated files # 32 | ###################### 33 | .DS_Store? 34 | ehthumbs.db 35 | Icon? 36 | Thumbs.db 37 | 38 | /node_modules/* 39 | 40 | 41 | 42 | forever/ 43 | 44 | **.swp 45 | 46 | # Redis cluster 47 | **/appendonly.aof 48 | **/dump.rdb 49 | **/nodes.conf 50 | 51 | # managed by dev-environment$ bin/update_build_scripts 52 | .npmrc 53 | -------------------------------------------------------------------------------- /.mocharc.json: -------------------------------------------------------------------------------- 1 | { 2 | "require": "test/setup.js" 3 | } 4 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 12.22.3 2 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | # This file was auto-generated, do not edit it directly. 2 | # Instead run bin/update_build_scripts from 3 | # https://github.com/sharelatex/sharelatex-dev-environment 4 | { 5 | "arrowParens": "avoid", 6 | "semi": false, 7 | "singleQuote": true, 8 | "trailingComma": "es5", 9 | "tabWidth": 2, 10 | "useTabs": false 11 | } 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # This file was auto-generated, do not edit it directly. 2 | # Instead run bin/update_build_scripts from 3 | # https://github.com/sharelatex/sharelatex-dev-environment 4 | 5 | FROM node:12.22.3 as base 6 | 7 | WORKDIR /app 8 | 9 | FROM base as app 10 | 11 | #wildcard as some files may not be in all repos 12 | COPY package*.json npm-shrink*.json /app/ 13 | 14 | RUN npm ci --quiet 15 | 16 | COPY . /app 17 | 18 | FROM base 19 | 20 | COPY --from=app /app /app 21 | USER node 22 | 23 | CMD ["node", "--expose-gc", "app.js"] 24 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This file was auto-generated, do not edit it directly. 2 | # Instead run bin/update_build_scripts from 3 | # https://github.com/sharelatex/sharelatex-dev-environment 4 | 5 | BUILD_NUMBER ?= local 6 | BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) 7 | PROJECT_NAME = document-updater 8 | BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') 9 | 10 | DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml 11 | DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ 12 | BRANCH_NAME=$(BRANCH_NAME) \ 13 | PROJECT_NAME=$(PROJECT_NAME) \ 14 | MOCHA_GREP=${MOCHA_GREP} \ 15 | docker-compose ${DOCKER_COMPOSE_FLAGS} 16 | 17 | DOCKER_COMPOSE_TEST_ACCEPTANCE = \ 18 | COMPOSE_PROJECT_NAME=test_acceptance_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) 19 | 20 | DOCKER_COMPOSE_TEST_UNIT = \ 21 | COMPOSE_PROJECT_NAME=test_unit_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) 22 | 23 | clean: 24 | -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) 25 | -docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) 26 | -$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local 27 | -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local 28 | 29 | format: 30 | $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format 31 | 32 | format_fix: 33 | $(DOCKER_COMPOSE) run --rm test_unit npm run --silent format:fix 34 | 35 | lint: 36 | $(DOCKER_COMPOSE) run --rm test_unit npm run --silent lint 37 | 38 | test: format lint test_unit test_acceptance 39 | 40 | test_unit: 41 | ifneq (,$(wildcard test/unit)) 42 | $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit 43 | $(MAKE) test_unit_clean 44 | endif 45 | 46 | test_clean: test_unit_clean 47 | test_unit_clean: 48 | ifneq (,$(wildcard test/unit)) 49 | $(DOCKER_COMPOSE_TEST_UNIT) down -v -t 0 50 | endif 51 | 52 | test_acceptance: test_acceptance_clean test_acceptance_pre_run test_acceptance_run 53 | $(MAKE) test_acceptance_clean 54 | 55 | test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_acceptance_run_debug 56 | $(MAKE) test_acceptance_clean 57 | 58 | test_acceptance_run: 59 | ifneq (,$(wildcard test/acceptance)) 60 | $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance 61 | endif 62 | 63 | test_acceptance_run_debug: 64 | ifneq (,$(wildcard test/acceptance)) 65 | $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk 66 | endif 67 | 68 | test_clean: test_acceptance_clean 69 | test_acceptance_clean: 70 | $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 71 | 72 | test_acceptance_pre_run: 73 | ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) 74 | $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run 75 | endif 76 | 77 | build: 78 | docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ 79 | --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ 80 | . 81 | 82 | tar: 83 | $(DOCKER_COMPOSE) up tar 84 | 85 | publish: 86 | 87 | docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) 88 | 89 | 90 | .PHONY: clean test test_unit test_acceptance test_clean build publish 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ⚠️ This repository has been migrated into [`overleaf/overleaf`](https://github.com/overleaf/overleaf). See the [monorepo announcement](https://github.com/overleaf/overleaf/issues/923) for more info. ⚠️ 2 | 3 | --- 4 | 5 | overleaf/document-updater 6 | =========================== 7 | 8 | An API for applying incoming updates to documents in real-time. 9 | 10 | License 11 | ------- 12 | 13 | The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the `LICENSE` file. 14 | 15 | Copyright (c) Overleaf, 2014-2019. 16 | 17 | -------------------------------------------------------------------------------- /app/js/DeleteQueueManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS101: Remove unnecessary use of Array.from 11 | * DS102: Remove unnecessary code created because of implicit returns 12 | * DS207: Consider shorter variations of null checks 13 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 14 | */ 15 | let DeleteQueueManager 16 | const Settings = require('@overleaf/settings') 17 | const RedisManager = require('./RedisManager') 18 | const ProjectManager = require('./ProjectManager') 19 | const logger = require('logger-sharelatex') 20 | const metrics = require('./Metrics') 21 | const async = require('async') 22 | 23 | // Maintain a sorted set of project flushAndDelete requests, ordered by timestamp 24 | // (ZADD), and process them from oldest to newest. A flushAndDelete request comes 25 | // from real-time and is triggered when a user leaves a project. 26 | // 27 | // The aim is to remove the project from redis 5 minutes after the last request 28 | // if there has been no activity (document updates) in that time. If there is 29 | // activity we can expect a further flushAndDelete request when the editing user 30 | // leaves the project. 31 | // 32 | // If a new flushAndDelete request comes in while an existing request is already 33 | // in the queue we update the timestamp as we can postpone flushing further. 34 | // 35 | // Documents are processed by checking the queue, seeing if the first entry is 36 | // older than 5 minutes, and popping it from the queue in that case. 37 | 38 | module.exports = DeleteQueueManager = { 39 | flushAndDeleteOldProjects(options, callback) { 40 | const startTime = Date.now() 41 | const cutoffTime = 42 | startTime - options.min_delete_age + 100 * (Math.random() - 0.5) 43 | let count = 0 44 | 45 | const flushProjectIfNotModified = (project_id, flushTimestamp, cb) => 46 | ProjectManager.getProjectDocsTimestamps( 47 | project_id, 48 | function (err, timestamps) { 49 | if (err != null) { 50 | return callback(err) 51 | } 52 | if (timestamps.length === 0) { 53 | logger.log( 54 | { project_id }, 55 | 'skipping flush of queued project - no timestamps' 56 | ) 57 | return cb() 58 | } 59 | // are any of the timestamps newer than the time the project was flushed? 60 | for (const timestamp of Array.from(timestamps)) { 61 | if (timestamp > flushTimestamp) { 62 | metrics.inc('queued-delete-skipped') 63 | logger.debug( 64 | { project_id, timestamps, flushTimestamp }, 65 | 'found newer timestamp, will skip delete' 66 | ) 67 | return cb() 68 | } 69 | } 70 | logger.log({ project_id, flushTimestamp }, 'flushing queued project') 71 | return ProjectManager.flushAndDeleteProjectWithLocks( 72 | project_id, 73 | { skip_history_flush: false }, 74 | function (err) { 75 | if (err != null) { 76 | logger.err({ project_id, err }, 'error flushing queued project') 77 | } 78 | metrics.inc('queued-delete-completed') 79 | return cb(null, true) 80 | } 81 | ) 82 | } 83 | ) 84 | 85 | var flushNextProject = function () { 86 | const now = Date.now() 87 | if (now - startTime > options.timeout) { 88 | logger.log('hit time limit on flushing old projects') 89 | return callback(null, count) 90 | } 91 | if (count > options.limit) { 92 | logger.log('hit count limit on flushing old projects') 93 | return callback(null, count) 94 | } 95 | return RedisManager.getNextProjectToFlushAndDelete( 96 | cutoffTime, 97 | function (err, project_id, flushTimestamp, queueLength) { 98 | if (err != null) { 99 | return callback(err) 100 | } 101 | if (project_id == null) { 102 | return callback(null, count) 103 | } 104 | logger.log({ project_id, queueLength }, 'flushing queued project') 105 | metrics.globalGauge('queued-flush-backlog', queueLength) 106 | return flushProjectIfNotModified( 107 | project_id, 108 | flushTimestamp, 109 | function (err, flushed) { 110 | if (flushed) { 111 | count++ 112 | } 113 | return flushNextProject() 114 | } 115 | ) 116 | } 117 | ) 118 | } 119 | 120 | return flushNextProject() 121 | }, 122 | 123 | startBackgroundFlush() { 124 | const SHORT_DELAY = 10 125 | const LONG_DELAY = 1000 126 | var doFlush = function () { 127 | if (Settings.shuttingDown) { 128 | logger.warn('discontinuing background flush due to shutdown') 129 | return 130 | } 131 | return DeleteQueueManager.flushAndDeleteOldProjects( 132 | { 133 | timeout: 1000, 134 | min_delete_age: 3 * 60 * 1000, 135 | limit: 1000, // high value, to ensure we always flush enough projects 136 | }, 137 | (err, flushed) => 138 | setTimeout(doFlush, flushed > 10 ? SHORT_DELAY : LONG_DELAY) 139 | ) 140 | } 141 | return doFlush() 142 | }, 143 | } 144 | -------------------------------------------------------------------------------- /app/js/DiffCodec.js: -------------------------------------------------------------------------------- 1 | const DMP = require('diff-match-patch') 2 | const dmp = new DMP() 3 | 4 | // Do not attempt to produce a diff for more than 100ms 5 | dmp.Diff_Timeout = 0.1 6 | 7 | module.exports = { 8 | ADDED: 1, 9 | REMOVED: -1, 10 | UNCHANGED: 0, 11 | 12 | diffAsShareJsOp(before, after, callback) { 13 | const diffs = dmp.diff_main(before.join('\n'), after.join('\n')) 14 | dmp.diff_cleanupSemantic(diffs) 15 | 16 | const ops = [] 17 | let position = 0 18 | for (const diff of diffs) { 19 | const type = diff[0] 20 | const content = diff[1] 21 | if (type === this.ADDED) { 22 | ops.push({ 23 | i: content, 24 | p: position, 25 | }) 26 | position += content.length 27 | } else if (type === this.REMOVED) { 28 | ops.push({ 29 | d: content, 30 | p: position, 31 | }) 32 | } else if (type === this.UNCHANGED) { 33 | position += content.length 34 | } else { 35 | throw new Error('Unknown type') 36 | } 37 | } 38 | callback(null, ops) 39 | }, 40 | } 41 | -------------------------------------------------------------------------------- /app/js/DispatchManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS101: Remove unnecessary use of Array.from 11 | * DS102: Remove unnecessary code created because of implicit returns 12 | * DS202: Simplify dynamic range loops 13 | * DS205: Consider reworking code to avoid use of IIFEs 14 | * DS207: Consider shorter variations of null checks 15 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 16 | */ 17 | let DispatchManager 18 | const Settings = require('@overleaf/settings') 19 | const logger = require('logger-sharelatex') 20 | const Keys = require('./UpdateKeys') 21 | const redis = require('@overleaf/redis-wrapper') 22 | const Errors = require('./Errors') 23 | const _ = require('lodash') 24 | 25 | const UpdateManager = require('./UpdateManager') 26 | const Metrics = require('./Metrics') 27 | const RateLimitManager = require('./RateLimitManager') 28 | 29 | module.exports = DispatchManager = { 30 | createDispatcher(RateLimiter, queueShardNumber) { 31 | let pendingListKey 32 | if (queueShardNumber === 0) { 33 | pendingListKey = 'pending-updates-list' 34 | } else { 35 | pendingListKey = `pending-updates-list-${queueShardNumber}` 36 | } 37 | 38 | const client = redis.createClient(Settings.redis.documentupdater) 39 | var worker = { 40 | client, 41 | _waitForUpdateThenDispatchWorker(callback) { 42 | if (callback == null) { 43 | callback = function (error) {} 44 | } 45 | const timer = new Metrics.Timer('worker.waiting') 46 | return worker.client.blpop(pendingListKey, 0, function (error, result) { 47 | logger.log(`getting ${queueShardNumber}`, error, result) 48 | timer.done() 49 | if (error != null) { 50 | return callback(error) 51 | } 52 | if (result == null) { 53 | return callback() 54 | } 55 | const [list_name, doc_key] = Array.from(result) 56 | const [project_id, doc_id] = Array.from( 57 | Keys.splitProjectIdAndDocId(doc_key) 58 | ) 59 | // Dispatch this in the background 60 | const backgroundTask = cb => 61 | UpdateManager.processOutstandingUpdatesWithLock( 62 | project_id, 63 | doc_id, 64 | function (error) { 65 | // log everything except OpRangeNotAvailable errors, these are normal 66 | if (error != null) { 67 | // downgrade OpRangeNotAvailable and "Delete component" errors so they are not sent to sentry 68 | const logAsWarning = 69 | error instanceof Errors.OpRangeNotAvailableError || 70 | error instanceof Errors.DeleteMismatchError 71 | if (logAsWarning) { 72 | logger.warn( 73 | { err: error, project_id, doc_id }, 74 | 'error processing update' 75 | ) 76 | } else { 77 | logger.error( 78 | { err: error, project_id, doc_id }, 79 | 'error processing update' 80 | ) 81 | } 82 | } 83 | return cb() 84 | } 85 | ) 86 | return RateLimiter.run(backgroundTask, callback) 87 | }) 88 | }, 89 | 90 | run() { 91 | if (Settings.shuttingDown) { 92 | return 93 | } 94 | return worker._waitForUpdateThenDispatchWorker(error => { 95 | if (error != null) { 96 | logger.error({ err: error }, 'Error in worker process') 97 | throw error 98 | } else { 99 | return worker.run() 100 | } 101 | }) 102 | }, 103 | } 104 | 105 | return worker 106 | }, 107 | 108 | createAndStartDispatchers(number) { 109 | const RateLimiter = new RateLimitManager(number) 110 | _.times(number, function (shardNumber) { 111 | return DispatchManager.createDispatcher(RateLimiter, shardNumber).run() 112 | }) 113 | }, 114 | } 115 | -------------------------------------------------------------------------------- /app/js/Errors.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-proto, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | let Errors 8 | var NotFoundError = function (message) { 9 | const error = new Error(message) 10 | error.name = 'NotFoundError' 11 | error.__proto__ = NotFoundError.prototype 12 | return error 13 | } 14 | NotFoundError.prototype.__proto__ = Error.prototype 15 | 16 | var OpRangeNotAvailableError = function (message) { 17 | const error = new Error(message) 18 | error.name = 'OpRangeNotAvailableError' 19 | error.__proto__ = OpRangeNotAvailableError.prototype 20 | return error 21 | } 22 | OpRangeNotAvailableError.prototype.__proto__ = Error.prototype 23 | 24 | var ProjectStateChangedError = function (message) { 25 | const error = new Error(message) 26 | error.name = 'ProjectStateChangedError' 27 | error.__proto__ = ProjectStateChangedError.prototype 28 | return error 29 | } 30 | ProjectStateChangedError.prototype.__proto__ = Error.prototype 31 | 32 | var DeleteMismatchError = function (message) { 33 | const error = new Error(message) 34 | error.name = 'DeleteMismatchError' 35 | error.__proto__ = DeleteMismatchError.prototype 36 | return error 37 | } 38 | DeleteMismatchError.prototype.__proto__ = Error.prototype 39 | 40 | module.exports = Errors = { 41 | NotFoundError, 42 | OpRangeNotAvailableError, 43 | ProjectStateChangedError, 44 | DeleteMismatchError, 45 | } 46 | -------------------------------------------------------------------------------- /app/js/HistoryRedisManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | let HistoryRedisManager 15 | const Settings = require('@overleaf/settings') 16 | const rclient = require('@overleaf/redis-wrapper').createClient( 17 | Settings.redis.history 18 | ) 19 | const Keys = Settings.redis.history.key_schema 20 | const logger = require('logger-sharelatex') 21 | 22 | module.exports = HistoryRedisManager = { 23 | recordDocHasHistoryOps(project_id, doc_id, ops, callback) { 24 | if (ops == null) { 25 | ops = [] 26 | } 27 | if (callback == null) { 28 | callback = function (error) {} 29 | } 30 | if (ops.length === 0) { 31 | return callback(new Error('cannot push no ops')) // This should never be called with no ops, but protect against a redis error if we sent an empty array to rpush 32 | } 33 | logger.log({ project_id, doc_id }, 'marking doc in project for history ops') 34 | return rclient.sadd( 35 | Keys.docsWithHistoryOps({ project_id }), 36 | doc_id, 37 | function (error) { 38 | if (error != null) { 39 | return callback(error) 40 | } 41 | return callback() 42 | } 43 | ) 44 | }, 45 | } 46 | -------------------------------------------------------------------------------- /app/js/LockManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-return-assign, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | let LockManager 15 | const metrics = require('./Metrics') 16 | const Settings = require('@overleaf/settings') 17 | const redis = require('@overleaf/redis-wrapper') 18 | const rclient = redis.createClient(Settings.redis.lock) 19 | const keys = Settings.redis.lock.key_schema 20 | const logger = require('logger-sharelatex') 21 | const os = require('os') 22 | const crypto = require('crypto') 23 | 24 | const Profiler = require('./Profiler') 25 | 26 | const HOST = os.hostname() 27 | const PID = process.pid 28 | const RND = crypto.randomBytes(4).toString('hex') 29 | let COUNT = 0 30 | 31 | const MAX_REDIS_REQUEST_LENGTH = 5000 // 5 seconds 32 | 33 | module.exports = LockManager = { 34 | LOCK_TEST_INTERVAL: 50, // 50ms between each test of the lock 35 | MAX_TEST_INTERVAL: 1000, // back off to 1s between each test of the lock 36 | MAX_LOCK_WAIT_TIME: 10000, // 10s maximum time to spend trying to get the lock 37 | LOCK_TTL: 30, // seconds. Time until lock auto expires in redis. 38 | 39 | // Use a signed lock value as described in 40 | // http://redis.io/topics/distlock#correct-implementation-with-a-single-instance 41 | // to prevent accidental unlocking by multiple processes 42 | randomLock() { 43 | const time = Date.now() 44 | return `locked:host=${HOST}:pid=${PID}:random=${RND}:time=${time}:count=${COUNT++}` 45 | }, 46 | 47 | unlockScript: 48 | 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end', 49 | 50 | tryLock(doc_id, callback) { 51 | if (callback == null) { 52 | callback = function (err, isFree) {} 53 | } 54 | const lockValue = LockManager.randomLock() 55 | const key = keys.blockingKey({ doc_id }) 56 | const profile = new Profiler('tryLock', { doc_id, key, lockValue }) 57 | return rclient.set( 58 | key, 59 | lockValue, 60 | 'EX', 61 | this.LOCK_TTL, 62 | 'NX', 63 | function (err, gotLock) { 64 | if (err != null) { 65 | return callback(err) 66 | } 67 | if (gotLock === 'OK') { 68 | metrics.inc('doc-not-blocking') 69 | const timeTaken = profile.log('got lock').end() 70 | if (timeTaken > MAX_REDIS_REQUEST_LENGTH) { 71 | // took too long, so try to free the lock 72 | return LockManager.releaseLock( 73 | doc_id, 74 | lockValue, 75 | function (err, result) { 76 | if (err != null) { 77 | return callback(err) 78 | } // error freeing lock 79 | return callback(null, false) 80 | } 81 | ) // tell caller they didn't get the lock 82 | } else { 83 | return callback(null, true, lockValue) 84 | } 85 | } else { 86 | metrics.inc('doc-blocking') 87 | profile.log('doc is locked').end() 88 | return callback(null, false) 89 | } 90 | } 91 | ) 92 | }, 93 | 94 | getLock(doc_id, callback) { 95 | let attempt 96 | if (callback == null) { 97 | callback = function (error, lockValue) {} 98 | } 99 | const startTime = Date.now() 100 | let testInterval = LockManager.LOCK_TEST_INTERVAL 101 | const profile = new Profiler('getLock', { doc_id }) 102 | return (attempt = function () { 103 | if (Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME) { 104 | const e = new Error('Timeout') 105 | e.doc_id = doc_id 106 | profile.log('timeout').end() 107 | return callback(e) 108 | } 109 | 110 | return LockManager.tryLock(doc_id, function (error, gotLock, lockValue) { 111 | if (error != null) { 112 | return callback(error) 113 | } 114 | profile.log('tryLock') 115 | if (gotLock) { 116 | profile.end() 117 | return callback(null, lockValue) 118 | } else { 119 | setTimeout(attempt, testInterval) 120 | // back off when the lock is taken to avoid overloading 121 | return (testInterval = Math.min( 122 | testInterval * 2, 123 | LockManager.MAX_TEST_INTERVAL 124 | )) 125 | } 126 | }) 127 | })() 128 | }, 129 | 130 | checkLock(doc_id, callback) { 131 | if (callback == null) { 132 | callback = function (err, isFree) {} 133 | } 134 | const key = keys.blockingKey({ doc_id }) 135 | return rclient.exists(key, function (err, exists) { 136 | if (err != null) { 137 | return callback(err) 138 | } 139 | exists = parseInt(exists) 140 | if (exists === 1) { 141 | metrics.inc('doc-blocking') 142 | return callback(null, false) 143 | } else { 144 | metrics.inc('doc-not-blocking') 145 | return callback(null, true) 146 | } 147 | }) 148 | }, 149 | 150 | releaseLock(doc_id, lockValue, callback) { 151 | const key = keys.blockingKey({ doc_id }) 152 | const profile = new Profiler('releaseLock', { doc_id, key, lockValue }) 153 | return rclient.eval( 154 | LockManager.unlockScript, 155 | 1, 156 | key, 157 | lockValue, 158 | function (err, result) { 159 | if (err != null) { 160 | return callback(err) 161 | } else if (result != null && result !== 1) { 162 | // successful unlock should release exactly one key 163 | profile.log('unlockScript:expired-lock').end() 164 | logger.error( 165 | { doc_id, key, lockValue, redis_err: err, redis_result: result }, 166 | 'unlocking error' 167 | ) 168 | metrics.inc('unlock-error') 169 | return callback(new Error('tried to release timed out lock')) 170 | } else { 171 | profile.log('unlockScript:ok').end() 172 | return callback(null, result) 173 | } 174 | } 175 | ) 176 | }, 177 | } 178 | -------------------------------------------------------------------------------- /app/js/LoggerSerializers.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-return-assign, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS102: Remove unnecessary code created because of implicit returns 9 | * DS103: Rewrite code to no longer use __guard__ 10 | * DS207: Consider shorter variations of null checks 11 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 12 | */ 13 | const _ = require('lodash') 14 | 15 | const showLength = function (thing) { 16 | if (thing != null ? thing.length : undefined) { 17 | return thing.length 18 | } else { 19 | return thing 20 | } 21 | } 22 | 23 | const showUpdateLength = function (update) { 24 | if ((update != null ? update.op : undefined) instanceof Array) { 25 | const copy = _.cloneDeep(update) 26 | copy.op.forEach(function (element, index) { 27 | if ( 28 | __guard__(element != null ? element.i : undefined, x => x.length) != 29 | null 30 | ) { 31 | copy.op[index].i = element.i.length 32 | } 33 | if ( 34 | __guard__(element != null ? element.d : undefined, x1 => x1.length) != 35 | null 36 | ) { 37 | copy.op[index].d = element.d.length 38 | } 39 | if ( 40 | __guard__(element != null ? element.c : undefined, x2 => x2.length) != 41 | null 42 | ) { 43 | return (copy.op[index].c = element.c.length) 44 | } 45 | }) 46 | return copy 47 | } else { 48 | return update 49 | } 50 | } 51 | 52 | module.exports = { 53 | // replace long values with their length 54 | lines: showLength, 55 | oldLines: showLength, 56 | newLines: showLength, 57 | docLines: showLength, 58 | newDocLines: showLength, 59 | ranges: showLength, 60 | update: showUpdateLength, 61 | } 62 | 63 | function __guard__(value, transform) { 64 | return typeof value !== 'undefined' && value !== null 65 | ? transform(value) 66 | : undefined 67 | } 68 | -------------------------------------------------------------------------------- /app/js/Metrics.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | module.exports = require('@overleaf/metrics') 4 | -------------------------------------------------------------------------------- /app/js/Profiler.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-unused-vars, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS206: Consider reworking classes to avoid initClass 9 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 10 | */ 11 | let Profiler 12 | const Settings = require('@overleaf/settings') 13 | const logger = require('logger-sharelatex') 14 | 15 | const deltaMs = function (ta, tb) { 16 | const nanoSeconds = (ta[0] - tb[0]) * 1e9 + (ta[1] - tb[1]) 17 | const milliSeconds = Math.floor(nanoSeconds * 1e-6) 18 | return milliSeconds 19 | } 20 | 21 | module.exports = Profiler = (function () { 22 | Profiler = class Profiler { 23 | static initClass() { 24 | this.prototype.LOG_CUTOFF_TIME = 1000 25 | } 26 | 27 | constructor(name, args) { 28 | this.name = name 29 | this.args = args 30 | this.t0 = this.t = process.hrtime() 31 | this.start = new Date() 32 | this.updateTimes = [] 33 | } 34 | 35 | log(label) { 36 | const t1 = process.hrtime() 37 | const dtMilliSec = deltaMs(t1, this.t) 38 | this.t = t1 39 | this.updateTimes.push([label, dtMilliSec]) // timings in ms 40 | return this // make it chainable 41 | } 42 | 43 | end(message) { 44 | const totalTime = deltaMs(this.t, this.t0) 45 | if (totalTime > this.LOG_CUTOFF_TIME) { 46 | // log anything greater than cutoff 47 | const args = {} 48 | for (const k in this.args) { 49 | const v = this.args[k] 50 | args[k] = v 51 | } 52 | args.updateTimes = this.updateTimes 53 | args.start = this.start 54 | args.end = new Date() 55 | logger.log(args, this.name) 56 | } 57 | return totalTime 58 | } 59 | } 60 | Profiler.initClass() 61 | return Profiler 62 | })() 63 | -------------------------------------------------------------------------------- /app/js/ProjectFlusher.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS205: Consider reworking code to avoid use of IIFEs 12 | * DS207: Consider shorter variations of null checks 13 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 14 | */ 15 | const request = require('request') 16 | const Settings = require('@overleaf/settings') 17 | const RedisManager = require('./RedisManager') 18 | const { rclient } = RedisManager 19 | const docUpdaterKeys = Settings.redis.documentupdater.key_schema 20 | const async = require('async') 21 | const ProjectManager = require('./ProjectManager') 22 | const _ = require('lodash') 23 | const logger = require('logger-sharelatex') 24 | 25 | var ProjectFlusher = { 26 | // iterate over keys asynchronously using redis scan (non-blocking) 27 | // handle all the cluster nodes or single redis server 28 | _getKeys(pattern, limit, callback) { 29 | const nodes = (typeof rclient.nodes === 'function' 30 | ? rclient.nodes('master') 31 | : undefined) || [rclient] 32 | const doKeyLookupForNode = (node, cb) => 33 | ProjectFlusher._getKeysFromNode(node, pattern, limit, cb) 34 | return async.concatSeries(nodes, doKeyLookupForNode, callback) 35 | }, 36 | 37 | _getKeysFromNode(node, pattern, limit, callback) { 38 | if (limit == null) { 39 | limit = 1000 40 | } 41 | let cursor = 0 // redis iterator 42 | const keySet = {} // use hash to avoid duplicate results 43 | const batchSize = limit != null ? Math.min(limit, 1000) : 1000 44 | // scan over all keys looking for pattern 45 | var doIteration = ( 46 | cb // avoid hitting redis too hard 47 | ) => 48 | node.scan( 49 | cursor, 50 | 'MATCH', 51 | pattern, 52 | 'COUNT', 53 | batchSize, 54 | function (error, reply) { 55 | let keys 56 | if (error != null) { 57 | return callback(error) 58 | } 59 | ;[cursor, keys] = Array.from(reply) 60 | for (const key of Array.from(keys)) { 61 | keySet[key] = true 62 | } 63 | keys = Object.keys(keySet) 64 | const noResults = cursor === '0' // redis returns string results not numeric 65 | const limitReached = limit != null && keys.length >= limit 66 | if (noResults || limitReached) { 67 | return callback(null, keys) 68 | } else { 69 | return setTimeout(doIteration, 10) 70 | } 71 | } 72 | ) 73 | return doIteration() 74 | }, 75 | 76 | // extract ids from keys like DocsWithHistoryOps:57fd0b1f53a8396d22b2c24b 77 | // or docsInProject:{57fd0b1f53a8396d22b2c24b} (for redis cluster) 78 | _extractIds(keyList) { 79 | const ids = (() => { 80 | const result = [] 81 | for (const key of Array.from(keyList)) { 82 | const m = key.match(/:\{?([0-9a-f]{24})\}?/) // extract object id 83 | result.push(m[1]) 84 | } 85 | return result 86 | })() 87 | return ids 88 | }, 89 | 90 | flushAllProjects(options, callback) { 91 | logger.log({ options }, 'flushing all projects') 92 | return ProjectFlusher._getKeys( 93 | docUpdaterKeys.docsInProject({ project_id: '*' }), 94 | options.limit, 95 | function (error, project_keys) { 96 | if (error != null) { 97 | logger.err({ err: error }, 'error getting keys for flushing') 98 | return callback(error) 99 | } 100 | const project_ids = ProjectFlusher._extractIds(project_keys) 101 | if (options.dryRun) { 102 | return callback(null, project_ids) 103 | } 104 | const jobs = _.map( 105 | project_ids, 106 | project_id => cb => 107 | ProjectManager.flushAndDeleteProjectWithLocks( 108 | project_id, 109 | { background: true }, 110 | cb 111 | ) 112 | ) 113 | return async.parallelLimit( 114 | async.reflectAll(jobs), 115 | options.concurrency, 116 | function (error, results) { 117 | const success = [] 118 | const failure = [] 119 | _.each(results, function (result, i) { 120 | if (result.error != null) { 121 | return failure.push(project_ids[i]) 122 | } else { 123 | return success.push(project_ids[i]) 124 | } 125 | }) 126 | logger.log({ success, failure }, 'finished flushing all projects') 127 | return callback(error, { success, failure }) 128 | } 129 | ) 130 | } 131 | ) 132 | }, 133 | } 134 | 135 | module.exports = ProjectFlusher 136 | -------------------------------------------------------------------------------- /app/js/ProjectHistoryRedisManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS103: Rewrite code to no longer use __guard__ 12 | * DS201: Simplify complex destructure assignments 13 | * DS207: Consider shorter variations of null checks 14 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 15 | */ 16 | let ProjectHistoryRedisManager 17 | const Settings = require('@overleaf/settings') 18 | const projectHistoryKeys = __guard__( 19 | Settings.redis != null ? Settings.redis.project_history : undefined, 20 | x => x.key_schema 21 | ) 22 | const rclient = require('@overleaf/redis-wrapper').createClient( 23 | Settings.redis.project_history 24 | ) 25 | const logger = require('logger-sharelatex') 26 | const metrics = require('./Metrics') 27 | 28 | module.exports = ProjectHistoryRedisManager = { 29 | queueOps(project_id, ...rest) { 30 | // Record metric for ops pushed onto queue 31 | const adjustedLength = Math.max(rest.length, 1) 32 | const ops = rest.slice(0, adjustedLength - 1) 33 | const val = rest[adjustedLength - 1] 34 | const callback = val != null ? val : function (error, projectUpdateCount) {} 35 | for (const op of Array.from(ops)) { 36 | metrics.summary('redis.projectHistoryOps', op.length, { status: 'push' }) 37 | } 38 | const multi = rclient.multi() 39 | // Push the ops onto the project history queue 40 | multi.rpush( 41 | projectHistoryKeys.projectHistoryOps({ project_id }), 42 | ...Array.from(ops) 43 | ) 44 | // To record the age of the oldest op on the queue set a timestamp if not 45 | // already present (SETNX). 46 | multi.setnx( 47 | projectHistoryKeys.projectHistoryFirstOpTimestamp({ project_id }), 48 | Date.now() 49 | ) 50 | return multi.exec(function (error, result) { 51 | if (error != null) { 52 | return callback(error) 53 | } 54 | // return the number of entries pushed onto the project history queue 55 | return callback(null, result[0]) 56 | }) 57 | }, 58 | 59 | queueRenameEntity( 60 | project_id, 61 | projectHistoryId, 62 | entity_type, 63 | entity_id, 64 | user_id, 65 | projectUpdate, 66 | callback 67 | ) { 68 | projectUpdate = { 69 | pathname: projectUpdate.pathname, 70 | new_pathname: projectUpdate.newPathname, 71 | meta: { 72 | user_id, 73 | ts: new Date(), 74 | }, 75 | version: projectUpdate.version, 76 | projectHistoryId, 77 | } 78 | projectUpdate[entity_type] = entity_id 79 | 80 | logger.log( 81 | { project_id, projectUpdate }, 82 | 'queue rename operation to project-history' 83 | ) 84 | const jsonUpdate = JSON.stringify(projectUpdate) 85 | 86 | return ProjectHistoryRedisManager.queueOps(project_id, jsonUpdate, callback) 87 | }, 88 | 89 | queueAddEntity( 90 | project_id, 91 | projectHistoryId, 92 | entity_type, 93 | entitiy_id, 94 | user_id, 95 | projectUpdate, 96 | callback 97 | ) { 98 | if (callback == null) { 99 | callback = function (error) {} 100 | } 101 | projectUpdate = { 102 | pathname: projectUpdate.pathname, 103 | docLines: projectUpdate.docLines, 104 | url: projectUpdate.url, 105 | meta: { 106 | user_id, 107 | ts: new Date(), 108 | }, 109 | version: projectUpdate.version, 110 | projectHistoryId, 111 | } 112 | projectUpdate[entity_type] = entitiy_id 113 | 114 | logger.log( 115 | { project_id, projectUpdate }, 116 | 'queue add operation to project-history' 117 | ) 118 | const jsonUpdate = JSON.stringify(projectUpdate) 119 | 120 | return ProjectHistoryRedisManager.queueOps(project_id, jsonUpdate, callback) 121 | }, 122 | 123 | queueResyncProjectStructure( 124 | project_id, 125 | projectHistoryId, 126 | docs, 127 | files, 128 | callback 129 | ) { 130 | logger.log({ project_id, docs, files }, 'queue project structure resync') 131 | const projectUpdate = { 132 | resyncProjectStructure: { docs, files }, 133 | projectHistoryId, 134 | meta: { 135 | ts: new Date(), 136 | }, 137 | } 138 | const jsonUpdate = JSON.stringify(projectUpdate) 139 | return ProjectHistoryRedisManager.queueOps(project_id, jsonUpdate, callback) 140 | }, 141 | 142 | queueResyncDocContent( 143 | project_id, 144 | projectHistoryId, 145 | doc_id, 146 | lines, 147 | version, 148 | pathname, 149 | callback 150 | ) { 151 | logger.log( 152 | { project_id, doc_id, lines, version, pathname }, 153 | 'queue doc content resync' 154 | ) 155 | const projectUpdate = { 156 | resyncDocContent: { 157 | content: lines.join('\n'), 158 | version, 159 | }, 160 | projectHistoryId, 161 | path: pathname, 162 | doc: doc_id, 163 | meta: { 164 | ts: new Date(), 165 | }, 166 | } 167 | const jsonUpdate = JSON.stringify(projectUpdate) 168 | return ProjectHistoryRedisManager.queueOps(project_id, jsonUpdate, callback) 169 | }, 170 | } 171 | 172 | function __guard__(value, transform) { 173 | return typeof value !== 'undefined' && value !== null 174 | ? transform(value) 175 | : undefined 176 | } 177 | -------------------------------------------------------------------------------- /app/js/RangesManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | let RangesManager 15 | const RangesTracker = require('./RangesTracker') 16 | const logger = require('logger-sharelatex') 17 | const _ = require('lodash') 18 | 19 | module.exports = RangesManager = { 20 | MAX_COMMENTS: 500, 21 | MAX_CHANGES: 2000, 22 | 23 | applyUpdate(project_id, doc_id, entries, updates, newDocLines, callback) { 24 | let error 25 | if (entries == null) { 26 | entries = {} 27 | } 28 | if (updates == null) { 29 | updates = [] 30 | } 31 | if (callback == null) { 32 | callback = function (error, new_entries, ranges_were_collapsed) {} 33 | } 34 | const { changes, comments } = _.cloneDeep(entries) 35 | const rangesTracker = new RangesTracker(changes, comments) 36 | const emptyRangeCountBefore = RangesManager._emptyRangesCount(rangesTracker) 37 | for (const update of Array.from(updates)) { 38 | rangesTracker.track_changes = !!update.meta.tc 39 | if (update.meta.tc) { 40 | rangesTracker.setIdSeed(update.meta.tc) 41 | } 42 | for (const op of Array.from(update.op)) { 43 | try { 44 | rangesTracker.applyOp(op, { 45 | user_id: update.meta != null ? update.meta.user_id : undefined, 46 | }) 47 | } catch (error1) { 48 | error = error1 49 | return callback(error) 50 | } 51 | } 52 | } 53 | 54 | if ( 55 | (rangesTracker.changes != null 56 | ? rangesTracker.changes.length 57 | : undefined) > RangesManager.MAX_CHANGES || 58 | (rangesTracker.comments != null 59 | ? rangesTracker.comments.length 60 | : undefined) > RangesManager.MAX_COMMENTS 61 | ) { 62 | return callback(new Error('too many comments or tracked changes')) 63 | } 64 | 65 | try { 66 | // This is a consistency check that all of our ranges and 67 | // comments still match the corresponding text 68 | rangesTracker.validate(newDocLines.join('\n')) 69 | } catch (error2) { 70 | error = error2 71 | logger.error( 72 | { err: error, project_id, doc_id, newDocLines, updates }, 73 | 'error validating ranges' 74 | ) 75 | return callback(error) 76 | } 77 | 78 | const emptyRangeCountAfter = RangesManager._emptyRangesCount(rangesTracker) 79 | const rangesWereCollapsed = emptyRangeCountAfter > emptyRangeCountBefore 80 | const response = RangesManager._getRanges(rangesTracker) 81 | logger.log( 82 | { 83 | project_id, 84 | doc_id, 85 | changesCount: 86 | response.changes != null ? response.changes.length : undefined, 87 | commentsCount: 88 | response.comments != null ? response.comments.length : undefined, 89 | rangesWereCollapsed, 90 | }, 91 | 'applied updates to ranges' 92 | ) 93 | return callback(null, response, rangesWereCollapsed) 94 | }, 95 | 96 | acceptChanges(change_ids, ranges, callback) { 97 | if (callback == null) { 98 | callback = function (error, ranges) {} 99 | } 100 | const { changes, comments } = ranges 101 | logger.log(`accepting ${change_ids.length} changes in ranges`) 102 | const rangesTracker = new RangesTracker(changes, comments) 103 | rangesTracker.removeChangeIds(change_ids) 104 | const response = RangesManager._getRanges(rangesTracker) 105 | return callback(null, response) 106 | }, 107 | 108 | deleteComment(comment_id, ranges, callback) { 109 | if (callback == null) { 110 | callback = function (error, ranges) {} 111 | } 112 | const { changes, comments } = ranges 113 | logger.log({ comment_id }, 'deleting comment in ranges') 114 | const rangesTracker = new RangesTracker(changes, comments) 115 | rangesTracker.removeCommentId(comment_id) 116 | const response = RangesManager._getRanges(rangesTracker) 117 | return callback(null, response) 118 | }, 119 | 120 | _getRanges(rangesTracker) { 121 | // Return the minimal data structure needed, since most documents won't have any 122 | // changes or comments 123 | let response = {} 124 | if ( 125 | (rangesTracker.changes != null 126 | ? rangesTracker.changes.length 127 | : undefined) > 0 128 | ) { 129 | if (response == null) { 130 | response = {} 131 | } 132 | response.changes = rangesTracker.changes 133 | } 134 | if ( 135 | (rangesTracker.comments != null 136 | ? rangesTracker.comments.length 137 | : undefined) > 0 138 | ) { 139 | if (response == null) { 140 | response = {} 141 | } 142 | response.comments = rangesTracker.comments 143 | } 144 | return response 145 | }, 146 | 147 | _emptyRangesCount(ranges) { 148 | let count = 0 149 | for (const comment of Array.from(ranges.comments || [])) { 150 | if (comment.op.c === '') { 151 | count++ 152 | } 153 | } 154 | for (const change of Array.from(ranges.changes || [])) { 155 | if (change.op.i != null) { 156 | if (change.op.i === '') { 157 | count++ 158 | } 159 | } 160 | } 161 | return count 162 | }, 163 | } 164 | -------------------------------------------------------------------------------- /app/js/RateLimitManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-unused-vars, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS102: Remove unnecessary code created because of implicit returns 9 | * DS207: Consider shorter variations of null checks 10 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 11 | */ 12 | let RateLimiter 13 | const Settings = require('@overleaf/settings') 14 | const logger = require('logger-sharelatex') 15 | const Metrics = require('./Metrics') 16 | 17 | module.exports = RateLimiter = class RateLimiter { 18 | constructor(number) { 19 | if (number == null) { 20 | number = 10 21 | } 22 | this.ActiveWorkerCount = 0 23 | this.CurrentWorkerLimit = number 24 | this.BaseWorkerCount = number 25 | } 26 | 27 | _adjustLimitUp() { 28 | this.CurrentWorkerLimit += 0.1 // allow target worker limit to increase gradually 29 | return Metrics.gauge('currentLimit', Math.ceil(this.CurrentWorkerLimit)) 30 | } 31 | 32 | _adjustLimitDown() { 33 | this.CurrentWorkerLimit = Math.max( 34 | this.BaseWorkerCount, 35 | this.CurrentWorkerLimit * 0.9 36 | ) 37 | logger.log( 38 | { currentLimit: Math.ceil(this.CurrentWorkerLimit) }, 39 | 'reducing rate limit' 40 | ) 41 | return Metrics.gauge('currentLimit', Math.ceil(this.CurrentWorkerLimit)) 42 | } 43 | 44 | _trackAndRun(task, callback) { 45 | if (callback == null) { 46 | callback = function () {} 47 | } 48 | this.ActiveWorkerCount++ 49 | Metrics.gauge('processingUpdates', this.ActiveWorkerCount) 50 | return task(err => { 51 | this.ActiveWorkerCount-- 52 | Metrics.gauge('processingUpdates', this.ActiveWorkerCount) 53 | return callback(err) 54 | }) 55 | } 56 | 57 | run(task, callback) { 58 | if (this.ActiveWorkerCount < this.CurrentWorkerLimit) { 59 | this._trackAndRun(task) // below the limit, just put the task in the background 60 | callback() // return immediately 61 | if (this.CurrentWorkerLimit > this.BaseWorkerCount) { 62 | return this._adjustLimitDown() 63 | } 64 | } else { 65 | logger.log( 66 | { 67 | active: this.ActiveWorkerCount, 68 | currentLimit: Math.ceil(this.CurrentWorkerLimit), 69 | }, 70 | 'hit rate limit' 71 | ) 72 | return this._trackAndRun(task, err => { 73 | if (err == null) { 74 | this._adjustLimitUp() 75 | } // don't increment rate limit if there was an error 76 | return callback(err) 77 | }) // only return after task completes 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /app/js/RealTimeRedisManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | let RealTimeRedisManager 15 | const Settings = require('@overleaf/settings') 16 | const rclient = require('@overleaf/redis-wrapper').createClient( 17 | Settings.redis.documentupdater 18 | ) 19 | const pubsubClient = require('@overleaf/redis-wrapper').createClient( 20 | Settings.redis.pubsub 21 | ) 22 | const Keys = Settings.redis.documentupdater.key_schema 23 | const logger = require('logger-sharelatex') 24 | const os = require('os') 25 | const crypto = require('crypto') 26 | const metrics = require('./Metrics') 27 | 28 | const HOST = os.hostname() 29 | const RND = crypto.randomBytes(4).toString('hex') // generate a random key for this process 30 | let COUNT = 0 31 | 32 | const MAX_OPS_PER_ITERATION = 8 // process a limited number of ops for safety 33 | 34 | module.exports = RealTimeRedisManager = { 35 | getPendingUpdatesForDoc(doc_id, callback) { 36 | const multi = rclient.multi() 37 | multi.lrange(Keys.pendingUpdates({ doc_id }), 0, MAX_OPS_PER_ITERATION - 1) 38 | multi.ltrim(Keys.pendingUpdates({ doc_id }), MAX_OPS_PER_ITERATION, -1) 39 | return multi.exec(function (error, replys) { 40 | let jsonUpdate 41 | if (error != null) { 42 | return callback(error) 43 | } 44 | const jsonUpdates = replys[0] 45 | for (jsonUpdate of Array.from(jsonUpdates)) { 46 | // record metric for each update removed from queue 47 | metrics.summary('redis.pendingUpdates', jsonUpdate.length, { 48 | status: 'pop', 49 | }) 50 | } 51 | const updates = [] 52 | for (jsonUpdate of Array.from(jsonUpdates)) { 53 | var update 54 | try { 55 | update = JSON.parse(jsonUpdate) 56 | } catch (e) { 57 | return callback(e) 58 | } 59 | updates.push(update) 60 | } 61 | return callback(error, updates) 62 | }) 63 | }, 64 | 65 | getUpdatesLength(doc_id, callback) { 66 | return rclient.llen(Keys.pendingUpdates({ doc_id }), callback) 67 | }, 68 | 69 | sendData(data) { 70 | // create a unique message id using a counter 71 | const message_id = `doc:${HOST}:${RND}-${COUNT++}` 72 | if (data != null) { 73 | data._id = message_id 74 | } 75 | 76 | const blob = JSON.stringify(data) 77 | metrics.summary('redis.publish.applied-ops', blob.length) 78 | 79 | // publish on separate channels for individual projects and docs when 80 | // configured (needs realtime to be configured for this too). 81 | if (Settings.publishOnIndividualChannels) { 82 | return pubsubClient.publish(`applied-ops:${data.doc_id}`, blob) 83 | } else { 84 | return pubsubClient.publish('applied-ops', blob) 85 | } 86 | }, 87 | } 88 | -------------------------------------------------------------------------------- /app/js/ShareJsDB.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | let ShareJsDB 15 | const Keys = require('./UpdateKeys') 16 | const RedisManager = require('./RedisManager') 17 | const Errors = require('./Errors') 18 | 19 | module.exports = ShareJsDB = class ShareJsDB { 20 | constructor(project_id, doc_id, lines, version) { 21 | this.project_id = project_id 22 | this.doc_id = doc_id 23 | this.lines = lines 24 | this.version = version 25 | this.appliedOps = {} 26 | // ShareJS calls this detacted from the instance, so we need 27 | // bind it to keep our context that can access @appliedOps 28 | this.writeOp = this._writeOp.bind(this) 29 | } 30 | 31 | getOps(doc_key, start, end, callback) { 32 | if (start === end) { 33 | return callback(null, []) 34 | } 35 | 36 | // In redis, lrange values are inclusive. 37 | if (end != null) { 38 | end-- 39 | } else { 40 | end = -1 41 | } 42 | 43 | const [project_id, doc_id] = Array.from( 44 | Keys.splitProjectIdAndDocId(doc_key) 45 | ) 46 | return RedisManager.getPreviousDocOps(doc_id, start, end, callback) 47 | } 48 | 49 | _writeOp(doc_key, opData, callback) { 50 | if (this.appliedOps[doc_key] == null) { 51 | this.appliedOps[doc_key] = [] 52 | } 53 | this.appliedOps[doc_key].push(opData) 54 | return callback() 55 | } 56 | 57 | getSnapshot(doc_key, callback) { 58 | if ( 59 | doc_key !== Keys.combineProjectIdAndDocId(this.project_id, this.doc_id) 60 | ) { 61 | return callback( 62 | new Errors.NotFoundError( 63 | `unexpected doc_key ${doc_key}, expected ${Keys.combineProjectIdAndDocId( 64 | this.project_id, 65 | this.doc_id 66 | )}` 67 | ) 68 | ) 69 | } else { 70 | return callback(null, { 71 | snapshot: this.lines.join('\n'), 72 | v: parseInt(this.version, 10), 73 | type: 'text', 74 | }) 75 | } 76 | } 77 | 78 | // To be able to remove a doc from the ShareJS memory 79 | // we need to called Model::delete, which calls this 80 | // method on the database. However, we will handle removing 81 | // it from Redis ourselves 82 | delete(docName, dbMeta, callback) { 83 | return callback() 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /app/js/ShareJsUpdateManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS101: Remove unnecessary use of Array.from 11 | * DS102: Remove unnecessary code created because of implicit returns 12 | * DS207: Consider shorter variations of null checks 13 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 14 | */ 15 | let ShareJsUpdateManager 16 | const ShareJsModel = require('./sharejs/server/model') 17 | const ShareJsDB = require('./ShareJsDB') 18 | const logger = require('logger-sharelatex') 19 | const Settings = require('@overleaf/settings') 20 | const Keys = require('./UpdateKeys') 21 | const { EventEmitter } = require('events') 22 | const util = require('util') 23 | const RealTimeRedisManager = require('./RealTimeRedisManager') 24 | const crypto = require('crypto') 25 | const metrics = require('./Metrics') 26 | const Errors = require('./Errors') 27 | 28 | ShareJsModel.prototype = {} 29 | util.inherits(ShareJsModel, EventEmitter) 30 | 31 | const MAX_AGE_OF_OP = 80 32 | 33 | module.exports = ShareJsUpdateManager = { 34 | getNewShareJsModel(project_id, doc_id, lines, version) { 35 | const db = new ShareJsDB(project_id, doc_id, lines, version) 36 | const model = new ShareJsModel(db, { 37 | maxDocLength: Settings.max_doc_length, 38 | maximumAge: MAX_AGE_OF_OP, 39 | }) 40 | model.db = db 41 | return model 42 | }, 43 | 44 | applyUpdate(project_id, doc_id, update, lines, version, callback) { 45 | if (callback == null) { 46 | callback = function (error, updatedDocLines) {} 47 | } 48 | logger.log({ project_id, doc_id, update }, 'applying sharejs updates') 49 | const jobs = [] 50 | // record the update version before it is modified 51 | const incomingUpdateVersion = update.v 52 | // We could use a global model for all docs, but we're hitting issues with the 53 | // internal state of ShareJS not being accessible for clearing caches, and 54 | // getting stuck due to queued callbacks (line 260 of sharejs/server/model.coffee) 55 | // This adds a small but hopefully acceptable overhead (~12ms per 1000 updates on 56 | // my 2009 MBP). 57 | const model = this.getNewShareJsModel(project_id, doc_id, lines, version) 58 | this._listenForOps(model) 59 | const doc_key = Keys.combineProjectIdAndDocId(project_id, doc_id) 60 | return model.applyOp(doc_key, update, function (error) { 61 | if (error != null) { 62 | if (error === 'Op already submitted') { 63 | metrics.inc('sharejs.already-submitted') 64 | logger.warn( 65 | { project_id, doc_id, update }, 66 | 'op has already been submitted' 67 | ) 68 | update.dup = true 69 | ShareJsUpdateManager._sendOp(project_id, doc_id, update) 70 | } else if (/^Delete component/.test(error)) { 71 | metrics.inc('sharejs.delete-mismatch') 72 | logger.warn( 73 | { project_id, doc_id, update, shareJsErr: error }, 74 | 'sharejs delete does not match' 75 | ) 76 | error = new Errors.DeleteMismatchError( 77 | 'Delete component does not match' 78 | ) 79 | return callback(error) 80 | } else { 81 | metrics.inc('sharejs.other-error') 82 | return callback(error) 83 | } 84 | } 85 | logger.log({ project_id, doc_id, error }, 'applied update') 86 | return model.getSnapshot(doc_key, (error, data) => { 87 | if (error != null) { 88 | return callback(error) 89 | } 90 | const docSizeAfter = data.snapshot.length 91 | if (docSizeAfter > Settings.max_doc_length) { 92 | const docSizeBefore = lines.join('\n').length 93 | const err = new Error( 94 | 'blocking persistence of ShareJs update: doc size exceeds limits' 95 | ) 96 | logger.error( 97 | { project_id, doc_id, err, docSizeBefore, docSizeAfter }, 98 | err.message 99 | ) 100 | metrics.inc('sharejs.other-error') 101 | const publicError = 'Update takes doc over max doc size' 102 | return callback(publicError) 103 | } 104 | // only check hash when present and no other updates have been applied 105 | if (update.hash != null && incomingUpdateVersion === version) { 106 | const ourHash = ShareJsUpdateManager._computeHash(data.snapshot) 107 | if (ourHash !== update.hash) { 108 | metrics.inc('sharejs.hash-fail') 109 | return callback(new Error('Invalid hash')) 110 | } else { 111 | metrics.inc('sharejs.hash-pass', 0.001) 112 | } 113 | } 114 | const docLines = data.snapshot.split(/\r\n|\n|\r/) 115 | return callback( 116 | null, 117 | docLines, 118 | data.v, 119 | model.db.appliedOps[doc_key] || [] 120 | ) 121 | }) 122 | }) 123 | }, 124 | 125 | _listenForOps(model) { 126 | return model.on('applyOp', function (doc_key, opData) { 127 | const [project_id, doc_id] = Array.from( 128 | Keys.splitProjectIdAndDocId(doc_key) 129 | ) 130 | return ShareJsUpdateManager._sendOp(project_id, doc_id, opData) 131 | }) 132 | }, 133 | 134 | _sendOp(project_id, doc_id, op) { 135 | return RealTimeRedisManager.sendData({ project_id, doc_id, op }) 136 | }, 137 | 138 | _computeHash(content) { 139 | return crypto 140 | .createHash('sha1') 141 | .update('blob ' + content.length + '\x00') 142 | .update(content, 'utf8') 143 | .digest('hex') 144 | }, 145 | } 146 | -------------------------------------------------------------------------------- /app/js/SnapshotManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-return-assign, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | let SnapshotManager 15 | const { db, ObjectId } = require('./mongodb') 16 | 17 | module.exports = SnapshotManager = { 18 | recordSnapshot( 19 | project_id, 20 | doc_id, 21 | version, 22 | pathname, 23 | lines, 24 | ranges, 25 | callback 26 | ) { 27 | try { 28 | project_id = ObjectId(project_id) 29 | doc_id = ObjectId(doc_id) 30 | } catch (error) { 31 | return callback(error) 32 | } 33 | db.docSnapshots.insertOne( 34 | { 35 | project_id, 36 | doc_id, 37 | version, 38 | lines, 39 | pathname, 40 | ranges: SnapshotManager.jsonRangesToMongo(ranges), 41 | ts: new Date(), 42 | }, 43 | callback 44 | ) 45 | }, 46 | // Suggested indexes: 47 | // db.docSnapshots.createIndex({project_id:1, doc_id:1}) 48 | // db.docSnapshots.createIndex({ts:1},{expiresAfterSeconds: 30*24*3600)) # expires after 30 days 49 | 50 | jsonRangesToMongo(ranges) { 51 | if (ranges == null) { 52 | return null 53 | } 54 | 55 | const updateMetadata = function (metadata) { 56 | if ((metadata != null ? metadata.ts : undefined) != null) { 57 | metadata.ts = new Date(metadata.ts) 58 | } 59 | if ((metadata != null ? metadata.user_id : undefined) != null) { 60 | return (metadata.user_id = SnapshotManager._safeObjectId( 61 | metadata.user_id 62 | )) 63 | } 64 | } 65 | 66 | for (const change of Array.from(ranges.changes || [])) { 67 | change.id = SnapshotManager._safeObjectId(change.id) 68 | updateMetadata(change.metadata) 69 | } 70 | for (const comment of Array.from(ranges.comments || [])) { 71 | comment.id = SnapshotManager._safeObjectId(comment.id) 72 | if ((comment.op != null ? comment.op.t : undefined) != null) { 73 | comment.op.t = SnapshotManager._safeObjectId(comment.op.t) 74 | } 75 | updateMetadata(comment.metadata) 76 | } 77 | return ranges 78 | }, 79 | 80 | _safeObjectId(data) { 81 | try { 82 | return ObjectId(data) 83 | } catch (error) { 84 | return data 85 | } 86 | }, 87 | } 88 | -------------------------------------------------------------------------------- /app/js/UpdateKeys.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | module.exports = { 7 | combineProjectIdAndDocId(project_id, doc_id) { 8 | return `${project_id}:${doc_id}` 9 | }, 10 | splitProjectIdAndDocId(project_and_doc_id) { 11 | return project_and_doc_id.split(':') 12 | }, 13 | } 14 | -------------------------------------------------------------------------------- /app/js/mongodb.js: -------------------------------------------------------------------------------- 1 | const Settings = require('@overleaf/settings') 2 | const { MongoClient, ObjectId } = require('mongodb') 3 | 4 | const clientPromise = MongoClient.connect( 5 | Settings.mongo.url, 6 | Settings.mongo.options 7 | ) 8 | 9 | async function healthCheck() { 10 | const internalDb = (await clientPromise).db() 11 | const res = await internalDb.command({ ping: 1 }) 12 | if (!res.ok) { 13 | throw new Error('failed mongo ping') 14 | } 15 | } 16 | 17 | let setupDbPromise 18 | async function waitForDb() { 19 | if (!setupDbPromise) { 20 | setupDbPromise = setupDb() 21 | } 22 | await setupDbPromise 23 | } 24 | 25 | const db = {} 26 | async function setupDb() { 27 | const internalDb = (await clientPromise).db() 28 | 29 | db.docSnapshots = internalDb.collection('docSnapshots') 30 | } 31 | 32 | module.exports = { 33 | db, 34 | ObjectId, 35 | healthCheck: require('util').callbackify(healthCheck), 36 | waitForDb, 37 | } 38 | -------------------------------------------------------------------------------- /app/js/sharejs/README.md: -------------------------------------------------------------------------------- 1 | This directory contains all the operational transform code. Each file defines a type. 2 | 3 | Most of the types in here are for testing or demonstration. The only types which are sent to the webclient 4 | are `text` and `json`. 5 | 6 | 7 | # An OT type 8 | 9 | All OT types have the following fields: 10 | 11 | `name`: _(string)_ Name of the type. Should match the filename. 12 | `create() -> snapshot`: Function which creates and returns a new document snapshot 13 | 14 | `apply(snapshot, op) -> snapshot`: A function which creates a new document snapshot with the op applied 15 | `transform(op1, op2, side) -> op1'`: OT transform function. 16 | 17 | Given op1, op2, `apply(s, op2, transform(op1, op2, 'left')) == apply(s, op1, transform(op2, op1, 'right'))`. 18 | 19 | Transform and apply must never modify their arguments. 20 | 21 | 22 | Optional properties: 23 | 24 | `tp2`: _(bool)_ True if the transform function supports TP2. This allows p2p architectures to work. 25 | `compose(op1, op2) -> op`: Create and return a new op which has the same effect as op1 + op2. 26 | `serialize(snapshot) -> JSON object`: Serialize a document to something we can JSON.stringify() 27 | `deserialize(object) -> snapshot`: Deserialize a JSON object into the document's internal snapshot format 28 | `prune(op1', op2, side) -> op1`: Inserse transform function. Only required for TP2 types. 29 | `normalize(op) -> op`: Fix up an op to make it valid. Eg, remove skips of size zero. 30 | `api`: _(object)_ Set of helper methods which will be mixed in to the client document object for manipulating documents. See below. 31 | 32 | 33 | # Examples 34 | 35 | `count` and `simple` are two trivial OT type definitions if you want to take a look. JSON defines 36 | the ot-for-JSON type (see the wiki for documentation) and all the text types define different text 37 | implementations. (I still have no idea which one I like the most, and they're fun to write!) 38 | 39 | 40 | # API 41 | 42 | Types can also define API functions. These methods are mixed into the client's Doc object when a document is created. 43 | You can use them to help construct ops programatically (so users don't need to understand how ops are structured). 44 | 45 | For example, the three text types defined here (text, text-composable and text-tp2) all provide the text API, supplying 46 | `.insert()`, `.del()`, `.getLength` and `.getText` methods. 47 | 48 | See text-api.coffee for an example. 49 | -------------------------------------------------------------------------------- /app/js/sharejs/count.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS101: Remove unnecessary use of Array.from 6 | * DS102: Remove unnecessary code created because of implicit returns 7 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 8 | */ 9 | // This is a simple type used for testing other OT code. Each op is [expectedSnapshot, increment] 10 | 11 | exports.name = 'count' 12 | exports.create = () => 1 13 | 14 | exports.apply = function (snapshot, op) { 15 | const [v, inc] = Array.from(op) 16 | if (snapshot !== v) { 17 | throw new Error(`Op ${v} != snapshot ${snapshot}`) 18 | } 19 | return snapshot + inc 20 | } 21 | 22 | // transform op1 by op2. Return transformed version of op1. 23 | exports.transform = function (op1, op2) { 24 | if (op1[0] !== op2[0]) { 25 | throw new Error(`Op1 ${op1[0]} != op2 ${op2[0]}`) 26 | } 27 | return [op1[0] + op2[1], op1[1]] 28 | } 29 | 30 | exports.compose = function (op1, op2) { 31 | if (op1[0] + op1[1] !== op2[0]) { 32 | throw new Error(`Op1 ${op1} + 1 != op2 ${op2}`) 33 | } 34 | return [op1[0], op1[1] + op2[1]] 35 | } 36 | 37 | exports.generateRandomOp = doc => [[doc, 1], doc + 1] 38 | -------------------------------------------------------------------------------- /app/js/sharejs/helpers.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-return-assign, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | // These methods let you build a transform function from a transformComponent function 15 | // for OT types like text and JSON in which operations are lists of components 16 | // and transforming them requires N^2 work. 17 | 18 | // Add transform and transformX functions for an OT type which has transformComponent defined. 19 | // transformComponent(destination array, component, other component, side) 20 | let bootstrapTransform 21 | exports._bt = bootstrapTransform = function ( 22 | type, 23 | transformComponent, 24 | checkValidOp, 25 | append 26 | ) { 27 | let transformX 28 | const transformComponentX = function (left, right, destLeft, destRight) { 29 | transformComponent(destLeft, left, right, 'left') 30 | return transformComponent(destRight, right, left, 'right') 31 | } 32 | 33 | // Transforms rightOp by leftOp. Returns ['rightOp', clientOp'] 34 | type.transformX = 35 | type.transformX = 36 | transformX = 37 | function (leftOp, rightOp) { 38 | checkValidOp(leftOp) 39 | checkValidOp(rightOp) 40 | 41 | const newRightOp = [] 42 | 43 | for (let rightComponent of Array.from(rightOp)) { 44 | // Generate newLeftOp by composing leftOp by rightComponent 45 | const newLeftOp = [] 46 | 47 | let k = 0 48 | while (k < leftOp.length) { 49 | var l 50 | const nextC = [] 51 | transformComponentX(leftOp[k], rightComponent, newLeftOp, nextC) 52 | k++ 53 | 54 | if (nextC.length === 1) { 55 | rightComponent = nextC[0] 56 | } else if (nextC.length === 0) { 57 | for (l of Array.from(leftOp.slice(k))) { 58 | append(newLeftOp, l) 59 | } 60 | rightComponent = null 61 | break 62 | } else { 63 | // Recurse. 64 | const [l_, r_] = Array.from(transformX(leftOp.slice(k), nextC)) 65 | for (l of Array.from(l_)) { 66 | append(newLeftOp, l) 67 | } 68 | for (const r of Array.from(r_)) { 69 | append(newRightOp, r) 70 | } 71 | rightComponent = null 72 | break 73 | } 74 | } 75 | 76 | if (rightComponent != null) { 77 | append(newRightOp, rightComponent) 78 | } 79 | leftOp = newLeftOp 80 | } 81 | 82 | return [leftOp, newRightOp] 83 | } 84 | 85 | // Transforms op with specified type ('left' or 'right') by otherOp. 86 | return (type.transform = type.transform = 87 | function (op, otherOp, type) { 88 | let _ 89 | if (type !== 'left' && type !== 'right') { 90 | throw new Error("type must be 'left' or 'right'") 91 | } 92 | 93 | if (otherOp.length === 0) { 94 | return op 95 | } 96 | 97 | // TODO: Benchmark with and without this line. I _think_ it'll make a big difference...? 98 | if (op.length === 1 && otherOp.length === 1) { 99 | return transformComponent([], op[0], otherOp[0], type) 100 | } 101 | 102 | if (type === 'left') { 103 | let left 104 | ;[left, _] = Array.from(transformX(op, otherOp)) 105 | return left 106 | } else { 107 | let right 108 | ;[_, right] = Array.from(transformX(otherOp, op)) 109 | return right 110 | } 111 | }) 112 | } 113 | 114 | if (typeof WEB === 'undefined') { 115 | exports.bootstrapTransform = bootstrapTransform 116 | } 117 | -------------------------------------------------------------------------------- /app/js/sharejs/index.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS102: Remove unnecessary code created because of implicit returns 6 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 7 | */ 8 | 9 | const register = function (file) { 10 | const type = require(file) 11 | exports[type.name] = type 12 | try { 13 | return require(`${file}-api`) 14 | } catch (error) {} 15 | } 16 | 17 | // Import all the built-in types. 18 | register('./simple') 19 | register('./count') 20 | 21 | register('./text') 22 | register('./text-composable') 23 | register('./text-tp2') 24 | 25 | register('./json') 26 | -------------------------------------------------------------------------------- /app/js/sharejs/server/syncqueue.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS101: Remove unnecessary use of Array.from 6 | * DS102: Remove unnecessary code created because of implicit returns 7 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 8 | */ 9 | // A synchronous processing queue. The queue calls process on the arguments, 10 | // ensuring that process() is only executing once at a time. 11 | // 12 | // process(data, callback) _MUST_ eventually call its callback. 13 | // 14 | // Example: 15 | // 16 | // queue = require 'syncqueue' 17 | // 18 | // fn = queue (data, callback) -> 19 | // asyncthing data, -> 20 | // callback(321) 21 | // 22 | // fn(1) 23 | // fn(2) 24 | // fn(3, (result) -> console.log(result)) 25 | // 26 | // ^--- async thing will only be running once at any time. 27 | 28 | module.exports = function (process) { 29 | if (typeof process !== 'function') { 30 | throw new Error('process is not a function') 31 | } 32 | const queue = [] 33 | 34 | const enqueue = function (data, callback) { 35 | queue.push([data, callback]) 36 | return flush() 37 | } 38 | 39 | enqueue.busy = false 40 | 41 | var flush = function () { 42 | if (enqueue.busy || queue.length === 0) { 43 | return 44 | } 45 | 46 | enqueue.busy = true 47 | const [data, callback] = Array.from(queue.shift()) 48 | return process(data, function (...result) { 49 | // TODO: Make this not use varargs - varargs are really slow. 50 | enqueue.busy = false 51 | // This is called after busy = false so a user can check if enqueue.busy is set in the callback. 52 | if (callback) { 53 | callback.apply(null, result) 54 | } 55 | return flush() 56 | }) 57 | } 58 | 59 | return enqueue 60 | } 61 | -------------------------------------------------------------------------------- /app/js/sharejs/simple.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS102: Remove unnecessary code created because of implicit returns 6 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 7 | */ 8 | // This is a really simple OT type. Its not compiled with the web client, but it could be. 9 | // 10 | // Its mostly included for demonstration purposes and its used in a lot of unit tests. 11 | // 12 | // This defines a really simple text OT type which only allows inserts. (No deletes). 13 | // 14 | // Ops look like: 15 | // {position:#, text:"asdf"} 16 | // 17 | // Document snapshots look like: 18 | // {str:string} 19 | 20 | module.exports = { 21 | // The name of the OT type. The type is stored in types[type.name]. The name can be 22 | // used in place of the actual type in all the API methods. 23 | name: 'simple', 24 | 25 | // Create a new document snapshot 26 | create() { 27 | return { str: '' } 28 | }, 29 | 30 | // Apply the given op to the document snapshot. Returns the new snapshot. 31 | // 32 | // The original snapshot should not be modified. 33 | apply(snapshot, op) { 34 | if (!(op.position >= 0 && op.position <= snapshot.str.length)) { 35 | throw new Error('Invalid position') 36 | } 37 | 38 | let { str } = snapshot 39 | str = str.slice(0, op.position) + op.text + str.slice(op.position) 40 | return { str } 41 | }, 42 | 43 | // transform op1 by op2. Return transformed version of op1. 44 | // sym describes the symmetry of the op. Its 'left' or 'right' depending on whether the 45 | // op being transformed comes from the client or the server. 46 | transform(op1, op2, sym) { 47 | let pos = op1.position 48 | if (op2.position < pos || (op2.position === pos && sym === 'left')) { 49 | pos += op2.text.length 50 | } 51 | 52 | return { position: pos, text: op1.text } 53 | }, 54 | } 55 | -------------------------------------------------------------------------------- /app/js/sharejs/syncqueue.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS101: Remove unnecessary use of Array.from 6 | * DS102: Remove unnecessary code created because of implicit returns 7 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 8 | */ 9 | // A synchronous processing queue. The queue calls process on the arguments, 10 | // ensuring that process() is only executing once at a time. 11 | // 12 | // process(data, callback) _MUST_ eventually call its callback. 13 | // 14 | // Example: 15 | // 16 | // queue = require 'syncqueue' 17 | // 18 | // fn = queue (data, callback) -> 19 | // asyncthing data, -> 20 | // callback(321) 21 | // 22 | // fn(1) 23 | // fn(2) 24 | // fn(3, (result) -> console.log(result)) 25 | // 26 | // ^--- async thing will only be running once at any time. 27 | 28 | module.exports = function (process) { 29 | if (typeof process !== 'function') { 30 | throw new Error('process is not a function') 31 | } 32 | const queue = [] 33 | 34 | const enqueue = function (data, callback) { 35 | queue.push([data, callback]) 36 | return flush() 37 | } 38 | 39 | enqueue.busy = false 40 | 41 | var flush = function () { 42 | if (enqueue.busy || queue.length === 0) { 43 | return 44 | } 45 | 46 | enqueue.busy = true 47 | const [data, callback] = Array.from(queue.shift()) 48 | return process(data, function (...result) { 49 | // TODO: Make this not use varargs - varargs are really slow. 50 | enqueue.busy = false 51 | // This is called after busy = false so a user can check if enqueue.busy is set in the callback. 52 | if (callback) { 53 | callback.apply(null, result) 54 | } 55 | return flush() 56 | }) 57 | } 58 | 59 | return enqueue 60 | } 61 | -------------------------------------------------------------------------------- /app/js/sharejs/text-api.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS101: Remove unnecessary use of Array.from 6 | * DS102: Remove unnecessary code created because of implicit returns 7 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 8 | */ 9 | // Text document API for text 10 | 11 | let text 12 | if (typeof WEB === 'undefined') { 13 | text = require('./text') 14 | } 15 | 16 | text.api = { 17 | provides: { text: true }, 18 | 19 | // The number of characters in the string 20 | getLength() { 21 | return this.snapshot.length 22 | }, 23 | 24 | // Get the text contents of a document 25 | getText() { 26 | return this.snapshot 27 | }, 28 | 29 | insert(pos, text, callback) { 30 | const op = [{ p: pos, i: text }] 31 | 32 | this.submitOp(op, callback) 33 | return op 34 | }, 35 | 36 | del(pos, length, callback) { 37 | const op = [{ p: pos, d: this.snapshot.slice(pos, pos + length) }] 38 | 39 | this.submitOp(op, callback) 40 | return op 41 | }, 42 | 43 | _register() { 44 | return this.on('remoteop', function (op) { 45 | return Array.from(op).map(component => 46 | component.i !== undefined 47 | ? this.emit('insert', component.p, component.i) 48 | : this.emit('delete', component.p, component.d) 49 | ) 50 | }) 51 | }, 52 | } 53 | -------------------------------------------------------------------------------- /app/js/sharejs/text-composable-api.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-undef, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS101: Remove unnecessary use of Array.from 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS205: Consider reworking code to avoid use of IIFEs 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | // Text document API for text 15 | 16 | let type 17 | if (typeof WEB !== 'undefined' && WEB !== null) { 18 | type = exports.types['text-composable'] 19 | } else { 20 | type = require('./text-composable') 21 | } 22 | 23 | type.api = { 24 | provides: { text: true }, 25 | 26 | // The number of characters in the string 27 | getLength() { 28 | return this.snapshot.length 29 | }, 30 | 31 | // Get the text contents of a document 32 | getText() { 33 | return this.snapshot 34 | }, 35 | 36 | insert(pos, text, callback) { 37 | const op = type.normalize([pos, { i: text }, this.snapshot.length - pos]) 38 | 39 | this.submitOp(op, callback) 40 | return op 41 | }, 42 | 43 | del(pos, length, callback) { 44 | const op = type.normalize([ 45 | pos, 46 | { d: this.snapshot.slice(pos, pos + length) }, 47 | this.snapshot.length - pos - length, 48 | ]) 49 | 50 | this.submitOp(op, callback) 51 | return op 52 | }, 53 | 54 | _register() { 55 | return this.on('remoteop', function (op) { 56 | let pos = 0 57 | return (() => { 58 | const result = [] 59 | for (const component of Array.from(op)) { 60 | if (typeof component === 'number') { 61 | result.push((pos += component)) 62 | } else if (component.i !== undefined) { 63 | this.emit('insert', pos, component.i) 64 | result.push((pos += component.i.length)) 65 | } else { 66 | // delete 67 | result.push(this.emit('delete', pos, component.d)) 68 | } 69 | } 70 | return result 71 | })() 72 | }) 73 | }, 74 | } 75 | // We don't increment pos, because the position 76 | // specified is after the delete has happened. 77 | -------------------------------------------------------------------------------- /app/js/sharejs/text-tp2-api.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-undef, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS101: Remove unnecessary use of Array.from 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS205: Consider reworking code to avoid use of IIFEs 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | // Text document API for text-tp2 15 | 16 | let type 17 | if (typeof WEB !== 'undefined' && WEB !== null) { 18 | type = exports.types['text-tp2'] 19 | } else { 20 | type = require('./text-tp2') 21 | } 22 | 23 | const { _takeDoc: takeDoc, _append: append } = type 24 | 25 | const appendSkipChars = (op, doc, pos, maxlength) => 26 | (() => { 27 | const result = [] 28 | while ( 29 | (maxlength === undefined || maxlength > 0) && 30 | pos.index < doc.data.length 31 | ) { 32 | const part = takeDoc(doc, pos, maxlength, true) 33 | if (maxlength !== undefined && typeof part === 'string') { 34 | maxlength -= part.length 35 | } 36 | result.push(append(op, part.length || part)) 37 | } 38 | return result 39 | })() 40 | 41 | type.api = { 42 | provides: { text: true }, 43 | 44 | // The number of characters in the string 45 | getLength() { 46 | return this.snapshot.charLength 47 | }, 48 | 49 | // Flatten a document into a string 50 | getText() { 51 | const strings = Array.from(this.snapshot.data).filter( 52 | elem => typeof elem === 'string' 53 | ) 54 | return strings.join('') 55 | }, 56 | 57 | insert(pos, text, callback) { 58 | if (pos === undefined) { 59 | pos = 0 60 | } 61 | 62 | const op = [] 63 | const docPos = { index: 0, offset: 0 } 64 | 65 | appendSkipChars(op, this.snapshot, docPos, pos) 66 | append(op, { i: text }) 67 | appendSkipChars(op, this.snapshot, docPos) 68 | 69 | this.submitOp(op, callback) 70 | return op 71 | }, 72 | 73 | del(pos, length, callback) { 74 | const op = [] 75 | const docPos = { index: 0, offset: 0 } 76 | 77 | appendSkipChars(op, this.snapshot, docPos, pos) 78 | 79 | while (length > 0) { 80 | const part = takeDoc(this.snapshot, docPos, length, true) 81 | if (typeof part === 'string') { 82 | append(op, { d: part.length }) 83 | length -= part.length 84 | } else { 85 | append(op, part) 86 | } 87 | } 88 | 89 | appendSkipChars(op, this.snapshot, docPos) 90 | 91 | this.submitOp(op, callback) 92 | return op 93 | }, 94 | 95 | _register() { 96 | // Interpret recieved ops + generate more detailed events for them 97 | return this.on('remoteop', function (op, snapshot) { 98 | let textPos = 0 99 | const docPos = { index: 0, offset: 0 } 100 | 101 | for (const component of Array.from(op)) { 102 | var part, remainder 103 | if (typeof component === 'number') { 104 | // Skip 105 | remainder = component 106 | while (remainder > 0) { 107 | part = takeDoc(snapshot, docPos, remainder) 108 | if (typeof part === 'string') { 109 | textPos += part.length 110 | } 111 | remainder -= part.length || part 112 | } 113 | } else if (component.i !== undefined) { 114 | // Insert 115 | if (typeof component.i === 'string') { 116 | this.emit('insert', textPos, component.i) 117 | textPos += component.i.length 118 | } 119 | } else { 120 | // Delete 121 | remainder = component.d 122 | while (remainder > 0) { 123 | part = takeDoc(snapshot, docPos, remainder) 124 | if (typeof part === 'string') { 125 | this.emit('delete', textPos, part) 126 | } 127 | remainder -= part.length || part 128 | } 129 | } 130 | } 131 | }) 132 | }, 133 | } 134 | -------------------------------------------------------------------------------- /app/js/sharejs/types/count.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS101: Remove unnecessary use of Array.from 6 | * DS102: Remove unnecessary code created because of implicit returns 7 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 8 | */ 9 | // This is a simple type used for testing other OT code. Each op is [expectedSnapshot, increment] 10 | 11 | exports.name = 'count' 12 | exports.create = () => 1 13 | 14 | exports.apply = function (snapshot, op) { 15 | const [v, inc] = Array.from(op) 16 | if (snapshot !== v) { 17 | throw new Error(`Op ${v} != snapshot ${snapshot}`) 18 | } 19 | return snapshot + inc 20 | } 21 | 22 | // transform op1 by op2. Return transformed version of op1. 23 | exports.transform = function (op1, op2) { 24 | if (op1[0] !== op2[0]) { 25 | throw new Error(`Op1 ${op1[0]} != op2 ${op2[0]}`) 26 | } 27 | return [op1[0] + op2[1], op1[1]] 28 | } 29 | 30 | exports.compose = function (op1, op2) { 31 | if (op1[0] + op1[1] !== op2[0]) { 32 | throw new Error(`Op1 ${op1} + 1 != op2 ${op2}`) 33 | } 34 | return [op1[0], op1[1] + op2[1]] 35 | } 36 | 37 | exports.generateRandomOp = doc => [[doc, 1], doc + 1] 38 | -------------------------------------------------------------------------------- /app/js/sharejs/types/helpers.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-return-assign, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | // These methods let you build a transform function from a transformComponent function 15 | // for OT types like text and JSON in which operations are lists of components 16 | // and transforming them requires N^2 work. 17 | 18 | // Add transform and transformX functions for an OT type which has transformComponent defined. 19 | // transformComponent(destination array, component, other component, side) 20 | let bootstrapTransform 21 | exports._bt = bootstrapTransform = function ( 22 | type, 23 | transformComponent, 24 | checkValidOp, 25 | append 26 | ) { 27 | let transformX 28 | const transformComponentX = function (left, right, destLeft, destRight) { 29 | transformComponent(destLeft, left, right, 'left') 30 | return transformComponent(destRight, right, left, 'right') 31 | } 32 | 33 | // Transforms rightOp by leftOp. Returns ['rightOp', clientOp'] 34 | type.transformX = 35 | type.transformX = 36 | transformX = 37 | function (leftOp, rightOp) { 38 | checkValidOp(leftOp) 39 | checkValidOp(rightOp) 40 | 41 | const newRightOp = [] 42 | 43 | for (let rightComponent of Array.from(rightOp)) { 44 | // Generate newLeftOp by composing leftOp by rightComponent 45 | const newLeftOp = [] 46 | 47 | let k = 0 48 | while (k < leftOp.length) { 49 | var l 50 | const nextC = [] 51 | transformComponentX(leftOp[k], rightComponent, newLeftOp, nextC) 52 | k++ 53 | 54 | if (nextC.length === 1) { 55 | rightComponent = nextC[0] 56 | } else if (nextC.length === 0) { 57 | for (l of Array.from(leftOp.slice(k))) { 58 | append(newLeftOp, l) 59 | } 60 | rightComponent = null 61 | break 62 | } else { 63 | // Recurse. 64 | const [l_, r_] = Array.from(transformX(leftOp.slice(k), nextC)) 65 | for (l of Array.from(l_)) { 66 | append(newLeftOp, l) 67 | } 68 | for (const r of Array.from(r_)) { 69 | append(newRightOp, r) 70 | } 71 | rightComponent = null 72 | break 73 | } 74 | } 75 | 76 | if (rightComponent != null) { 77 | append(newRightOp, rightComponent) 78 | } 79 | leftOp = newLeftOp 80 | } 81 | 82 | return [leftOp, newRightOp] 83 | } 84 | 85 | // Transforms op with specified type ('left' or 'right') by otherOp. 86 | return (type.transform = type.transform = 87 | function (op, otherOp, type) { 88 | let _ 89 | if (type !== 'left' && type !== 'right') { 90 | throw new Error("type must be 'left' or 'right'") 91 | } 92 | 93 | if (otherOp.length === 0) { 94 | return op 95 | } 96 | 97 | // TODO: Benchmark with and without this line. I _think_ it'll make a big difference...? 98 | if (op.length === 1 && otherOp.length === 1) { 99 | return transformComponent([], op[0], otherOp[0], type) 100 | } 101 | 102 | if (type === 'left') { 103 | let left 104 | ;[left, _] = Array.from(transformX(op, otherOp)) 105 | return left 106 | } else { 107 | let right 108 | ;[_, right] = Array.from(transformX(otherOp, op)) 109 | return right 110 | } 111 | }) 112 | } 113 | 114 | if (typeof WEB === 'undefined') { 115 | exports.bootstrapTransform = bootstrapTransform 116 | } 117 | -------------------------------------------------------------------------------- /app/js/sharejs/types/index.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS102: Remove unnecessary code created because of implicit returns 6 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 7 | */ 8 | 9 | const register = function (file) { 10 | const type = require(file) 11 | exports[type.name] = type 12 | try { 13 | return require(`${file}-api`) 14 | } catch (error) {} 15 | } 16 | 17 | // Import all the built-in types. 18 | register('./simple') 19 | register('./count') 20 | 21 | register('./text') 22 | register('./text-composable') 23 | register('./text-tp2') 24 | 25 | register('./json') 26 | -------------------------------------------------------------------------------- /app/js/sharejs/types/simple.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS102: Remove unnecessary code created because of implicit returns 6 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 7 | */ 8 | // This is a really simple OT type. Its not compiled with the web client, but it could be. 9 | // 10 | // Its mostly included for demonstration purposes and its used in a lot of unit tests. 11 | // 12 | // This defines a really simple text OT type which only allows inserts. (No deletes). 13 | // 14 | // Ops look like: 15 | // {position:#, text:"asdf"} 16 | // 17 | // Document snapshots look like: 18 | // {str:string} 19 | 20 | module.exports = { 21 | // The name of the OT type. The type is stored in types[type.name]. The name can be 22 | // used in place of the actual type in all the API methods. 23 | name: 'simple', 24 | 25 | // Create a new document snapshot 26 | create() { 27 | return { str: '' } 28 | }, 29 | 30 | // Apply the given op to the document snapshot. Returns the new snapshot. 31 | // 32 | // The original snapshot should not be modified. 33 | apply(snapshot, op) { 34 | if (!(op.position >= 0 && op.position <= snapshot.str.length)) { 35 | throw new Error('Invalid position') 36 | } 37 | 38 | let { str } = snapshot 39 | str = str.slice(0, op.position) + op.text + str.slice(op.position) 40 | return { str } 41 | }, 42 | 43 | // transform op1 by op2. Return transformed version of op1. 44 | // sym describes the symmetry of the op. Its 'left' or 'right' depending on whether the 45 | // op being transformed comes from the client or the server. 46 | transform(op1, op2, sym) { 47 | let pos = op1.position 48 | if (op2.position < pos || (op2.position === pos && sym === 'left')) { 49 | pos += op2.text.length 50 | } 51 | 52 | return { position: pos, text: op1.text } 53 | }, 54 | } 55 | -------------------------------------------------------------------------------- /app/js/sharejs/types/syncqueue.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS101: Remove unnecessary use of Array.from 6 | * DS102: Remove unnecessary code created because of implicit returns 7 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 8 | */ 9 | // A synchronous processing queue. The queue calls process on the arguments, 10 | // ensuring that process() is only executing once at a time. 11 | // 12 | // process(data, callback) _MUST_ eventually call its callback. 13 | // 14 | // Example: 15 | // 16 | // queue = require 'syncqueue' 17 | // 18 | // fn = queue (data, callback) -> 19 | // asyncthing data, -> 20 | // callback(321) 21 | // 22 | // fn(1) 23 | // fn(2) 24 | // fn(3, (result) -> console.log(result)) 25 | // 26 | // ^--- async thing will only be running once at any time. 27 | 28 | module.exports = function (process) { 29 | if (typeof process !== 'function') { 30 | throw new Error('process is not a function') 31 | } 32 | const queue = [] 33 | 34 | const enqueue = function (data, callback) { 35 | queue.push([data, callback]) 36 | return flush() 37 | } 38 | 39 | enqueue.busy = false 40 | 41 | var flush = function () { 42 | if (enqueue.busy || queue.length === 0) { 43 | return 44 | } 45 | 46 | enqueue.busy = true 47 | const [data, callback] = Array.from(queue.shift()) 48 | return process(data, function (...result) { 49 | // TODO: Make this not use varargs - varargs are really slow. 50 | enqueue.busy = false 51 | // This is called after busy = false so a user can check if enqueue.busy is set in the callback. 52 | if (callback) { 53 | callback.apply(null, result) 54 | } 55 | return flush() 56 | }) 57 | } 58 | 59 | return enqueue 60 | } 61 | -------------------------------------------------------------------------------- /app/js/sharejs/types/text-api.js: -------------------------------------------------------------------------------- 1 | // TODO: This file was created by bulk-decaffeinate. 2 | // Sanity-check the conversion and remove this comment. 3 | /* 4 | * decaffeinate suggestions: 5 | * DS101: Remove unnecessary use of Array.from 6 | * DS102: Remove unnecessary code created because of implicit returns 7 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 8 | */ 9 | // Text document API for text 10 | 11 | let text 12 | if (typeof WEB === 'undefined') { 13 | text = require('./text') 14 | } 15 | 16 | text.api = { 17 | provides: { text: true }, 18 | 19 | // The number of characters in the string 20 | getLength() { 21 | return this.snapshot.length 22 | }, 23 | 24 | // Get the text contents of a document 25 | getText() { 26 | return this.snapshot 27 | }, 28 | 29 | insert(pos, text, callback) { 30 | const op = [{ p: pos, i: text }] 31 | 32 | this.submitOp(op, callback) 33 | return op 34 | }, 35 | 36 | del(pos, length, callback) { 37 | const op = [{ p: pos, d: this.snapshot.slice(pos, pos + length) }] 38 | 39 | this.submitOp(op, callback) 40 | return op 41 | }, 42 | 43 | _register() { 44 | return this.on('remoteop', function (op) { 45 | return Array.from(op).map(component => 46 | component.i !== undefined 47 | ? this.emit('insert', component.p, component.i) 48 | : this.emit('delete', component.p, component.d) 49 | ) 50 | }) 51 | }, 52 | } 53 | -------------------------------------------------------------------------------- /app/js/sharejs/types/text-composable-api.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-undef, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS101: Remove unnecessary use of Array.from 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS205: Consider reworking code to avoid use of IIFEs 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | // Text document API for text 15 | 16 | let type 17 | if (typeof WEB !== 'undefined' && WEB !== null) { 18 | type = exports.types['text-composable'] 19 | } else { 20 | type = require('./text-composable') 21 | } 22 | 23 | type.api = { 24 | provides: { text: true }, 25 | 26 | // The number of characters in the string 27 | getLength() { 28 | return this.snapshot.length 29 | }, 30 | 31 | // Get the text contents of a document 32 | getText() { 33 | return this.snapshot 34 | }, 35 | 36 | insert(pos, text, callback) { 37 | const op = type.normalize([pos, { i: text }, this.snapshot.length - pos]) 38 | 39 | this.submitOp(op, callback) 40 | return op 41 | }, 42 | 43 | del(pos, length, callback) { 44 | const op = type.normalize([ 45 | pos, 46 | { d: this.snapshot.slice(pos, pos + length) }, 47 | this.snapshot.length - pos - length, 48 | ]) 49 | 50 | this.submitOp(op, callback) 51 | return op 52 | }, 53 | 54 | _register() { 55 | return this.on('remoteop', function (op) { 56 | let pos = 0 57 | return (() => { 58 | const result = [] 59 | for (const component of Array.from(op)) { 60 | if (typeof component === 'number') { 61 | result.push((pos += component)) 62 | } else if (component.i !== undefined) { 63 | this.emit('insert', pos, component.i) 64 | result.push((pos += component.i.length)) 65 | } else { 66 | // delete 67 | result.push(this.emit('delete', pos, component.d)) 68 | } 69 | } 70 | return result 71 | })() 72 | }) 73 | }, 74 | } 75 | // We don't increment pos, because the position 76 | // specified is after the delete has happened. 77 | -------------------------------------------------------------------------------- /app/js/sharejs/types/text-tp2-api.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-undef, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS101: Remove unnecessary use of Array.from 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS205: Consider reworking code to avoid use of IIFEs 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | // Text document API for text-tp2 15 | 16 | let type 17 | if (typeof WEB !== 'undefined' && WEB !== null) { 18 | type = exports.types['text-tp2'] 19 | } else { 20 | type = require('./text-tp2') 21 | } 22 | 23 | const { _takeDoc: takeDoc, _append: append } = type 24 | 25 | const appendSkipChars = (op, doc, pos, maxlength) => 26 | (() => { 27 | const result = [] 28 | while ( 29 | (maxlength === undefined || maxlength > 0) && 30 | pos.index < doc.data.length 31 | ) { 32 | const part = takeDoc(doc, pos, maxlength, true) 33 | if (maxlength !== undefined && typeof part === 'string') { 34 | maxlength -= part.length 35 | } 36 | result.push(append(op, part.length || part)) 37 | } 38 | return result 39 | })() 40 | 41 | type.api = { 42 | provides: { text: true }, 43 | 44 | // The number of characters in the string 45 | getLength() { 46 | return this.snapshot.charLength 47 | }, 48 | 49 | // Flatten a document into a string 50 | getText() { 51 | const strings = Array.from(this.snapshot.data).filter( 52 | elem => typeof elem === 'string' 53 | ) 54 | return strings.join('') 55 | }, 56 | 57 | insert(pos, text, callback) { 58 | if (pos === undefined) { 59 | pos = 0 60 | } 61 | 62 | const op = [] 63 | const docPos = { index: 0, offset: 0 } 64 | 65 | appendSkipChars(op, this.snapshot, docPos, pos) 66 | append(op, { i: text }) 67 | appendSkipChars(op, this.snapshot, docPos) 68 | 69 | this.submitOp(op, callback) 70 | return op 71 | }, 72 | 73 | del(pos, length, callback) { 74 | const op = [] 75 | const docPos = { index: 0, offset: 0 } 76 | 77 | appendSkipChars(op, this.snapshot, docPos, pos) 78 | 79 | while (length > 0) { 80 | const part = takeDoc(this.snapshot, docPos, length, true) 81 | if (typeof part === 'string') { 82 | append(op, { d: part.length }) 83 | length -= part.length 84 | } else { 85 | append(op, part) 86 | } 87 | } 88 | 89 | appendSkipChars(op, this.snapshot, docPos) 90 | 91 | this.submitOp(op, callback) 92 | return op 93 | }, 94 | 95 | _register() { 96 | // Interpret recieved ops + generate more detailed events for them 97 | return this.on('remoteop', function (op, snapshot) { 98 | let textPos = 0 99 | const docPos = { index: 0, offset: 0 } 100 | 101 | for (const component of Array.from(op)) { 102 | var part, remainder 103 | if (typeof component === 'number') { 104 | // Skip 105 | remainder = component 106 | while (remainder > 0) { 107 | part = takeDoc(snapshot, docPos, remainder) 108 | if (typeof part === 'string') { 109 | textPos += part.length 110 | } 111 | remainder -= part.length || part 112 | } 113 | } else if (component.i !== undefined) { 114 | // Insert 115 | if (typeof component.i === 'string') { 116 | this.emit('insert', textPos, component.i) 117 | textPos += component.i.length 118 | } 119 | } else { 120 | // Delete 121 | remainder = component.d 122 | while (remainder > 0) { 123 | part = takeDoc(snapshot, docPos, remainder) 124 | if (typeof part === 'string') { 125 | this.emit('delete', textPos, part) 126 | } 127 | remainder -= part.length || part 128 | } 129 | } 130 | } 131 | }) 132 | }, 133 | } 134 | -------------------------------------------------------------------------------- /app/js/sharejs/types/web-prelude.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-unused-vars, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | // This is included at the top of each compiled type file for the web. 7 | 8 | /** 9 | @const 10 | @type {boolean} 11 | */ 12 | const WEB = true 13 | 14 | const exports = window.sharejs 15 | -------------------------------------------------------------------------------- /app/js/sharejs/web-prelude.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-unused-vars, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | // This is included at the top of each compiled type file for the web. 7 | 8 | /** 9 | @const 10 | @type {boolean} 11 | */ 12 | const WEB = true 13 | 14 | const exports = window.sharejs 15 | -------------------------------------------------------------------------------- /buildscript.txt: -------------------------------------------------------------------------------- 1 | document-updater 2 | --dependencies=mongo,redis 3 | --docker-repos=gcr.io/overleaf-ops 4 | --env-add= 5 | --env-pass-through= 6 | --node-version=12.22.3 7 | --public-repo=True 8 | --script-version=3.11.0 9 | -------------------------------------------------------------------------------- /docker-compose.ci.yml: -------------------------------------------------------------------------------- 1 | # This file was auto-generated, do not edit it directly. 2 | # Instead run bin/update_build_scripts from 3 | # https://github.com/sharelatex/sharelatex-dev-environment 4 | 5 | version: "2.3" 6 | 7 | services: 8 | test_unit: 9 | image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER 10 | user: node 11 | command: npm run test:unit:_run 12 | environment: 13 | NODE_ENV: test 14 | NODE_OPTIONS: "--unhandled-rejections=strict" 15 | 16 | 17 | test_acceptance: 18 | build: . 19 | image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER 20 | environment: 21 | ELASTIC_SEARCH_DSN: es:9200 22 | REDIS_HOST: redis 23 | QUEUES_REDIS_HOST: redis 24 | MONGO_HOST: mongo 25 | POSTGRES_HOST: postgres 26 | MOCHA_GREP: ${MOCHA_GREP} 27 | NODE_ENV: test 28 | NODE_OPTIONS: "--unhandled-rejections=strict" 29 | depends_on: 30 | mongo: 31 | condition: service_healthy 32 | redis: 33 | condition: service_healthy 34 | user: node 35 | command: npm run test:acceptance:_run 36 | 37 | 38 | tar: 39 | build: . 40 | image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER 41 | volumes: 42 | - ./:/tmp/build/ 43 | command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . 44 | user: root 45 | redis: 46 | image: redis 47 | healthcheck: 48 | test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ] 49 | interval: 1s 50 | retries: 20 51 | 52 | mongo: 53 | image: mongo:4.0 54 | healthcheck: 55 | test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" 56 | interval: 1s 57 | retries: 20 58 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # This file was auto-generated, do not edit it directly. 2 | # Instead run bin/update_build_scripts from 3 | # https://github.com/sharelatex/sharelatex-dev-environment 4 | 5 | version: "2.3" 6 | 7 | services: 8 | test_unit: 9 | image: node:12.22.3 10 | volumes: 11 | - .:/app 12 | working_dir: /app 13 | environment: 14 | MOCHA_GREP: ${MOCHA_GREP} 15 | NODE_ENV: test 16 | NODE_OPTIONS: "--unhandled-rejections=strict" 17 | command: npm run --silent test:unit 18 | user: node 19 | 20 | test_acceptance: 21 | image: node:12.22.3 22 | volumes: 23 | - .:/app 24 | working_dir: /app 25 | environment: 26 | ELASTIC_SEARCH_DSN: es:9200 27 | REDIS_HOST: redis 28 | QUEUES_REDIS_HOST: redis 29 | MONGO_HOST: mongo 30 | POSTGRES_HOST: postgres 31 | MOCHA_GREP: ${MOCHA_GREP} 32 | LOG_LEVEL: ERROR 33 | NODE_ENV: test 34 | NODE_OPTIONS: "--unhandled-rejections=strict" 35 | user: node 36 | depends_on: 37 | mongo: 38 | condition: service_healthy 39 | redis: 40 | condition: service_healthy 41 | command: npm run --silent test:acceptance 42 | 43 | redis: 44 | image: redis 45 | healthcheck: 46 | test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ] 47 | interval: 1s 48 | retries: 20 49 | 50 | mongo: 51 | image: mongo:4.0 52 | healthcheck: 53 | test: "mongo --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'" 54 | interval: 1s 55 | retries: 20 56 | 57 | -------------------------------------------------------------------------------- /expire_docops.js: -------------------------------------------------------------------------------- 1 | const Settings = require('@overleaf/settings') 2 | const rclient = require('@overleaf/redis-wrapper').createClient( 3 | Settings.redis.documentupdater 4 | ) 5 | let keys = Settings.redis.documentupdater.key_schema 6 | const async = require('async') 7 | const RedisManager = require('./app/js/RedisManager') 8 | 9 | const getKeysFromNode = function (node, pattern, callback) { 10 | let cursor = 0 // redis iterator 11 | const keySet = {} // use hash to avoid duplicate results 12 | // scan over all keys looking for pattern 13 | const doIteration = () => 14 | node.scan(cursor, 'MATCH', pattern, 'COUNT', 1000, function (error, reply) { 15 | if (error) { 16 | return callback(error) 17 | } 18 | ;[cursor, keys] = reply 19 | console.log('SCAN', keys.length) 20 | for (const key of keys) { 21 | keySet[key] = true 22 | } 23 | if (cursor === '0') { 24 | // note redis returns string result not numeric 25 | return callback(null, Object.keys(keySet)) 26 | } else { 27 | return doIteration() 28 | } 29 | }) 30 | return doIteration() 31 | } 32 | 33 | const getKeys = function (pattern, callback) { 34 | const nodes = (typeof rclient.nodes === 'function' 35 | ? rclient.nodes('master') 36 | : undefined) || [rclient] 37 | console.log('GOT NODES', nodes.length) 38 | const doKeyLookupForNode = (node, cb) => getKeysFromNode(node, pattern, cb) 39 | return async.concatSeries(nodes, doKeyLookupForNode, callback) 40 | } 41 | 42 | const expireDocOps = callback => 43 | // eslint-disable-next-line handle-callback-err 44 | getKeys(keys.docOps({ doc_id: '*' }), (error, keys) => 45 | async.mapSeries( 46 | keys, 47 | function (key, cb) { 48 | console.log(`EXPIRE ${key} ${RedisManager.DOC_OPS_TTL}`) 49 | return rclient.expire(key, RedisManager.DOC_OPS_TTL, cb) 50 | }, 51 | callback 52 | ) 53 | ) 54 | 55 | setTimeout( 56 | () => 57 | // Give redis a chance to connect 58 | expireDocOps(function (error) { 59 | if (error) { 60 | throw error 61 | } 62 | return process.exit() 63 | }), 64 | 1000 65 | ) 66 | -------------------------------------------------------------------------------- /nodemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignore": [ 3 | ".git", 4 | "node_modules/" 5 | ], 6 | "verbose": true, 7 | "legacyWatch": true, 8 | "execMap": { 9 | "js": "npm run start" 10 | }, 11 | "watch": [ 12 | "app/js/", 13 | "app.js", 14 | "config/" 15 | ], 16 | "ext": "js" 17 | } 18 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "document-updater-sharelatex", 3 | "version": "0.1.4", 4 | "description": "An API for applying incoming updates to documents in real-time", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/sharelatex/document-updater-sharelatex.git" 8 | }, 9 | "scripts": { 10 | "start": "node $NODE_APP_OPTIONS app.js", 11 | "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", 12 | "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", 13 | "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js", 14 | "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", 15 | "nodemon": "nodemon --config nodemon.json", 16 | "lint": "eslint --max-warnings 0 --format unix .", 17 | "format": "prettier --list-different $PWD/'**/*.js'", 18 | "format:fix": "prettier --write $PWD/'**/*.js'", 19 | "lint:fix": "eslint --fix ." 20 | }, 21 | "dependencies": { 22 | "@overleaf/metrics": "^3.5.1", 23 | "@overleaf/o-error": "^3.3.1", 24 | "@overleaf/redis-wrapper": "^2.0.1", 25 | "@overleaf/settings": "^2.1.1", 26 | "async": "^2.5.0", 27 | "body-parser": "^1.19.0", 28 | "bunyan": "^1.8.15", 29 | "diff-match-patch": "https://github.com/overleaf/diff-match-patch/archive/89805f9c671a77a263fc53461acd62aa7498f688.tar.gz", 30 | "express": "4.17.1", 31 | "lodash": "^4.17.21", 32 | "logger-sharelatex": "^2.2.0", 33 | "mongodb": "^3.6.6", 34 | "request": "^2.88.2", 35 | "requestretry": "^4.1.2" 36 | }, 37 | "devDependencies": { 38 | "chai": "^4.2.0", 39 | "chai-as-promised": "^7.1.1", 40 | "cluster-key-slot": "^1.0.5", 41 | "eslint": "^7.21.0", 42 | "eslint-config-prettier": "^8.1.0", 43 | "eslint-config-standard": "^16.0.2", 44 | "eslint-plugin-chai-expect": "^2.2.0", 45 | "eslint-plugin-chai-friendly": "^0.6.0", 46 | "eslint-plugin-import": "^2.22.1", 47 | "eslint-plugin-mocha": "^8.0.0", 48 | "eslint-plugin-node": "^11.1.0", 49 | "eslint-plugin-prettier": "^3.1.2", 50 | "eslint-plugin-promise": "^4.2.1", 51 | "mocha": "^8.3.2", 52 | "prettier": "^2.2.1", 53 | "sandboxed-module": "^2.0.4", 54 | "sinon": "^9.0.2", 55 | "timekeeper": "^2.0.0" 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /redis_cluster/7000/redis.conf: -------------------------------------------------------------------------------- 1 | port 7000 2 | cluster-enabled yes 3 | cluster-config-file nodes.conf 4 | cluster-node-timeout 5000 5 | appendonly yes -------------------------------------------------------------------------------- /redis_cluster/7001/redis.conf: -------------------------------------------------------------------------------- 1 | port 7001 2 | cluster-enabled yes 3 | cluster-config-file nodes.conf 4 | cluster-node-timeout 5000 5 | appendonly yes -------------------------------------------------------------------------------- /redis_cluster/7002/redis.conf: -------------------------------------------------------------------------------- 1 | port 7002 2 | cluster-enabled yes 3 | cluster-config-file nodes.conf 4 | cluster-node-timeout 5000 5 | appendonly yes -------------------------------------------------------------------------------- /redis_cluster/7003/redis.conf: -------------------------------------------------------------------------------- 1 | port 7003 2 | cluster-enabled yes 3 | cluster-config-file nodes.conf 4 | cluster-node-timeout 5000 5 | appendonly yes -------------------------------------------------------------------------------- /redis_cluster/7004/redis.conf: -------------------------------------------------------------------------------- 1 | port 7004 2 | cluster-enabled yes 3 | cluster-config-file nodes.conf 4 | cluster-node-timeout 5000 5 | appendonly yes -------------------------------------------------------------------------------- /redis_cluster/7005/redis.conf: -------------------------------------------------------------------------------- 1 | port 7005 2 | cluster-enabled yes 3 | cluster-config-file nodes.conf 4 | cluster-node-timeout 5000 5 | appendonly yes -------------------------------------------------------------------------------- /redis_cluster/redis-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | (cd 7000 && redis-server redis.conf) & 4 | PID1="$!" 5 | 6 | (cd 7001 && redis-server redis.conf) & 7 | PID2="$!" 8 | 9 | (cd 7002 && redis-server redis.conf) & 10 | PID3="$!" 11 | 12 | (cd 7003 && redis-server redis.conf) & 13 | PID4="$!" 14 | 15 | (cd 7004 && redis-server redis.conf) & 16 | PID5="$!" 17 | 18 | (cd 7005 && redis-server redis.conf) & 19 | PID6="$!" 20 | 21 | trap "kill $PID1 $PID2 $PID3 $PID4 $PID5 $PID6" exit INT TERM 22 | 23 | wait -------------------------------------------------------------------------------- /test/acceptance/js/DeletingADocumentTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | handle-callback-err, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS101: Remove unnecessary use of Array.from 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS207: Consider shorter variations of null checks 11 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 12 | */ 13 | const sinon = require('sinon') 14 | const MockTrackChangesApi = require('./helpers/MockTrackChangesApi') 15 | const MockProjectHistoryApi = require('./helpers/MockProjectHistoryApi') 16 | const MockWebApi = require('./helpers/MockWebApi') 17 | const DocUpdaterClient = require('./helpers/DocUpdaterClient') 18 | const DocUpdaterApp = require('./helpers/DocUpdaterApp') 19 | 20 | describe('Deleting a document', function () { 21 | before(function (done) { 22 | this.lines = ['one', 'two', 'three'] 23 | this.version = 42 24 | this.update = { 25 | doc: this.doc_id, 26 | op: [ 27 | { 28 | i: 'one and a half\n', 29 | p: 4, 30 | }, 31 | ], 32 | v: this.version, 33 | } 34 | this.result = ['one', 'one and a half', 'two', 'three'] 35 | 36 | sinon.spy(MockTrackChangesApi, 'flushDoc') 37 | sinon.spy(MockProjectHistoryApi, 'flushProject') 38 | return DocUpdaterApp.ensureRunning(done) 39 | }) 40 | 41 | after(function () { 42 | MockTrackChangesApi.flushDoc.restore() 43 | return MockProjectHistoryApi.flushProject.restore() 44 | }) 45 | 46 | describe('when the updated doc exists in the doc updater', function () { 47 | before(function (done) { 48 | ;[this.project_id, this.doc_id] = Array.from([ 49 | DocUpdaterClient.randomId(), 50 | DocUpdaterClient.randomId(), 51 | ]) 52 | sinon.spy(MockWebApi, 'setDocument') 53 | sinon.spy(MockWebApi, 'getDocument') 54 | 55 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 56 | lines: this.lines, 57 | version: this.version, 58 | }) 59 | return DocUpdaterClient.preloadDoc( 60 | this.project_id, 61 | this.doc_id, 62 | error => { 63 | if (error != null) { 64 | throw error 65 | } 66 | return DocUpdaterClient.sendUpdate( 67 | this.project_id, 68 | this.doc_id, 69 | this.update, 70 | error => { 71 | if (error != null) { 72 | throw error 73 | } 74 | return setTimeout(() => { 75 | return DocUpdaterClient.deleteDoc( 76 | this.project_id, 77 | this.doc_id, 78 | (error, res, body) => { 79 | this.statusCode = res.statusCode 80 | return setTimeout(done, 200) 81 | } 82 | ) 83 | }, 200) 84 | } 85 | ) 86 | } 87 | ) 88 | }) 89 | 90 | after(function () { 91 | MockWebApi.setDocument.restore() 92 | return MockWebApi.getDocument.restore() 93 | }) 94 | 95 | it('should return a 204 status code', function () { 96 | return this.statusCode.should.equal(204) 97 | }) 98 | 99 | it('should send the updated document and version to the web api', function () { 100 | return MockWebApi.setDocument 101 | .calledWith(this.project_id, this.doc_id, this.result, this.version + 1) 102 | .should.equal(true) 103 | }) 104 | 105 | it('should need to reload the doc if read again', function (done) { 106 | MockWebApi.getDocument.resetHistory() 107 | MockWebApi.getDocument.called.should.equals(false) 108 | return DocUpdaterClient.getDoc( 109 | this.project_id, 110 | this.doc_id, 111 | (error, res, doc) => { 112 | MockWebApi.getDocument 113 | .calledWith(this.project_id, this.doc_id) 114 | .should.equal(true) 115 | return done() 116 | } 117 | ) 118 | }) 119 | 120 | it('should flush track changes', function () { 121 | return MockTrackChangesApi.flushDoc 122 | .calledWith(this.doc_id) 123 | .should.equal(true) 124 | }) 125 | 126 | return it('should flush project history', function () { 127 | return MockProjectHistoryApi.flushProject 128 | .calledWith(this.project_id) 129 | .should.equal(true) 130 | }) 131 | }) 132 | 133 | return describe('when the doc is not in the doc updater', function () { 134 | before(function (done) { 135 | ;[this.project_id, this.doc_id] = Array.from([ 136 | DocUpdaterClient.randomId(), 137 | DocUpdaterClient.randomId(), 138 | ]) 139 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 140 | lines: this.lines, 141 | }) 142 | sinon.spy(MockWebApi, 'setDocument') 143 | sinon.spy(MockWebApi, 'getDocument') 144 | return DocUpdaterClient.deleteDoc( 145 | this.project_id, 146 | this.doc_id, 147 | (error, res, body) => { 148 | this.statusCode = res.statusCode 149 | return setTimeout(done, 200) 150 | } 151 | ) 152 | }) 153 | 154 | after(function () { 155 | MockWebApi.setDocument.restore() 156 | return MockWebApi.getDocument.restore() 157 | }) 158 | 159 | it('should return a 204 status code', function () { 160 | return this.statusCode.should.equal(204) 161 | }) 162 | 163 | it('should not need to send the updated document to the web api', function () { 164 | return MockWebApi.setDocument.called.should.equal(false) 165 | }) 166 | 167 | it('should need to reload the doc if read again', function (done) { 168 | MockWebApi.getDocument.called.should.equals(false) 169 | return DocUpdaterClient.getDoc( 170 | this.project_id, 171 | this.doc_id, 172 | (error, res, doc) => { 173 | MockWebApi.getDocument 174 | .calledWith(this.project_id, this.doc_id) 175 | .should.equal(true) 176 | return done() 177 | } 178 | ) 179 | }) 180 | 181 | it('should flush track changes', function () { 182 | return MockTrackChangesApi.flushDoc 183 | .calledWith(this.doc_id) 184 | .should.equal(true) 185 | }) 186 | 187 | return it('should flush project history', function () { 188 | return MockProjectHistoryApi.flushProject 189 | .calledWith(this.project_id) 190 | .should.equal(true) 191 | }) 192 | }) 193 | }) 194 | -------------------------------------------------------------------------------- /test/acceptance/js/FlushingAProjectTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | const sinon = require('sinon') 15 | const async = require('async') 16 | 17 | const MockWebApi = require('./helpers/MockWebApi') 18 | const DocUpdaterClient = require('./helpers/DocUpdaterClient') 19 | const DocUpdaterApp = require('./helpers/DocUpdaterApp') 20 | 21 | describe('Flushing a project', function () { 22 | before(function (done) { 23 | let doc_id0, doc_id1 24 | this.project_id = DocUpdaterClient.randomId() 25 | this.docs = [ 26 | { 27 | id: (doc_id0 = DocUpdaterClient.randomId()), 28 | lines: ['one', 'two', 'three'], 29 | update: { 30 | doc: doc_id0, 31 | op: [ 32 | { 33 | i: 'one and a half\n', 34 | p: 4, 35 | }, 36 | ], 37 | v: 0, 38 | }, 39 | updatedLines: ['one', 'one and a half', 'two', 'three'], 40 | }, 41 | { 42 | id: (doc_id1 = DocUpdaterClient.randomId()), 43 | lines: ['four', 'five', 'six'], 44 | update: { 45 | doc: doc_id1, 46 | op: [ 47 | { 48 | i: 'four and a half\n', 49 | p: 5, 50 | }, 51 | ], 52 | v: 0, 53 | }, 54 | updatedLines: ['four', 'four and a half', 'five', 'six'], 55 | }, 56 | ] 57 | for (const doc of Array.from(this.docs)) { 58 | MockWebApi.insertDoc(this.project_id, doc.id, { 59 | lines: doc.lines, 60 | version: doc.update.v, 61 | }) 62 | } 63 | return DocUpdaterApp.ensureRunning(done) 64 | }) 65 | 66 | return describe('with documents which have been updated', function () { 67 | before(function (done) { 68 | sinon.spy(MockWebApi, 'setDocument') 69 | 70 | return async.series( 71 | this.docs.map(doc => { 72 | return callback => { 73 | return DocUpdaterClient.preloadDoc( 74 | this.project_id, 75 | doc.id, 76 | error => { 77 | if (error != null) { 78 | return callback(error) 79 | } 80 | return DocUpdaterClient.sendUpdate( 81 | this.project_id, 82 | doc.id, 83 | doc.update, 84 | error => { 85 | return callback(error) 86 | } 87 | ) 88 | } 89 | ) 90 | } 91 | }), 92 | error => { 93 | if (error != null) { 94 | throw error 95 | } 96 | return setTimeout(() => { 97 | return DocUpdaterClient.flushProject( 98 | this.project_id, 99 | (error, res, body) => { 100 | this.statusCode = res.statusCode 101 | return done() 102 | } 103 | ) 104 | }, 200) 105 | } 106 | ) 107 | }) 108 | 109 | after(function () { 110 | return MockWebApi.setDocument.restore() 111 | }) 112 | 113 | it('should return a 204 status code', function () { 114 | return this.statusCode.should.equal(204) 115 | }) 116 | 117 | it('should send each document to the web api', function () { 118 | return Array.from(this.docs).map(doc => 119 | MockWebApi.setDocument 120 | .calledWith(this.project_id, doc.id, doc.updatedLines) 121 | .should.equal(true) 122 | ) 123 | }) 124 | 125 | return it('should update the lines in the doc updater', function (done) { 126 | return async.series( 127 | this.docs.map(doc => { 128 | return callback => { 129 | return DocUpdaterClient.getDoc( 130 | this.project_id, 131 | doc.id, 132 | (error, res, returnedDoc) => { 133 | returnedDoc.lines.should.deep.equal(doc.updatedLines) 134 | return callback() 135 | } 136 | ) 137 | } 138 | }), 139 | done 140 | ) 141 | }) 142 | }) 143 | }) 144 | -------------------------------------------------------------------------------- /test/acceptance/js/FlushingDocsTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-return-assign, 5 | no-unused-vars, 6 | */ 7 | // TODO: This file was created by bulk-decaffeinate. 8 | // Fix any style issues and re-enable lint. 9 | /* 10 | * decaffeinate suggestions: 11 | * DS101: Remove unnecessary use of Array.from 12 | * DS102: Remove unnecessary code created because of implicit returns 13 | * DS207: Consider shorter variations of null checks 14 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 15 | */ 16 | const sinon = require('sinon') 17 | const { expect } = require('chai') 18 | const async = require('async') 19 | 20 | const MockWebApi = require('./helpers/MockWebApi') 21 | const DocUpdaterClient = require('./helpers/DocUpdaterClient') 22 | const DocUpdaterApp = require('./helpers/DocUpdaterApp') 23 | 24 | describe('Flushing a doc to Mongo', function () { 25 | before(function (done) { 26 | this.lines = ['one', 'two', 'three'] 27 | this.version = 42 28 | this.update = { 29 | doc: this.doc_id, 30 | meta: { user_id: 'last-author-fake-id' }, 31 | op: [ 32 | { 33 | i: 'one and a half\n', 34 | p: 4, 35 | }, 36 | ], 37 | v: this.version, 38 | } 39 | this.result = ['one', 'one and a half', 'two', 'three'] 40 | return DocUpdaterApp.ensureRunning(done) 41 | }) 42 | 43 | describe('when the updated doc exists in the doc updater', function () { 44 | before(function (done) { 45 | ;[this.project_id, this.doc_id] = Array.from([ 46 | DocUpdaterClient.randomId(), 47 | DocUpdaterClient.randomId(), 48 | ]) 49 | sinon.spy(MockWebApi, 'setDocument') 50 | 51 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 52 | lines: this.lines, 53 | version: this.version, 54 | }) 55 | return DocUpdaterClient.sendUpdates( 56 | this.project_id, 57 | this.doc_id, 58 | [this.update], 59 | error => { 60 | if (error != null) { 61 | throw error 62 | } 63 | return setTimeout(() => { 64 | return DocUpdaterClient.flushDoc(this.project_id, this.doc_id, done) 65 | }, 200) 66 | } 67 | ) 68 | }) 69 | 70 | after(function () { 71 | return MockWebApi.setDocument.restore() 72 | }) 73 | 74 | it('should flush the updated doc lines and version to the web api', function () { 75 | return MockWebApi.setDocument 76 | .calledWith(this.project_id, this.doc_id, this.result, this.version + 1) 77 | .should.equal(true) 78 | }) 79 | 80 | return it('should flush the last update author and time to the web api', function () { 81 | const lastUpdatedAt = MockWebApi.setDocument.lastCall.args[5] 82 | parseInt(lastUpdatedAt).should.be.closeTo(new Date().getTime(), 30000) 83 | 84 | const lastUpdatedBy = MockWebApi.setDocument.lastCall.args[6] 85 | return lastUpdatedBy.should.equal('last-author-fake-id') 86 | }) 87 | }) 88 | 89 | describe('when the doc does not exist in the doc updater', function () { 90 | before(function (done) { 91 | ;[this.project_id, this.doc_id] = Array.from([ 92 | DocUpdaterClient.randomId(), 93 | DocUpdaterClient.randomId(), 94 | ]) 95 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 96 | lines: this.lines, 97 | }) 98 | sinon.spy(MockWebApi, 'setDocument') 99 | return DocUpdaterClient.flushDoc(this.project_id, this.doc_id, done) 100 | }) 101 | 102 | after(function () { 103 | return MockWebApi.setDocument.restore() 104 | }) 105 | 106 | return it('should not flush the doc to the web api', function () { 107 | return MockWebApi.setDocument.called.should.equal(false) 108 | }) 109 | }) 110 | 111 | return describe('when the web api http request takes a long time on first request', function () { 112 | before(function (done) { 113 | ;[this.project_id, this.doc_id] = Array.from([ 114 | DocUpdaterClient.randomId(), 115 | DocUpdaterClient.randomId(), 116 | ]) 117 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 118 | lines: this.lines, 119 | version: this.version, 120 | }) 121 | let t = 30000 122 | sinon 123 | .stub(MockWebApi, 'setDocument') 124 | .callsFake( 125 | ( 126 | project_id, 127 | doc_id, 128 | lines, 129 | version, 130 | ranges, 131 | lastUpdatedAt, 132 | lastUpdatedBy, 133 | callback 134 | ) => { 135 | if (callback == null) { 136 | callback = function (error) {} 137 | } 138 | setTimeout(callback, t) 139 | return (t = 0) 140 | } 141 | ) 142 | return DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, done) 143 | }) 144 | 145 | after(function () { 146 | return MockWebApi.setDocument.restore() 147 | }) 148 | 149 | return it('should still work', function (done) { 150 | const start = Date.now() 151 | return DocUpdaterClient.flushDoc( 152 | this.project_id, 153 | this.doc_id, 154 | (error, res, doc) => { 155 | res.statusCode.should.equal(204) 156 | const delta = Date.now() - start 157 | expect(delta).to.be.below(20000) 158 | return done() 159 | } 160 | ) 161 | }) 162 | }) 163 | }) 164 | -------------------------------------------------------------------------------- /test/acceptance/js/GettingProjectDocsTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | handle-callback-err, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS101: Remove unnecessary use of Array.from 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | const sinon = require('sinon') 15 | const { expect } = require('chai') 16 | 17 | const MockWebApi = require('./helpers/MockWebApi') 18 | const DocUpdaterClient = require('./helpers/DocUpdaterClient') 19 | const DocUpdaterApp = require('./helpers/DocUpdaterApp') 20 | 21 | describe('Getting documents for project', function () { 22 | before(function (done) { 23 | this.lines = ['one', 'two', 'three'] 24 | this.version = 42 25 | return DocUpdaterApp.ensureRunning(done) 26 | }) 27 | 28 | describe('when project state hash does not match', function () { 29 | before(function (done) { 30 | this.projectStateHash = DocUpdaterClient.randomId() 31 | ;[this.project_id, this.doc_id] = Array.from([ 32 | DocUpdaterClient.randomId(), 33 | DocUpdaterClient.randomId(), 34 | ]) 35 | 36 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 37 | lines: this.lines, 38 | version: this.version, 39 | }) 40 | return DocUpdaterClient.preloadDoc( 41 | this.project_id, 42 | this.doc_id, 43 | error => { 44 | if (error != null) { 45 | throw error 46 | } 47 | return DocUpdaterClient.getProjectDocs( 48 | this.project_id, 49 | this.projectStateHash, 50 | (error, res, returnedDocs) => { 51 | this.res = res 52 | this.returnedDocs = returnedDocs 53 | return done() 54 | } 55 | ) 56 | } 57 | ) 58 | }) 59 | 60 | return it('should return a 409 Conflict response', function () { 61 | return this.res.statusCode.should.equal(409) 62 | }) 63 | }) 64 | 65 | describe('when project state hash matches', function () { 66 | before(function (done) { 67 | this.projectStateHash = DocUpdaterClient.randomId() 68 | ;[this.project_id, this.doc_id] = Array.from([ 69 | DocUpdaterClient.randomId(), 70 | DocUpdaterClient.randomId(), 71 | ]) 72 | 73 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 74 | lines: this.lines, 75 | version: this.version, 76 | }) 77 | return DocUpdaterClient.preloadDoc( 78 | this.project_id, 79 | this.doc_id, 80 | error => { 81 | if (error != null) { 82 | throw error 83 | } 84 | return DocUpdaterClient.getProjectDocs( 85 | this.project_id, 86 | this.projectStateHash, 87 | (error, res0, returnedDocs0) => { 88 | // set the hash 89 | this.res0 = res0 90 | this.returnedDocs0 = returnedDocs0 91 | return DocUpdaterClient.getProjectDocs( 92 | this.project_id, 93 | this.projectStateHash, 94 | (error, res, returnedDocs) => { 95 | // the hash should now match 96 | this.res = res 97 | this.returnedDocs = returnedDocs 98 | return done() 99 | } 100 | ) 101 | } 102 | ) 103 | } 104 | ) 105 | }) 106 | 107 | it('should return a 200 response', function () { 108 | return this.res.statusCode.should.equal(200) 109 | }) 110 | 111 | return it('should return the documents', function () { 112 | return this.returnedDocs.should.deep.equal([ 113 | { _id: this.doc_id, lines: this.lines, v: this.version }, 114 | ]) 115 | }) 116 | }) 117 | 118 | return describe('when the doc has been removed', function () { 119 | before(function (done) { 120 | this.projectStateHash = DocUpdaterClient.randomId() 121 | ;[this.project_id, this.doc_id] = Array.from([ 122 | DocUpdaterClient.randomId(), 123 | DocUpdaterClient.randomId(), 124 | ]) 125 | 126 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 127 | lines: this.lines, 128 | version: this.version, 129 | }) 130 | return DocUpdaterClient.preloadDoc( 131 | this.project_id, 132 | this.doc_id, 133 | error => { 134 | if (error != null) { 135 | throw error 136 | } 137 | return DocUpdaterClient.getProjectDocs( 138 | this.project_id, 139 | this.projectStateHash, 140 | (error, res0, returnedDocs0) => { 141 | // set the hash 142 | this.res0 = res0 143 | this.returnedDocs0 = returnedDocs0 144 | return DocUpdaterClient.deleteDoc( 145 | this.project_id, 146 | this.doc_id, 147 | (error, res, body) => { 148 | // delete the doc 149 | return DocUpdaterClient.getProjectDocs( 150 | this.project_id, 151 | this.projectStateHash, 152 | (error, res1, returnedDocs) => { 153 | // the hash would match, but the doc has been deleted 154 | this.res = res1 155 | this.returnedDocs = returnedDocs 156 | return done() 157 | } 158 | ) 159 | } 160 | ) 161 | } 162 | ) 163 | } 164 | ) 165 | }) 166 | 167 | return it('should return a 409 Conflict response', function () { 168 | return this.res.statusCode.should.equal(409) 169 | }) 170 | }) 171 | }) 172 | -------------------------------------------------------------------------------- /test/acceptance/js/PeekingADoc.js: -------------------------------------------------------------------------------- 1 | const sinon = require('sinon') 2 | const MockWebApi = require('./helpers/MockWebApi') 3 | const DocUpdaterClient = require('./helpers/DocUpdaterClient') 4 | const DocUpdaterApp = require('./helpers/DocUpdaterApp') 5 | 6 | describe('Peeking a document', function () { 7 | before(function (done) { 8 | this.lines = ['one', 'two', 'three'] 9 | this.version = 42 10 | return DocUpdaterApp.ensureRunning(done) 11 | }) 12 | 13 | describe('when the document is not loaded', function () { 14 | before(function (done) { 15 | this.project_id = DocUpdaterClient.randomId() 16 | this.doc_id = DocUpdaterClient.randomId() 17 | sinon.spy(MockWebApi, 'getDocument') 18 | 19 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 20 | lines: this.lines, 21 | version: this.version, 22 | }) 23 | 24 | return DocUpdaterClient.peekDoc( 25 | this.project_id, 26 | this.doc_id, 27 | (error, res, returnedDoc) => { 28 | this.error = error 29 | this.res = res 30 | this.returnedDoc = returnedDoc 31 | return done() 32 | } 33 | ) 34 | }) 35 | 36 | after(function () { 37 | return MockWebApi.getDocument.restore() 38 | }) 39 | 40 | it('should return a 404 response', function () { 41 | this.res.statusCode.should.equal(404) 42 | }) 43 | 44 | it('should not load the document from the web API', function () { 45 | return MockWebApi.getDocument.called.should.equal(false) 46 | }) 47 | }) 48 | 49 | describe('when the document is already loaded', function () { 50 | before(function (done) { 51 | this.project_id = DocUpdaterClient.randomId() 52 | this.doc_id = DocUpdaterClient.randomId() 53 | 54 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 55 | lines: this.lines, 56 | version: this.version, 57 | }) 58 | return DocUpdaterClient.preloadDoc( 59 | this.project_id, 60 | this.doc_id, 61 | error => { 62 | if (error != null) { 63 | throw error 64 | } 65 | sinon.spy(MockWebApi, 'getDocument') 66 | return DocUpdaterClient.getDoc( 67 | this.project_id, 68 | this.doc_id, 69 | (error, res, returnedDoc) => { 70 | this.res = res 71 | this.returnedDoc = returnedDoc 72 | return done() 73 | } 74 | ) 75 | } 76 | ) 77 | }) 78 | 79 | after(function () { 80 | return MockWebApi.getDocument.restore() 81 | }) 82 | 83 | it('should return a 200 response', function () { 84 | this.res.statusCode.should.equal(200) 85 | }) 86 | 87 | it('should return the document lines', function () { 88 | return this.returnedDoc.lines.should.deep.equal(this.lines) 89 | }) 90 | 91 | it('should return the document version', function () { 92 | return this.returnedDoc.version.should.equal(this.version) 93 | }) 94 | 95 | it('should not load the document from the web API', function () { 96 | return MockWebApi.getDocument.called.should.equal(false) 97 | }) 98 | }) 99 | }) 100 | -------------------------------------------------------------------------------- /test/acceptance/js/SizeCheckTests.js: -------------------------------------------------------------------------------- 1 | const { expect } = require('chai') 2 | const Settings = require('@overleaf/settings') 3 | 4 | const MockWebApi = require('./helpers/MockWebApi') 5 | const DocUpdaterClient = require('./helpers/DocUpdaterClient') 6 | const DocUpdaterApp = require('./helpers/DocUpdaterApp') 7 | 8 | describe('SizeChecks', function () { 9 | before(function (done) { 10 | DocUpdaterApp.ensureRunning(done) 11 | }) 12 | beforeEach(function () { 13 | this.version = 0 14 | this.update = { 15 | doc: this.doc_id, 16 | op: [ 17 | { 18 | i: 'insert some more lines that will bring it above the limit\n', 19 | p: 42, 20 | }, 21 | ], 22 | v: this.version, 23 | } 24 | this.project_id = DocUpdaterClient.randomId() 25 | this.doc_id = DocUpdaterClient.randomId() 26 | }) 27 | 28 | describe('when a doc is above the doc size limit already', function () { 29 | beforeEach(function () { 30 | this.lines = ['0123456789'.repeat(Settings.max_doc_length / 10 + 1)] 31 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 32 | lines: this.lines, 33 | v: this.version, 34 | }) 35 | }) 36 | 37 | it('should error when fetching the doc', function (done) { 38 | DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res) => { 39 | if (error) return done(error) 40 | expect(res.statusCode).to.equal(500) 41 | done() 42 | }) 43 | }) 44 | 45 | describe('when trying to update', function () { 46 | beforeEach(function (done) { 47 | const update = { 48 | doc: this.doc_id, 49 | op: this.update.op, 50 | v: this.version, 51 | } 52 | DocUpdaterClient.sendUpdate( 53 | this.project_id, 54 | this.doc_id, 55 | update, 56 | error => { 57 | if (error != null) { 58 | throw error 59 | } 60 | setTimeout(done, 200) 61 | } 62 | ) 63 | }) 64 | 65 | it('should still error when fetching the doc', function (done) { 66 | DocUpdaterClient.getDoc(this.project_id, this.doc_id, (error, res) => { 67 | if (error) return done(error) 68 | expect(res.statusCode).to.equal(500) 69 | done() 70 | }) 71 | }) 72 | }) 73 | }) 74 | 75 | describe('when a doc is just below the doc size limit', function () { 76 | beforeEach(function () { 77 | this.lines = ['0123456789'.repeat(Settings.max_doc_length / 10 - 1)] 78 | MockWebApi.insertDoc(this.project_id, this.doc_id, { 79 | lines: this.lines, 80 | v: this.version, 81 | }) 82 | }) 83 | 84 | it('should be able to fetch the doc', function (done) { 85 | DocUpdaterClient.getDoc( 86 | this.project_id, 87 | this.doc_id, 88 | (error, res, doc) => { 89 | if (error) return done(error) 90 | expect(doc.lines).to.deep.equal(this.lines) 91 | done() 92 | } 93 | ) 94 | }) 95 | 96 | describe('when trying to update', function () { 97 | beforeEach(function (done) { 98 | const update = { 99 | doc: this.doc_id, 100 | op: this.update.op, 101 | v: this.version, 102 | } 103 | DocUpdaterClient.sendUpdate( 104 | this.project_id, 105 | this.doc_id, 106 | update, 107 | error => { 108 | if (error != null) { 109 | throw error 110 | } 111 | setTimeout(done, 200) 112 | } 113 | ) 114 | }) 115 | 116 | it('should not update the doc', function (done) { 117 | DocUpdaterClient.getDoc( 118 | this.project_id, 119 | this.doc_id, 120 | (error, res, doc) => { 121 | if (error) return done(error) 122 | expect(doc.lines).to.deep.equal(this.lines) 123 | done() 124 | } 125 | ) 126 | }) 127 | }) 128 | }) 129 | }) 130 | -------------------------------------------------------------------------------- /test/acceptance/js/helpers/DocUpdaterApp.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | handle-callback-err, 3 | */ 4 | // TODO: This file was created by bulk-decaffeinate. 5 | // Fix any style issues and re-enable lint. 6 | /* 7 | * decaffeinate suggestions: 8 | * DS101: Remove unnecessary use of Array.from 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS205: Consider reworking code to avoid use of IIFEs 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | const app = require('../../../../app') 15 | const { waitForDb } = require('../../../../app/js/mongodb') 16 | require('logger-sharelatex').logger.level('fatal') 17 | 18 | module.exports = { 19 | running: false, 20 | initing: false, 21 | callbacks: [], 22 | ensureRunning(callback) { 23 | if (callback == null) { 24 | callback = function (error) {} 25 | } 26 | if (this.running) { 27 | return callback() 28 | } else if (this.initing) { 29 | return this.callbacks.push(callback) 30 | } 31 | this.initing = true 32 | this.callbacks.push(callback) 33 | waitForDb().then(() => { 34 | return app.listen(3003, 'localhost', error => { 35 | if (error != null) { 36 | throw error 37 | } 38 | this.running = true 39 | return (() => { 40 | const result = [] 41 | for (callback of Array.from(this.callbacks)) { 42 | result.push(callback()) 43 | } 44 | return result 45 | })() 46 | }) 47 | }) 48 | }, 49 | } 50 | -------------------------------------------------------------------------------- /test/acceptance/js/helpers/MockProjectHistoryApi.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS207: Consider shorter variations of null checks 11 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 12 | */ 13 | let MockProjectHistoryApi 14 | const express = require('express') 15 | const app = express() 16 | 17 | module.exports = MockProjectHistoryApi = { 18 | flushProject(doc_id, callback) { 19 | if (callback == null) { 20 | callback = function (error) {} 21 | } 22 | return callback() 23 | }, 24 | 25 | run() { 26 | app.post('/project/:project_id/flush', (req, res, next) => { 27 | return this.flushProject(req.params.project_id, error => { 28 | if (error != null) { 29 | return res.sendStatus(500) 30 | } else { 31 | return res.sendStatus(204) 32 | } 33 | }) 34 | }) 35 | 36 | return app.listen(3054, error => { 37 | if (error != null) { 38 | throw error 39 | } 40 | }) 41 | }, 42 | } 43 | 44 | MockProjectHistoryApi.run() 45 | -------------------------------------------------------------------------------- /test/acceptance/js/helpers/MockTrackChangesApi.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS207: Consider shorter variations of null checks 11 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 12 | */ 13 | let MockTrackChangesApi 14 | const express = require('express') 15 | const app = express() 16 | 17 | module.exports = MockTrackChangesApi = { 18 | flushDoc(doc_id, callback) { 19 | if (callback == null) { 20 | callback = function (error) {} 21 | } 22 | return callback() 23 | }, 24 | 25 | run() { 26 | app.post('/project/:project_id/doc/:doc_id/flush', (req, res, next) => { 27 | return this.flushDoc(req.params.doc_id, error => { 28 | if (error != null) { 29 | return res.sendStatus(500) 30 | } else { 31 | return res.sendStatus(204) 32 | } 33 | }) 34 | }) 35 | 36 | return app 37 | .listen(3015, error => { 38 | if (error != null) { 39 | throw error 40 | } 41 | }) 42 | .on('error', error => { 43 | console.error('error starting MockTrackChangesApi:', error.message) 44 | return process.exit(1) 45 | }) 46 | }, 47 | } 48 | 49 | MockTrackChangesApi.run() 50 | -------------------------------------------------------------------------------- /test/acceptance/js/helpers/MockWebApi.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-return-assign, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS207: Consider shorter variations of null checks 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | let MockWebApi 15 | const express = require('express') 16 | const bodyParser = require('body-parser') 17 | const app = express() 18 | const MAX_REQUEST_SIZE = 2 * (2 * 1024 * 1024 + 64 * 1024) 19 | 20 | module.exports = MockWebApi = { 21 | docs: {}, 22 | 23 | clearDocs() { 24 | return (this.docs = {}) 25 | }, 26 | 27 | insertDoc(project_id, doc_id, doc) { 28 | if (doc.version == null) { 29 | doc.version = 0 30 | } 31 | if (doc.lines == null) { 32 | doc.lines = [] 33 | } 34 | doc.pathname = '/a/b/c.tex' 35 | return (this.docs[`${project_id}:${doc_id}`] = doc) 36 | }, 37 | 38 | setDocument( 39 | project_id, 40 | doc_id, 41 | lines, 42 | version, 43 | ranges, 44 | lastUpdatedAt, 45 | lastUpdatedBy, 46 | callback 47 | ) { 48 | if (callback == null) { 49 | callback = function (error) {} 50 | } 51 | const doc = 52 | this.docs[`${project_id}:${doc_id}`] || 53 | (this.docs[`${project_id}:${doc_id}`] = {}) 54 | doc.lines = lines 55 | doc.version = version 56 | doc.ranges = ranges 57 | doc.pathname = '/a/b/c.tex' 58 | doc.lastUpdatedAt = lastUpdatedAt 59 | doc.lastUpdatedBy = lastUpdatedBy 60 | return callback(null) 61 | }, 62 | 63 | getDocument(project_id, doc_id, callback) { 64 | if (callback == null) { 65 | callback = function (error, doc) {} 66 | } 67 | return callback(null, this.docs[`${project_id}:${doc_id}`]) 68 | }, 69 | 70 | run() { 71 | app.get('/project/:project_id/doc/:doc_id', (req, res, next) => { 72 | return this.getDocument( 73 | req.params.project_id, 74 | req.params.doc_id, 75 | (error, doc) => { 76 | if (error != null) { 77 | return res.sendStatus(500) 78 | } else if (doc != null) { 79 | return res.send(JSON.stringify(doc)) 80 | } else { 81 | return res.sendStatus(404) 82 | } 83 | } 84 | ) 85 | }) 86 | 87 | app.post( 88 | '/project/:project_id/doc/:doc_id', 89 | bodyParser.json({ limit: MAX_REQUEST_SIZE }), 90 | (req, res, next) => { 91 | return MockWebApi.setDocument( 92 | req.params.project_id, 93 | req.params.doc_id, 94 | req.body.lines, 95 | req.body.version, 96 | req.body.ranges, 97 | req.body.lastUpdatedAt, 98 | req.body.lastUpdatedBy, 99 | error => { 100 | if (error != null) { 101 | return res.sendStatus(500) 102 | } else { 103 | return res.sendStatus(204) 104 | } 105 | } 106 | ) 107 | } 108 | ) 109 | 110 | return app 111 | .listen(3000, error => { 112 | if (error != null) { 113 | throw error 114 | } 115 | }) 116 | .on('error', error => { 117 | console.error('error starting MockWebApi:', error.message) 118 | return process.exit(1) 119 | }) 120 | }, 121 | } 122 | 123 | MockWebApi.run() 124 | -------------------------------------------------------------------------------- /test/cluster_failover/js/test_blpop_failover.js: -------------------------------------------------------------------------------- 1 | let listenInBackground, sendPings 2 | const redis = require('@overleaf/redis-wrapper') 3 | const rclient1 = redis.createClient({ 4 | cluster: [ 5 | { 6 | port: '7000', 7 | host: 'localhost', 8 | }, 9 | ], 10 | }) 11 | 12 | const rclient2 = redis.createClient({ 13 | cluster: [ 14 | { 15 | port: '7000', 16 | host: 'localhost', 17 | }, 18 | ], 19 | }) 20 | 21 | let counter = 0 22 | const sendPing = function (cb) { 23 | if (cb == null) { 24 | cb = function () {} 25 | } 26 | return rclient1.rpush('test-blpop', counter, error => { 27 | if (error != null) { 28 | console.error('[SENDING ERROR]', error.message) 29 | } 30 | if (error == null) { 31 | counter += 1 32 | } 33 | return cb() 34 | }) 35 | } 36 | 37 | let previous = null 38 | const listenForPing = cb => 39 | rclient2.blpop('test-blpop', 200, (error, result) => { 40 | if (error != null) { 41 | return cb(error) 42 | } 43 | let [, value] = Array.from(result) 44 | value = parseInt(value, 10) 45 | if (value % 10 === 0) { 46 | console.log('.') 47 | } 48 | if (previous != null && value !== previous + 1) { 49 | error = new Error( 50 | `Counter not in order. Got ${value}, expected ${previous + 1}` 51 | ) 52 | } 53 | previous = value 54 | return cb(error, value) 55 | }) 56 | 57 | const PING_DELAY = 100 58 | ;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() 59 | ;(listenInBackground = () => 60 | listenForPing(error => { 61 | if (error) { 62 | console.error('[RECEIVING ERROR]', error.message) 63 | } 64 | return setTimeout(listenInBackground) 65 | }))() 66 | -------------------------------------------------------------------------------- /test/cluster_failover/js/test_pubsub_failover.js: -------------------------------------------------------------------------------- 1 | let sendPings 2 | const redis = require('@overleaf/redis-wrapper') 3 | const rclient1 = redis.createClient({ 4 | cluster: [ 5 | { 6 | port: '7000', 7 | host: 'localhost', 8 | }, 9 | ], 10 | }) 11 | 12 | const rclient2 = redis.createClient({ 13 | cluster: [ 14 | { 15 | port: '7000', 16 | host: 'localhost', 17 | }, 18 | ], 19 | }) 20 | 21 | let counter = 0 22 | const sendPing = function (cb) { 23 | if (cb == null) { 24 | cb = function () {} 25 | } 26 | return rclient1.publish('test-pubsub', counter, error => { 27 | if (error) { 28 | console.error('[SENDING ERROR]', error.message) 29 | } 30 | if (error == null) { 31 | counter += 1 32 | } 33 | return cb() 34 | }) 35 | } 36 | 37 | let previous = null 38 | rclient2.subscribe('test-pubsub') 39 | rclient2.on('message', (channel, value) => { 40 | value = parseInt(value, 10) 41 | if (value % 10 === 0) { 42 | console.log('.') 43 | } 44 | if (previous != null && value !== previous + 1) { 45 | console.error( 46 | '[RECEIVING ERROR]', 47 | `Counter not in order. Got ${value}, expected ${previous + 1}` 48 | ) 49 | } 50 | return (previous = value) 51 | }) 52 | 53 | const PING_DELAY = 100 54 | ;(sendPings = () => sendPing(() => setTimeout(sendPings, PING_DELAY)))() 55 | -------------------------------------------------------------------------------- /test/setup.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai') 2 | const SandboxedModule = require('sandboxed-module') 3 | const sinon = require('sinon') 4 | 5 | // Chai configuration 6 | chai.should() 7 | 8 | // Global stubs 9 | const sandbox = sinon.createSandbox() 10 | const stubs = { 11 | logger: { 12 | debug: sandbox.stub(), 13 | log: sandbox.stub(), 14 | warn: sandbox.stub(), 15 | err: sandbox.stub(), 16 | error: sandbox.stub(), 17 | }, 18 | } 19 | 20 | // SandboxedModule configuration 21 | SandboxedModule.configure({ 22 | requires: { 23 | 'logger-sharelatex': stubs.logger, 24 | }, 25 | globals: { Buffer, JSON, Math, console, process }, 26 | }) 27 | 28 | // Mocha hooks 29 | exports.mochaHooks = { 30 | beforeEach() { 31 | this.logger = stubs.logger 32 | }, 33 | 34 | afterEach() { 35 | sandbox.reset() 36 | }, 37 | } 38 | -------------------------------------------------------------------------------- /test/unit/js/DiffCodec/DiffCodecTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | handle-callback-err, 3 | no-return-assign, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 12 | */ 13 | const sinon = require('sinon') 14 | const { expect } = require('chai') 15 | const modulePath = '../../../../app/js/DiffCodec.js' 16 | const SandboxedModule = require('sandboxed-module') 17 | 18 | describe('DiffCodec', function () { 19 | beforeEach(function () { 20 | this.callback = sinon.stub() 21 | return (this.DiffCodec = SandboxedModule.require(modulePath)) 22 | }) 23 | 24 | return describe('diffAsShareJsOps', function () { 25 | it('should insert new text correctly', function (done) { 26 | this.before = ['hello world'] 27 | this.after = ['hello beautiful world'] 28 | return this.DiffCodec.diffAsShareJsOp( 29 | this.before, 30 | this.after, 31 | (error, ops) => { 32 | expect(ops).to.deep.equal([ 33 | { 34 | i: 'beautiful ', 35 | p: 6, 36 | }, 37 | ]) 38 | return done() 39 | } 40 | ) 41 | }) 42 | 43 | it('should shift later inserts by previous inserts', function (done) { 44 | this.before = ['the boy played with the ball'] 45 | this.after = ['the tall boy played with the red ball'] 46 | return this.DiffCodec.diffAsShareJsOp( 47 | this.before, 48 | this.after, 49 | (error, ops) => { 50 | expect(ops).to.deep.equal([ 51 | { i: 'tall ', p: 4 }, 52 | { i: 'red ', p: 29 }, 53 | ]) 54 | return done() 55 | } 56 | ) 57 | }) 58 | 59 | it('should delete text correctly', function (done) { 60 | this.before = ['hello beautiful world'] 61 | this.after = ['hello world'] 62 | return this.DiffCodec.diffAsShareJsOp( 63 | this.before, 64 | this.after, 65 | (error, ops) => { 66 | expect(ops).to.deep.equal([ 67 | { 68 | d: 'beautiful ', 69 | p: 6, 70 | }, 71 | ]) 72 | return done() 73 | } 74 | ) 75 | }) 76 | 77 | return it('should shift later deletes by the first deletes', function (done) { 78 | this.before = ['the tall boy played with the red ball'] 79 | this.after = ['the boy played with the ball'] 80 | return this.DiffCodec.diffAsShareJsOp( 81 | this.before, 82 | this.after, 83 | (error, ops) => { 84 | expect(ops).to.deep.equal([ 85 | { d: 'tall ', p: 4 }, 86 | { d: 'red ', p: 24 }, 87 | ]) 88 | return done() 89 | } 90 | ) 91 | }) 92 | }) 93 | }) 94 | -------------------------------------------------------------------------------- /test/unit/js/HistoryRedisManager/HistoryRedisManagerTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-return-assign, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS101: Remove unnecessary use of Array.from 11 | * DS102: Remove unnecessary code created because of implicit returns 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | const sinon = require('sinon') 15 | const modulePath = '../../../../app/js/HistoryRedisManager.js' 16 | const SandboxedModule = require('sandboxed-module') 17 | const Errors = require('../../../../app/js/Errors') 18 | 19 | describe('HistoryRedisManager', function () { 20 | beforeEach(function () { 21 | this.rclient = { 22 | auth() {}, 23 | exec: sinon.stub(), 24 | } 25 | this.rclient.multi = () => this.rclient 26 | this.HistoryRedisManager = SandboxedModule.require(modulePath, { 27 | requires: { 28 | '@overleaf/redis-wrapper': { createClient: () => this.rclient }, 29 | '@overleaf/settings': { 30 | redis: { 31 | history: (this.settings = { 32 | key_schema: { 33 | uncompressedHistoryOps({ doc_id }) { 34 | return `UncompressedHistoryOps:${doc_id}` 35 | }, 36 | docsWithHistoryOps({ project_id }) { 37 | return `DocsWithHistoryOps:${project_id}` 38 | }, 39 | }, 40 | }), 41 | }, 42 | }, 43 | }, 44 | }) 45 | this.doc_id = 'doc-id-123' 46 | this.project_id = 'project-id-123' 47 | return (this.callback = sinon.stub()) 48 | }) 49 | 50 | return describe('recordDocHasHistoryOps', function () { 51 | beforeEach(function () { 52 | this.ops = [{ op: [{ i: 'foo', p: 4 }] }, { op: [{ i: 'bar', p: 56 }] }] 53 | return (this.rclient.sadd = sinon.stub().yields()) 54 | }) 55 | 56 | describe('with ops', function () { 57 | beforeEach(function (done) { 58 | return this.HistoryRedisManager.recordDocHasHistoryOps( 59 | this.project_id, 60 | this.doc_id, 61 | this.ops, 62 | (...args) => { 63 | this.callback(...Array.from(args || [])) 64 | return done() 65 | } 66 | ) 67 | }) 68 | 69 | return it('should add the doc_id to the set of which records the project docs', function () { 70 | return this.rclient.sadd 71 | .calledWith(`DocsWithHistoryOps:${this.project_id}`, this.doc_id) 72 | .should.equal(true) 73 | }) 74 | }) 75 | 76 | return describe('with no ops', function () { 77 | beforeEach(function (done) { 78 | return this.HistoryRedisManager.recordDocHasHistoryOps( 79 | this.project_id, 80 | this.doc_id, 81 | [], 82 | (...args) => { 83 | this.callback(...Array.from(args || [])) 84 | return done() 85 | } 86 | ) 87 | }) 88 | 89 | it('should not add the doc_id to the set of which records the project docs', function () { 90 | return this.rclient.sadd.called.should.equal(false) 91 | }) 92 | 93 | return it('should call the callback with an error', function () { 94 | return this.callback 95 | .calledWith(sinon.match.instanceOf(Error)) 96 | .should.equal(true) 97 | }) 98 | }) 99 | }) 100 | }) 101 | -------------------------------------------------------------------------------- /test/unit/js/LockManager/CheckingTheLock.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS206: Consider reworking classes to avoid initClass 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | const sinon = require('sinon') 15 | const assert = require('assert') 16 | const path = require('path') 17 | const modulePath = path.join(__dirname, '../../../../app/js/LockManager.js') 18 | const project_id = 1234 19 | const doc_id = 5678 20 | const blockingKey = `Blocking:${doc_id}` 21 | const SandboxedModule = require('sandboxed-module') 22 | 23 | describe('LockManager - checking the lock', function () { 24 | let Profiler 25 | const existsStub = sinon.stub() 26 | 27 | const mocks = { 28 | '@overleaf/redis-wrapper': { 29 | createClient() { 30 | return { 31 | auth() {}, 32 | exists: existsStub, 33 | } 34 | }, 35 | }, 36 | './Metrics': { inc() {} }, 37 | './Profiler': (Profiler = (function () { 38 | Profiler = class Profiler { 39 | static initClass() { 40 | this.prototype.log = sinon.stub().returns({ end: sinon.stub() }) 41 | this.prototype.end = sinon.stub() 42 | } 43 | } 44 | Profiler.initClass() 45 | return Profiler 46 | })()), 47 | } 48 | const LockManager = SandboxedModule.require(modulePath, { requires: mocks }) 49 | 50 | it('should return true if the key does not exists', function (done) { 51 | existsStub.yields(null, '0') 52 | return LockManager.checkLock(doc_id, (err, free) => { 53 | free.should.equal(true) 54 | return done() 55 | }) 56 | }) 57 | 58 | return it('should return false if the key does exists', function (done) { 59 | existsStub.yields(null, '1') 60 | return LockManager.checkLock(doc_id, (err, free) => { 61 | free.should.equal(false) 62 | return done() 63 | }) 64 | }) 65 | }) 66 | -------------------------------------------------------------------------------- /test/unit/js/LockManager/ReleasingTheLock.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-return-assign, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS206: Consider reworking classes to avoid initClass 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | const sinon = require('sinon') 15 | const assert = require('assert') 16 | const path = require('path') 17 | const modulePath = path.join(__dirname, '../../../../app/js/LockManager.js') 18 | const project_id = 1234 19 | const doc_id = 5678 20 | const SandboxedModule = require('sandboxed-module') 21 | 22 | describe('LockManager - releasing the lock', function () { 23 | beforeEach(function () { 24 | let Profiler 25 | this.client = { 26 | auth() {}, 27 | eval: sinon.stub(), 28 | } 29 | const mocks = { 30 | '@overleaf/redis-wrapper': { 31 | createClient: () => this.client, 32 | }, 33 | '@overleaf/settings': { 34 | redis: { 35 | lock: { 36 | key_schema: { 37 | blockingKey({ doc_id }) { 38 | return `Blocking:${doc_id}` 39 | }, 40 | }, 41 | }, 42 | }, 43 | }, 44 | './Metrics': { inc() {} }, 45 | './Profiler': (Profiler = (function () { 46 | Profiler = class Profiler { 47 | static initClass() { 48 | this.prototype.log = sinon.stub().returns({ end: sinon.stub() }) 49 | this.prototype.end = sinon.stub() 50 | } 51 | } 52 | Profiler.initClass() 53 | return Profiler 54 | })()), 55 | } 56 | this.LockManager = SandboxedModule.require(modulePath, { requires: mocks }) 57 | this.lockValue = 'lock-value-stub' 58 | return (this.callback = sinon.stub()) 59 | }) 60 | 61 | describe('when the lock is current', function () { 62 | beforeEach(function () { 63 | this.client.eval = sinon.stub().yields(null, 1) 64 | return this.LockManager.releaseLock(doc_id, this.lockValue, this.callback) 65 | }) 66 | 67 | it('should clear the data from redis', function () { 68 | return this.client.eval 69 | .calledWith( 70 | this.LockManager.unlockScript, 71 | 1, 72 | `Blocking:${doc_id}`, 73 | this.lockValue 74 | ) 75 | .should.equal(true) 76 | }) 77 | 78 | return it('should call the callback', function () { 79 | return this.callback.called.should.equal(true) 80 | }) 81 | }) 82 | 83 | return describe('when the lock has expired', function () { 84 | beforeEach(function () { 85 | this.client.eval = sinon.stub().yields(null, 0) 86 | return this.LockManager.releaseLock(doc_id, this.lockValue, this.callback) 87 | }) 88 | 89 | return it('should return an error if the lock has expired', function () { 90 | return this.callback 91 | .calledWith(sinon.match.instanceOf(Error)) 92 | .should.equal(true) 93 | }) 94 | }) 95 | }) 96 | -------------------------------------------------------------------------------- /test/unit/js/LockManager/getLockTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-return-assign, 5 | no-unused-vars, 6 | */ 7 | // TODO: This file was created by bulk-decaffeinate. 8 | // Fix any style issues and re-enable lint. 9 | /* 10 | * decaffeinate suggestions: 11 | * DS101: Remove unnecessary use of Array.from 12 | * DS102: Remove unnecessary code created because of implicit returns 13 | * DS206: Consider reworking classes to avoid initClass 14 | * DS207: Consider shorter variations of null checks 15 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 16 | */ 17 | const sinon = require('sinon') 18 | const modulePath = '../../../../app/js/LockManager.js' 19 | const SandboxedModule = require('sandboxed-module') 20 | 21 | describe('LockManager - getting the lock', function () { 22 | beforeEach(function () { 23 | let Profiler 24 | this.LockManager = SandboxedModule.require(modulePath, { 25 | requires: { 26 | '@overleaf/redis-wrapper': { 27 | createClient: () => { 28 | return { auth() {} } 29 | }, 30 | }, 31 | './Metrics': { inc() {} }, 32 | './Profiler': (Profiler = (function () { 33 | Profiler = class Profiler { 34 | static initClass() { 35 | this.prototype.log = sinon.stub().returns({ end: sinon.stub() }) 36 | this.prototype.end = sinon.stub() 37 | } 38 | } 39 | Profiler.initClass() 40 | return Profiler 41 | })()), 42 | }, 43 | }) 44 | this.callback = sinon.stub() 45 | return (this.doc_id = 'doc-id-123') 46 | }) 47 | 48 | describe('when the lock is not set', function () { 49 | beforeEach(function (done) { 50 | this.lockValue = 'mock-lock-value' 51 | this.LockManager.tryLock = sinon 52 | .stub() 53 | .callsArgWith(1, null, true, this.lockValue) 54 | return this.LockManager.getLock(this.doc_id, (...args) => { 55 | this.callback(...Array.from(args || [])) 56 | return done() 57 | }) 58 | }) 59 | 60 | it('should try to get the lock', function () { 61 | return this.LockManager.tryLock.calledWith(this.doc_id).should.equal(true) 62 | }) 63 | 64 | it('should only need to try once', function () { 65 | return this.LockManager.tryLock.callCount.should.equal(1) 66 | }) 67 | 68 | return it('should return the callback with the lock value', function () { 69 | return this.callback.calledWith(null, this.lockValue).should.equal(true) 70 | }) 71 | }) 72 | 73 | describe('when the lock is initially set', function () { 74 | beforeEach(function (done) { 75 | this.lockValue = 'mock-lock-value' 76 | const startTime = Date.now() 77 | let tries = 0 78 | this.LockManager.LOCK_TEST_INTERVAL = 5 79 | this.LockManager.tryLock = (doc_id, callback) => { 80 | if (callback == null) { 81 | callback = function (error, isFree) {} 82 | } 83 | if (Date.now() - startTime < 20 || tries < 2) { 84 | tries = tries + 1 85 | return callback(null, false) 86 | } else { 87 | return callback(null, true, this.lockValue) 88 | } 89 | } 90 | sinon.spy(this.LockManager, 'tryLock') 91 | 92 | return this.LockManager.getLock(this.doc_id, (...args) => { 93 | this.callback(...Array.from(args || [])) 94 | return done() 95 | }) 96 | }) 97 | 98 | it('should call tryLock multiple times until free', function () { 99 | return (this.LockManager.tryLock.callCount > 1).should.equal(true) 100 | }) 101 | 102 | return it('should return the callback with the lock value', function () { 103 | return this.callback.calledWith(null, this.lockValue).should.equal(true) 104 | }) 105 | }) 106 | 107 | return describe('when the lock times out', function () { 108 | beforeEach(function (done) { 109 | const time = Date.now() 110 | this.LockManager.MAX_LOCK_WAIT_TIME = 5 111 | this.LockManager.tryLock = sinon.stub().callsArgWith(1, null, false) 112 | return this.LockManager.getLock(this.doc_id, (...args) => { 113 | this.callback(...Array.from(args || [])) 114 | return done() 115 | }) 116 | }) 117 | 118 | return it('should return the callback with an error', function () { 119 | return this.callback 120 | .calledWith( 121 | sinon.match 122 | .instanceOf(Error) 123 | .and(sinon.match.has('doc_id', this.doc_id)) 124 | ) 125 | .should.equal(true) 126 | }) 127 | }) 128 | }) 129 | -------------------------------------------------------------------------------- /test/unit/js/LockManager/tryLockTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-return-assign, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * DS206: Consider reworking classes to avoid initClass 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | const sinon = require('sinon') 15 | const modulePath = '../../../../app/js/LockManager.js' 16 | const SandboxedModule = require('sandboxed-module') 17 | 18 | describe('LockManager - trying the lock', function () { 19 | beforeEach(function () { 20 | let Profiler 21 | this.LockManager = SandboxedModule.require(modulePath, { 22 | requires: { 23 | '@overleaf/redis-wrapper': { 24 | createClient: () => { 25 | return { 26 | auth() {}, 27 | set: (this.set = sinon.stub()), 28 | } 29 | }, 30 | }, 31 | './Metrics': { inc() {} }, 32 | '@overleaf/settings': { 33 | redis: { 34 | lock: { 35 | key_schema: { 36 | blockingKey({ doc_id }) { 37 | return `Blocking:${doc_id}` 38 | }, 39 | }, 40 | }, 41 | }, 42 | }, 43 | './Profiler': 44 | (this.Profiler = Profiler = 45 | (function () { 46 | Profiler = class Profiler { 47 | static initClass() { 48 | this.prototype.log = sinon 49 | .stub() 50 | .returns({ end: sinon.stub() }) 51 | this.prototype.end = sinon.stub() 52 | } 53 | } 54 | Profiler.initClass() 55 | return Profiler 56 | })()), 57 | }, 58 | }) 59 | 60 | this.callback = sinon.stub() 61 | return (this.doc_id = 'doc-id-123') 62 | }) 63 | 64 | describe('when the lock is not set', function () { 65 | beforeEach(function () { 66 | this.lockValue = 'mock-lock-value' 67 | this.LockManager.randomLock = sinon.stub().returns(this.lockValue) 68 | this.set.callsArgWith(5, null, 'OK') 69 | return this.LockManager.tryLock(this.doc_id, this.callback) 70 | }) 71 | 72 | it('should set the lock key with an expiry if it is not set', function () { 73 | return this.set 74 | .calledWith(`Blocking:${this.doc_id}`, this.lockValue, 'EX', 30, 'NX') 75 | .should.equal(true) 76 | }) 77 | 78 | return it('should return the callback with true and the lock value', function () { 79 | return this.callback 80 | .calledWith(null, true, this.lockValue) 81 | .should.equal(true) 82 | }) 83 | }) 84 | 85 | describe('when the lock is already set', function () { 86 | beforeEach(function () { 87 | this.set.callsArgWith(5, null, null) 88 | return this.LockManager.tryLock(this.doc_id, this.callback) 89 | }) 90 | 91 | return it('should return the callback with false', function () { 92 | return this.callback.calledWith(null, false).should.equal(true) 93 | }) 94 | }) 95 | 96 | return describe('when it takes a long time for redis to set the lock', function () { 97 | beforeEach(function () { 98 | this.Profiler.prototype.end = () => 7000 // take a long time 99 | this.Profiler.prototype.log = sinon 100 | .stub() 101 | .returns({ end: this.Profiler.prototype.end }) 102 | this.lockValue = 'mock-lock-value' 103 | this.LockManager.randomLock = sinon.stub().returns(this.lockValue) 104 | this.LockManager.releaseLock = sinon.stub().callsArgWith(2, null) 105 | return this.set.callsArgWith(5, null, 'OK') 106 | }) 107 | 108 | describe('in all cases', function () { 109 | beforeEach(function () { 110 | return this.LockManager.tryLock(this.doc_id, this.callback) 111 | }) 112 | 113 | it('should set the lock key with an expiry if it is not set', function () { 114 | return this.set 115 | .calledWith(`Blocking:${this.doc_id}`, this.lockValue, 'EX', 30, 'NX') 116 | .should.equal(true) 117 | }) 118 | 119 | return it('should try to release the lock', function () { 120 | return this.LockManager.releaseLock 121 | .calledWith(this.doc_id, this.lockValue) 122 | .should.equal(true) 123 | }) 124 | }) 125 | 126 | describe('if the lock is released successfully', function () { 127 | beforeEach(function () { 128 | this.LockManager.releaseLock = sinon.stub().callsArgWith(2, null) 129 | return this.LockManager.tryLock(this.doc_id, this.callback) 130 | }) 131 | 132 | return it('should return the callback with false', function () { 133 | return this.callback.calledWith(null, false).should.equal(true) 134 | }) 135 | }) 136 | 137 | return describe('if the lock has already timed out', function () { 138 | beforeEach(function () { 139 | this.LockManager.releaseLock = sinon 140 | .stub() 141 | .callsArgWith(2, new Error('tried to release timed out lock')) 142 | return this.LockManager.tryLock(this.doc_id, this.callback) 143 | }) 144 | 145 | return it('should return the callback with an error', function () { 146 | return this.callback 147 | .calledWith(sinon.match.instanceOf(Error)) 148 | .should.equal(true) 149 | }) 150 | }) 151 | }) 152 | }) 153 | -------------------------------------------------------------------------------- /test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-return-assign, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS101: Remove unnecessary use of Array.from 11 | * DS102: Remove unnecessary code created because of implicit returns 12 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 13 | */ 14 | const sinon = require('sinon') 15 | const modulePath = '../../../../app/js/ProjectHistoryRedisManager.js' 16 | const SandboxedModule = require('sandboxed-module') 17 | const tk = require('timekeeper') 18 | 19 | describe('ProjectHistoryRedisManager', function () { 20 | beforeEach(function () { 21 | this.project_id = 'project-id-123' 22 | this.projectHistoryId = 'history-id-123' 23 | this.user_id = 'user-id-123' 24 | this.callback = sinon.stub() 25 | this.rclient = {} 26 | tk.freeze(new Date()) 27 | return (this.ProjectHistoryRedisManager = SandboxedModule.require( 28 | modulePath, 29 | { 30 | requires: { 31 | '@overleaf/settings': (this.settings = { 32 | redis: { 33 | project_history: { 34 | key_schema: { 35 | projectHistoryOps({ project_id }) { 36 | return `ProjectHistory:Ops:${project_id}` 37 | }, 38 | projectHistoryFirstOpTimestamp({ project_id }) { 39 | return `ProjectHistory:FirstOpTimestamp:${project_id}` 40 | }, 41 | }, 42 | }, 43 | }, 44 | }), 45 | '@overleaf/redis-wrapper': { 46 | createClient: () => this.rclient, 47 | }, 48 | './Metrics': (this.metrics = { summary: sinon.stub() }), 49 | }, 50 | } 51 | )) 52 | }) 53 | 54 | afterEach(function () { 55 | return tk.reset() 56 | }) 57 | 58 | describe('queueOps', function () { 59 | beforeEach(function () { 60 | this.ops = ['mock-op-1', 'mock-op-2'] 61 | this.multi = { exec: sinon.stub() } 62 | this.multi.rpush = sinon.stub() 63 | this.multi.setnx = sinon.stub() 64 | this.rclient.multi = () => this.multi 65 | // @rclient = multi: () => @multi 66 | return this.ProjectHistoryRedisManager.queueOps( 67 | this.project_id, 68 | ...Array.from(this.ops), 69 | this.callback 70 | ) 71 | }) 72 | 73 | it('should queue an update', function () { 74 | return this.multi.rpush 75 | .calledWithExactly( 76 | `ProjectHistory:Ops:${this.project_id}`, 77 | this.ops[0], 78 | this.ops[1] 79 | ) 80 | .should.equal(true) 81 | }) 82 | 83 | return it('should set the queue timestamp if not present', function () { 84 | return this.multi.setnx 85 | .calledWithExactly( 86 | `ProjectHistory:FirstOpTimestamp:${this.project_id}`, 87 | Date.now() 88 | ) 89 | .should.equal(true) 90 | }) 91 | }) 92 | 93 | describe('queueRenameEntity', function () { 94 | beforeEach(function () { 95 | this.file_id = 1234 96 | 97 | this.rawUpdate = { 98 | pathname: (this.pathname = '/old'), 99 | newPathname: (this.newPathname = '/new'), 100 | version: (this.version = 2), 101 | } 102 | 103 | this.ProjectHistoryRedisManager.queueOps = sinon.stub() 104 | return this.ProjectHistoryRedisManager.queueRenameEntity( 105 | this.project_id, 106 | this.projectHistoryId, 107 | 'file', 108 | this.file_id, 109 | this.user_id, 110 | this.rawUpdate, 111 | this.callback 112 | ) 113 | }) 114 | 115 | return it('should queue an update', function () { 116 | const update = { 117 | pathname: this.pathname, 118 | new_pathname: this.newPathname, 119 | meta: { 120 | user_id: this.user_id, 121 | ts: new Date(), 122 | }, 123 | version: this.version, 124 | projectHistoryId: this.projectHistoryId, 125 | file: this.file_id, 126 | } 127 | 128 | return this.ProjectHistoryRedisManager.queueOps 129 | .calledWithExactly( 130 | this.project_id, 131 | JSON.stringify(update), 132 | this.callback 133 | ) 134 | .should.equal(true) 135 | }) 136 | }) 137 | 138 | return describe('queueAddEntity', function () { 139 | beforeEach(function () { 140 | this.rclient.rpush = sinon.stub().yields() 141 | this.doc_id = 1234 142 | 143 | this.rawUpdate = { 144 | pathname: (this.pathname = '/old'), 145 | docLines: (this.docLines = 'a\nb'), 146 | version: (this.version = 2), 147 | url: (this.url = 'filestore.example.com'), 148 | } 149 | 150 | this.ProjectHistoryRedisManager.queueOps = sinon.stub() 151 | return this.ProjectHistoryRedisManager.queueAddEntity( 152 | this.project_id, 153 | this.projectHistoryId, 154 | 'doc', 155 | this.doc_id, 156 | this.user_id, 157 | this.rawUpdate, 158 | this.callback 159 | ) 160 | }) 161 | 162 | it('should queue an update', function () { 163 | const update = { 164 | pathname: this.pathname, 165 | docLines: this.docLines, 166 | url: this.url, 167 | meta: { 168 | user_id: this.user_id, 169 | ts: new Date(), 170 | }, 171 | version: this.version, 172 | projectHistoryId: this.projectHistoryId, 173 | doc: this.doc_id, 174 | } 175 | 176 | return this.ProjectHistoryRedisManager.queueOps 177 | .calledWithExactly( 178 | this.project_id, 179 | JSON.stringify(update), 180 | this.callback 181 | ) 182 | .should.equal(true) 183 | }) 184 | 185 | describe('queueResyncProjectStructure', function () { 186 | return it('should queue an update', function () {}) 187 | }) 188 | 189 | return describe('queueResyncDocContent', function () { 190 | return it('should queue an update', function () {}) 191 | }) 192 | }) 193 | }) 194 | -------------------------------------------------------------------------------- /test/unit/js/ProjectManager/flushAndDeleteProjectTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-return-assign, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS101: Remove unnecessary use of Array.from 11 | * DS102: Remove unnecessary code created because of implicit returns 12 | * DS206: Consider reworking classes to avoid initClass 13 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 14 | */ 15 | const sinon = require('sinon') 16 | const modulePath = '../../../../app/js/ProjectManager.js' 17 | const SandboxedModule = require('sandboxed-module') 18 | 19 | describe('ProjectManager - flushAndDeleteProject', function () { 20 | beforeEach(function () { 21 | let Timer 22 | this.ProjectManager = SandboxedModule.require(modulePath, { 23 | requires: { 24 | './RedisManager': (this.RedisManager = {}), 25 | './ProjectHistoryRedisManager': (this.ProjectHistoryRedisManager = {}), 26 | './DocumentManager': (this.DocumentManager = {}), 27 | './HistoryManager': (this.HistoryManager = { 28 | flushProjectChanges: sinon.stub().callsArg(2), 29 | }), 30 | './Metrics': (this.Metrics = { 31 | Timer: (Timer = (function () { 32 | Timer = class Timer { 33 | static initClass() { 34 | this.prototype.done = sinon.stub() 35 | } 36 | } 37 | Timer.initClass() 38 | return Timer 39 | })()), 40 | }), 41 | }, 42 | }) 43 | this.project_id = 'project-id-123' 44 | return (this.callback = sinon.stub()) 45 | }) 46 | 47 | describe('successfully', function () { 48 | beforeEach(function (done) { 49 | this.doc_ids = ['doc-id-1', 'doc-id-2', 'doc-id-3'] 50 | this.RedisManager.getDocIdsInProject = sinon 51 | .stub() 52 | .callsArgWith(1, null, this.doc_ids) 53 | this.DocumentManager.flushAndDeleteDocWithLock = sinon.stub().callsArg(3) 54 | return this.ProjectManager.flushAndDeleteProjectWithLocks( 55 | this.project_id, 56 | {}, 57 | error => { 58 | this.callback(error) 59 | return done() 60 | } 61 | ) 62 | }) 63 | 64 | it('should get the doc ids in the project', function () { 65 | return this.RedisManager.getDocIdsInProject 66 | .calledWith(this.project_id) 67 | .should.equal(true) 68 | }) 69 | 70 | it('should delete each doc in the project', function () { 71 | return Array.from(this.doc_ids).map(doc_id => 72 | this.DocumentManager.flushAndDeleteDocWithLock 73 | .calledWith(this.project_id, doc_id, {}) 74 | .should.equal(true) 75 | ) 76 | }) 77 | 78 | it('should flush project history', function () { 79 | return this.HistoryManager.flushProjectChanges 80 | .calledWith(this.project_id, {}) 81 | .should.equal(true) 82 | }) 83 | 84 | it('should call the callback without error', function () { 85 | return this.callback.calledWith(null).should.equal(true) 86 | }) 87 | 88 | return it('should time the execution', function () { 89 | return this.Metrics.Timer.prototype.done.called.should.equal(true) 90 | }) 91 | }) 92 | 93 | return describe('when a doc errors', function () { 94 | beforeEach(function (done) { 95 | this.doc_ids = ['doc-id-1', 'doc-id-2', 'doc-id-3'] 96 | this.RedisManager.getDocIdsInProject = sinon 97 | .stub() 98 | .callsArgWith(1, null, this.doc_ids) 99 | this.DocumentManager.flushAndDeleteDocWithLock = sinon.spy( 100 | (project_id, doc_id, options, callback) => { 101 | if (doc_id === 'doc-id-1') { 102 | return callback( 103 | (this.error = new Error('oops, something went wrong')) 104 | ) 105 | } else { 106 | return callback() 107 | } 108 | } 109 | ) 110 | return this.ProjectManager.flushAndDeleteProjectWithLocks( 111 | this.project_id, 112 | {}, 113 | error => { 114 | this.callback(error) 115 | return done() 116 | } 117 | ) 118 | }) 119 | 120 | it('should still flush each doc in the project', function () { 121 | return Array.from(this.doc_ids).map(doc_id => 122 | this.DocumentManager.flushAndDeleteDocWithLock 123 | .calledWith(this.project_id, doc_id, {}) 124 | .should.equal(true) 125 | ) 126 | }) 127 | 128 | it('should still flush project history', function () { 129 | return this.HistoryManager.flushProjectChanges 130 | .calledWith(this.project_id, {}) 131 | .should.equal(true) 132 | }) 133 | 134 | it('should record the error', function () { 135 | return this.logger.error 136 | .calledWith( 137 | { err: this.error, projectId: this.project_id, docId: 'doc-id-1' }, 138 | 'error deleting doc' 139 | ) 140 | .should.equal(true) 141 | }) 142 | 143 | it('should call the callback with an error', function () { 144 | return this.callback 145 | .calledWith(sinon.match.instanceOf(Error)) 146 | .should.equal(true) 147 | }) 148 | 149 | return it('should time the execution', function () { 150 | return this.Metrics.Timer.prototype.done.called.should.equal(true) 151 | }) 152 | }) 153 | }) 154 | -------------------------------------------------------------------------------- /test/unit/js/ProjectManager/flushProjectTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | handle-callback-err, 4 | no-return-assign, 5 | no-unused-vars, 6 | */ 7 | // TODO: This file was created by bulk-decaffeinate. 8 | // Fix any style issues and re-enable lint. 9 | /* 10 | * decaffeinate suggestions: 11 | * DS101: Remove unnecessary use of Array.from 12 | * DS102: Remove unnecessary code created because of implicit returns 13 | * DS206: Consider reworking classes to avoid initClass 14 | * DS207: Consider shorter variations of null checks 15 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 16 | */ 17 | const sinon = require('sinon') 18 | const modulePath = '../../../../app/js/ProjectManager.js' 19 | const SandboxedModule = require('sandboxed-module') 20 | 21 | describe('ProjectManager - flushProject', function () { 22 | beforeEach(function () { 23 | let Timer 24 | this.ProjectManager = SandboxedModule.require(modulePath, { 25 | requires: { 26 | './RedisManager': (this.RedisManager = {}), 27 | './ProjectHistoryRedisManager': (this.ProjectHistoryRedisManager = {}), 28 | './DocumentManager': (this.DocumentManager = {}), 29 | './HistoryManager': (this.HistoryManager = {}), 30 | './Metrics': (this.Metrics = { 31 | Timer: (Timer = (function () { 32 | Timer = class Timer { 33 | static initClass() { 34 | this.prototype.done = sinon.stub() 35 | } 36 | } 37 | Timer.initClass() 38 | return Timer 39 | })()), 40 | }), 41 | }, 42 | }) 43 | this.project_id = 'project-id-123' 44 | return (this.callback = sinon.stub()) 45 | }) 46 | 47 | describe('successfully', function () { 48 | beforeEach(function (done) { 49 | this.doc_ids = ['doc-id-1', 'doc-id-2', 'doc-id-3'] 50 | this.RedisManager.getDocIdsInProject = sinon 51 | .stub() 52 | .callsArgWith(1, null, this.doc_ids) 53 | this.DocumentManager.flushDocIfLoadedWithLock = sinon.stub().callsArg(2) 54 | return this.ProjectManager.flushProjectWithLocks( 55 | this.project_id, 56 | error => { 57 | this.callback(error) 58 | return done() 59 | } 60 | ) 61 | }) 62 | 63 | it('should get the doc ids in the project', function () { 64 | return this.RedisManager.getDocIdsInProject 65 | .calledWith(this.project_id) 66 | .should.equal(true) 67 | }) 68 | 69 | it('should flush each doc in the project', function () { 70 | return Array.from(this.doc_ids).map(doc_id => 71 | this.DocumentManager.flushDocIfLoadedWithLock 72 | .calledWith(this.project_id, doc_id) 73 | .should.equal(true) 74 | ) 75 | }) 76 | 77 | it('should call the callback without error', function () { 78 | return this.callback.calledWith(null).should.equal(true) 79 | }) 80 | 81 | return it('should time the execution', function () { 82 | return this.Metrics.Timer.prototype.done.called.should.equal(true) 83 | }) 84 | }) 85 | 86 | return describe('when a doc errors', function () { 87 | beforeEach(function (done) { 88 | this.doc_ids = ['doc-id-1', 'doc-id-2', 'doc-id-3'] 89 | this.RedisManager.getDocIdsInProject = sinon 90 | .stub() 91 | .callsArgWith(1, null, this.doc_ids) 92 | this.DocumentManager.flushDocIfLoadedWithLock = sinon.spy( 93 | (project_id, doc_id, callback) => { 94 | if (callback == null) { 95 | callback = function (error) {} 96 | } 97 | if (doc_id === 'doc-id-1') { 98 | return callback( 99 | (this.error = new Error('oops, something went wrong')) 100 | ) 101 | } else { 102 | return callback() 103 | } 104 | } 105 | ) 106 | return this.ProjectManager.flushProjectWithLocks( 107 | this.project_id, 108 | error => { 109 | this.callback(error) 110 | return done() 111 | } 112 | ) 113 | }) 114 | 115 | it('should still flush each doc in the project', function () { 116 | return Array.from(this.doc_ids).map(doc_id => 117 | this.DocumentManager.flushDocIfLoadedWithLock 118 | .calledWith(this.project_id, doc_id) 119 | .should.equal(true) 120 | ) 121 | }) 122 | 123 | it('should record the error', function () { 124 | return this.logger.error 125 | .calledWith( 126 | { err: this.error, projectId: this.project_id, docId: 'doc-id-1' }, 127 | 'error flushing doc' 128 | ) 129 | .should.equal(true) 130 | }) 131 | 132 | it('should call the callback with an error', function () { 133 | return this.callback 134 | .calledWith(sinon.match.instanceOf(Error)) 135 | .should.equal(true) 136 | }) 137 | 138 | return it('should time the execution', function () { 139 | return this.Metrics.Timer.prototype.done.called.should.equal(true) 140 | }) 141 | }) 142 | }) 143 | -------------------------------------------------------------------------------- /test/unit/js/RateLimitManager/RateLimitManager.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-return-assign, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS206: Consider reworking classes to avoid initClass 11 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 12 | */ 13 | const sinon = require('sinon') 14 | const { expect } = require('chai') 15 | const modulePath = '../../../../app/js/RateLimitManager.js' 16 | const SandboxedModule = require('sandboxed-module') 17 | 18 | describe('RateLimitManager', function () { 19 | beforeEach(function () { 20 | let Timer 21 | this.RateLimitManager = SandboxedModule.require(modulePath, { 22 | requires: { 23 | '@overleaf/settings': (this.settings = {}), 24 | './Metrics': (this.Metrics = { 25 | Timer: (Timer = (function () { 26 | Timer = class Timer { 27 | static initClass() { 28 | this.prototype.done = sinon.stub() 29 | } 30 | } 31 | Timer.initClass() 32 | return Timer 33 | })()), 34 | gauge: sinon.stub(), 35 | }), 36 | }, 37 | }) 38 | this.callback = sinon.stub() 39 | return (this.RateLimiter = new this.RateLimitManager(1)) 40 | }) 41 | 42 | describe('for a single task', function () { 43 | beforeEach(function () { 44 | this.task = sinon.stub() 45 | return this.RateLimiter.run(this.task, this.callback) 46 | }) 47 | 48 | it('should execute the task in the background', function () { 49 | return this.task.called.should.equal(true) 50 | }) 51 | 52 | it('should call the callback', function () { 53 | return this.callback.called.should.equal(true) 54 | }) 55 | 56 | return it('should finish with a worker count of one', function () { 57 | // because it's in the background 58 | return expect(this.RateLimiter.ActiveWorkerCount).to.equal(1) 59 | }) 60 | }) 61 | 62 | describe('for multiple tasks', function () { 63 | beforeEach(function (done) { 64 | this.task = sinon.stub() 65 | this.finalTask = sinon.stub() 66 | const task = cb => { 67 | this.task() 68 | return setTimeout(cb, 100) 69 | } 70 | const finalTask = cb => { 71 | this.finalTask() 72 | return setTimeout(cb, 100) 73 | } 74 | this.RateLimiter.run(task, this.callback) 75 | this.RateLimiter.run(task, this.callback) 76 | this.RateLimiter.run(task, this.callback) 77 | return this.RateLimiter.run(finalTask, err => { 78 | this.callback(err) 79 | return done() 80 | }) 81 | }) 82 | 83 | it('should execute the first three tasks', function () { 84 | return this.task.calledThrice.should.equal(true) 85 | }) 86 | 87 | it('should execute the final task', function () { 88 | return this.finalTask.called.should.equal(true) 89 | }) 90 | 91 | it('should call the callback', function () { 92 | return this.callback.called.should.equal(true) 93 | }) 94 | 95 | return it('should finish with worker count of zero', function () { 96 | return expect(this.RateLimiter.ActiveWorkerCount).to.equal(0) 97 | }) 98 | }) 99 | 100 | return describe('for a mixture of long-running tasks', function () { 101 | beforeEach(function (done) { 102 | this.task = sinon.stub() 103 | this.finalTask = sinon.stub() 104 | const finalTask = cb => { 105 | this.finalTask() 106 | return setTimeout(cb, 100) 107 | } 108 | this.RateLimiter.run(this.task, this.callback) 109 | this.RateLimiter.run(this.task, this.callback) 110 | this.RateLimiter.run(this.task, this.callback) 111 | return this.RateLimiter.run(finalTask, err => { 112 | this.callback(err) 113 | return done() 114 | }) 115 | }) 116 | 117 | it('should execute the first three tasks', function () { 118 | return this.task.calledThrice.should.equal(true) 119 | }) 120 | 121 | it('should execute the final task', function () { 122 | return this.finalTask.called.should.equal(true) 123 | }) 124 | 125 | it('should call the callback', function () { 126 | return this.callback.called.should.equal(true) 127 | }) 128 | 129 | return it('should finish with worker count of three', function () { 130 | return expect(this.RateLimiter.ActiveWorkerCount).to.equal(3) 131 | }) 132 | }) 133 | }) 134 | -------------------------------------------------------------------------------- /test/unit/js/RealTimeRedisManager/RealTimeRedisManagerTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | camelcase, 3 | no-return-assign, 4 | no-unused-vars, 5 | */ 6 | // TODO: This file was created by bulk-decaffeinate. 7 | // Fix any style issues and re-enable lint. 8 | /* 9 | * decaffeinate suggestions: 10 | * DS102: Remove unnecessary code created because of implicit returns 11 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 12 | */ 13 | const sinon = require('sinon') 14 | const modulePath = '../../../../app/js/RealTimeRedisManager.js' 15 | const SandboxedModule = require('sandboxed-module') 16 | const Errors = require('../../../../app/js/Errors') 17 | 18 | describe('RealTimeRedisManager', function () { 19 | beforeEach(function () { 20 | this.rclient = { 21 | auth() {}, 22 | exec: sinon.stub(), 23 | } 24 | this.rclient.multi = () => this.rclient 25 | this.pubsubClient = { publish: sinon.stub() } 26 | this.RealTimeRedisManager = SandboxedModule.require(modulePath, { 27 | requires: { 28 | '@overleaf/redis-wrapper': { 29 | createClient: config => 30 | config.name === 'pubsub' ? this.pubsubClient : this.rclient, 31 | }, 32 | '@overleaf/settings': { 33 | redis: { 34 | documentupdater: (this.settings = { 35 | key_schema: { 36 | pendingUpdates({ doc_id }) { 37 | return `PendingUpdates:${doc_id}` 38 | }, 39 | }, 40 | }), 41 | pubsub: { 42 | name: 'pubsub', 43 | }, 44 | }, 45 | }, 46 | crypto: (this.crypto = { 47 | randomBytes: sinon 48 | .stub() 49 | .withArgs(4) 50 | .returns(Buffer.from([0x1, 0x2, 0x3, 0x4])), 51 | }), 52 | os: (this.os = { hostname: sinon.stub().returns('somehost') }), 53 | './Metrics': (this.metrics = { summary: sinon.stub() }), 54 | }, 55 | }) 56 | 57 | this.doc_id = 'doc-id-123' 58 | this.project_id = 'project-id-123' 59 | return (this.callback = sinon.stub()) 60 | }) 61 | 62 | describe('getPendingUpdatesForDoc', function () { 63 | beforeEach(function () { 64 | this.rclient.lrange = sinon.stub() 65 | return (this.rclient.ltrim = sinon.stub()) 66 | }) 67 | 68 | describe('successfully', function () { 69 | beforeEach(function () { 70 | this.updates = [ 71 | { op: [{ i: 'foo', p: 4 }] }, 72 | { op: [{ i: 'foo', p: 4 }] }, 73 | ] 74 | this.jsonUpdates = this.updates.map(update => JSON.stringify(update)) 75 | this.rclient.exec = sinon 76 | .stub() 77 | .callsArgWith(0, null, [this.jsonUpdates]) 78 | return this.RealTimeRedisManager.getPendingUpdatesForDoc( 79 | this.doc_id, 80 | this.callback 81 | ) 82 | }) 83 | 84 | it('should get the pending updates', function () { 85 | return this.rclient.lrange 86 | .calledWith(`PendingUpdates:${this.doc_id}`, 0, 7) 87 | .should.equal(true) 88 | }) 89 | 90 | it('should delete the pending updates', function () { 91 | return this.rclient.ltrim 92 | .calledWith(`PendingUpdates:${this.doc_id}`, 8, -1) 93 | .should.equal(true) 94 | }) 95 | 96 | return it('should call the callback with the updates', function () { 97 | return this.callback.calledWith(null, this.updates).should.equal(true) 98 | }) 99 | }) 100 | 101 | return describe("when the JSON doesn't parse", function () { 102 | beforeEach(function () { 103 | this.jsonUpdates = [ 104 | JSON.stringify({ op: [{ i: 'foo', p: 4 }] }), 105 | 'broken json', 106 | ] 107 | this.rclient.exec = sinon 108 | .stub() 109 | .callsArgWith(0, null, [this.jsonUpdates]) 110 | return this.RealTimeRedisManager.getPendingUpdatesForDoc( 111 | this.doc_id, 112 | this.callback 113 | ) 114 | }) 115 | 116 | return it('should return an error to the callback', function () { 117 | return this.callback 118 | .calledWith(sinon.match.has('name', 'SyntaxError')) 119 | .should.equal(true) 120 | }) 121 | }) 122 | }) 123 | 124 | describe('getUpdatesLength', function () { 125 | beforeEach(function () { 126 | this.rclient.llen = sinon.stub().yields(null, (this.length = 3)) 127 | return this.RealTimeRedisManager.getUpdatesLength( 128 | this.doc_id, 129 | this.callback 130 | ) 131 | }) 132 | 133 | it('should look up the length', function () { 134 | return this.rclient.llen 135 | .calledWith(`PendingUpdates:${this.doc_id}`) 136 | .should.equal(true) 137 | }) 138 | 139 | return it('should return the length', function () { 140 | return this.callback.calledWith(null, this.length).should.equal(true) 141 | }) 142 | }) 143 | 144 | return describe('sendData', function () { 145 | beforeEach(function () { 146 | this.message_id = 'doc:somehost:01020304-0' 147 | return this.RealTimeRedisManager.sendData({ op: 'thisop' }) 148 | }) 149 | 150 | it('should send the op with a message id', function () { 151 | return this.pubsubClient.publish 152 | .calledWith( 153 | 'applied-ops', 154 | JSON.stringify({ op: 'thisop', _id: this.message_id }) 155 | ) 156 | .should.equal(true) 157 | }) 158 | 159 | return it('should track the payload size', function () { 160 | return this.metrics.summary 161 | .calledWith( 162 | 'redis.publish.applied-ops', 163 | JSON.stringify({ op: 'thisop', _id: this.message_id }).length 164 | ) 165 | .should.equal(true) 166 | }) 167 | }) 168 | }) 169 | -------------------------------------------------------------------------------- /test/unit/js/ShareJsDB/ShareJsDBTests.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable 2 | no-return-assign, 3 | no-unused-vars, 4 | */ 5 | // TODO: This file was created by bulk-decaffeinate. 6 | // Fix any style issues and re-enable lint. 7 | /* 8 | * decaffeinate suggestions: 9 | * DS102: Remove unnecessary code created because of implicit returns 10 | * DS207: Consider shorter variations of null checks 11 | * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md 12 | */ 13 | const sinon = require('sinon') 14 | const { expect } = require('chai') 15 | const modulePath = '../../../../app/js/ShareJsDB.js' 16 | const SandboxedModule = require('sandboxed-module') 17 | const Errors = require('../../../../app/js/Errors') 18 | 19 | describe('ShareJsDB', function () { 20 | beforeEach(function () { 21 | this.doc_id = 'document-id' 22 | this.project_id = 'project-id' 23 | this.doc_key = `${this.project_id}:${this.doc_id}` 24 | this.callback = sinon.stub() 25 | this.ShareJsDB = SandboxedModule.require(modulePath, { 26 | requires: { 27 | './RedisManager': (this.RedisManager = {}), 28 | './Errors': Errors, 29 | }, 30 | }) 31 | 32 | this.version = 42 33 | this.lines = ['one', 'two', 'three'] 34 | return (this.db = new this.ShareJsDB( 35 | this.project_id, 36 | this.doc_id, 37 | this.lines, 38 | this.version 39 | )) 40 | }) 41 | 42 | describe('getSnapshot', function () { 43 | describe('successfully', function () { 44 | beforeEach(function () { 45 | return this.db.getSnapshot(this.doc_key, this.callback) 46 | }) 47 | 48 | it('should return the doc lines', function () { 49 | return this.callback.args[0][1].snapshot.should.equal( 50 | this.lines.join('\n') 51 | ) 52 | }) 53 | 54 | it('should return the doc version', function () { 55 | return this.callback.args[0][1].v.should.equal(this.version) 56 | }) 57 | 58 | return it('should return the type as text', function () { 59 | return this.callback.args[0][1].type.should.equal('text') 60 | }) 61 | }) 62 | 63 | return describe('when the key does not match', function () { 64 | beforeEach(function () { 65 | return this.db.getSnapshot('bad:key', this.callback) 66 | }) 67 | 68 | return it('should return the callback with a NotFoundError', function () { 69 | return this.callback 70 | .calledWith(sinon.match.instanceOf(Errors.NotFoundError)) 71 | .should.equal(true) 72 | }) 73 | }) 74 | }) 75 | 76 | describe('getOps', function () { 77 | describe('with start == end', function () { 78 | beforeEach(function () { 79 | this.start = this.end = 42 80 | return this.db.getOps(this.doc_key, this.start, this.end, this.callback) 81 | }) 82 | 83 | return it('should return an empty array', function () { 84 | return this.callback.calledWith(null, []).should.equal(true) 85 | }) 86 | }) 87 | 88 | describe('with a non empty range', function () { 89 | beforeEach(function () { 90 | this.start = 35 91 | this.end = 42 92 | this.RedisManager.getPreviousDocOps = sinon 93 | .stub() 94 | .callsArgWith(3, null, this.ops) 95 | return this.db.getOps(this.doc_key, this.start, this.end, this.callback) 96 | }) 97 | 98 | it('should get the range from redis', function () { 99 | return this.RedisManager.getPreviousDocOps 100 | .calledWith(this.doc_id, this.start, this.end - 1) 101 | .should.equal(true) 102 | }) 103 | 104 | return it('should return the ops', function () { 105 | return this.callback.calledWith(null, this.ops).should.equal(true) 106 | }) 107 | }) 108 | 109 | return describe('with no specified end', function () { 110 | beforeEach(function () { 111 | this.start = 35 112 | this.end = null 113 | this.RedisManager.getPreviousDocOps = sinon 114 | .stub() 115 | .callsArgWith(3, null, this.ops) 116 | return this.db.getOps(this.doc_key, this.start, this.end, this.callback) 117 | }) 118 | 119 | return it('should get until the end of the list', function () { 120 | return this.RedisManager.getPreviousDocOps 121 | .calledWith(this.doc_id, this.start, -1) 122 | .should.equal(true) 123 | }) 124 | }) 125 | }) 126 | 127 | return describe('writeOps', function () { 128 | return describe('writing an op', function () { 129 | beforeEach(function () { 130 | this.opData = { 131 | op: { p: 20, t: 'foo' }, 132 | meta: { source: 'bar' }, 133 | v: this.version, 134 | } 135 | return this.db.writeOp(this.doc_key, this.opData, this.callback) 136 | }) 137 | 138 | it('should write into appliedOps', function () { 139 | return expect(this.db.appliedOps[this.doc_key]).to.deep.equal([ 140 | this.opData, 141 | ]) 142 | }) 143 | 144 | return it('should call the callback without an error', function () { 145 | this.callback.called.should.equal(true) 146 | return (this.callback.args[0][0] != null).should.equal(false) 147 | }) 148 | }) 149 | }) 150 | }) 151 | --------------------------------------------------------------------------------