├── nodejs
    ├── config.json
    ├── package.json
    ├── monitor.js
    ├── stress.js
    ├── stress_regular_rest.js
    └── stress_vf.js
├── apex
    ├── Test.vfp
    └── LongTxn.cls
└── README.md


/nodejs/config.json:
--------------------------------------------------------------------------------
1 | {
2 | 	"url": "https://test.salesforce.com",
3 | 	"username": "<username>",
4 | 	"password": "<password>",
5 | 	"numberOfInstances": 20
6 | }
7 | 


--------------------------------------------------------------------------------
/nodejs/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "long-request",
 3 |   "version": "1.0.0",
 4 |   "description": "",
 5 |   "main": "index.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1"
 8 |   },
 9 |   "author": "",
10 |   "license": "ISC",
11 |   "dependencies": {
12 |     "jsforce": "^1.7.1",
13 |     "moment": "^2.17.0"
14 |   }
15 | }
16 | 


--------------------------------------------------------------------------------
/apex/Test.vfp:
--------------------------------------------------------------------------------
 1 | <apex:page controller="LongTxn">
 2 |     <script type="text/javascript">
 3 |     function getToken() {
 4 |         Visualforce.remoting.Manager.invokeAction(
 5 |             '{!$RemoteAction.LongTxn.remoteGet}', 
 6 |             function(result, event){
 7 |                 console.log('done!');
 8 |             }
 9 |         );
10 |     }
11 |     getToken();
12 |     </script>
13 | </apex:page>


--------------------------------------------------------------------------------
/apex/LongTxn.cls:
--------------------------------------------------------------------------------
 1 | @RestResource(urlMapping='/LongTxn')
 2 | global with sharing class LongTxn {
 3 |     /* Rest resources */
 4 |     @HttpPost
 5 |     global static void handlePost() {
 6 |         System.debug(LoggingLevel.ERROR, 'HandlePost called');
 7 |         RestResponse res = RestContext.response;
 8 |         res.addHeader('Content-Type', 'application/json');
 9 |         res.responseBody = Blob.valueOf('{}');
10 |     }
11 |     @HttpGet
12 |     global static void handleGet() {
13 |         System.debug(LoggingLevel.ERROR, 'HandleGet called');
14 |         RestResponse res = RestContext.response;
15 |         res.addHeader('Content-Type', 'application/json');
16 |         res.responseBody = Blob.valueOf('{}');
17 |     }
18 |     /* Remote action */
19 |     @RemoteAction
20 |     public static String remoteGet() {
21 |         System.debug(LoggingLevel.ERROR, 'RemoteGet called');
22 |         return 'RemoteGet response';
23 |     }
24 | }
25 | 


--------------------------------------------------------------------------------
/nodejs/monitor.js:
--------------------------------------------------------------------------------
 1 | /* This script monitors Salesforces' webservice availability */
 2 | var jsforce = require('jsforce');
 3 | var fs = require('fs');
 4 | var moment = require('moment');
 5 | var config = JSON.parse(fs.readFileSync('./config.json'));
 6 | 
 7 | class OrgMonitor {
 8 | 	constructor(url, username, password) {
 9 | 		this.url = url;
10 | 		this.username = username;
11 | 		this.password = password;
12 | 		this.cycle = 0;
13 | 		setInterval(this.check.bind(this), 2000);
14 | 	}
15 | 	check() {
16 | 		var localCycle = this.cycle;
17 | 		this.cycle++;
18 | 		var conn = new jsforce.Connection({
19 | 			loginUrl: this.url
20 | 		});
21 | 		var start = +new Date(), end;
22 | 		this.log(localCycle, 'Cycle started');
23 | 		try {
24 | 			conn.login(this.username, this.password, (err, res) => {
25 | 				if (err) {
26 | 					end = +new Date();
27 | 					this.log(localCycle, 'Error logging in: ' + err.message + ' trace: ' + JSON.stringify(err.stack) + ' took ' + (end - start) + 'ms');
28 | 					return console.error(err);
29 | 				}
30 | 				conn.apex.get("/services/apexrest/LongTxn", (err, res) => {
31 | 					end = +new Date();
32 | 					if (err) {
33 | 						this.log(localCycle, 'Error performing apex: ' + err.message + ' trace: ' + JSON.stringify(err.stack) + ' took ' + (end - start) + 'ms');
34 | 						return console.error(err);
35 | 					}
36 | 					this.log(localCycle, 'ALL OK - took ' + (end - start) + ' ms');
37 | 					return;
38 | 				});
39 | 			});
40 | 		} catch (e) {
41 | 			end = +new Date();
42 | 			this.log(localCycle, 'Exception',  + err.message + ' trace: ' + JSON.stringify(err.stack) + ' took ' + (end - start) + 'ms');
43 | 		}
44 | 	}
45 | 	log(currentCycle, data) {
46 | 		console.log('[' + moment().format() + ']', currentCycle, JSON.stringify(data));
47 | 	}
48 | }
49 | 
50 | var monitor = new OrgMonitor(config.url, config.username, config.password)
51 | 


--------------------------------------------------------------------------------
/nodejs/stress.js:
--------------------------------------------------------------------------------
 1 | /* This script attempts to create a large number of concurrent requests */
 2 | var fs = require('fs');
 3 | var jsforce = require('jsforce');
 4 | var config = JSON.parse(fs.readFileSync('config.json').toString());
 5 | var https = require('https');
 6 | 
 7 | var writeDelay = 20000;
 8 | var endDelay = 40000;
 9 | 
10 | class Instance {
11 | 	constructor(accessToken, url) {
12 | 		this.accessToken = accessToken;
13 | 		this.url = url;
14 | 
15 | 		this.start();
16 | 		setTimeout(this.write.bind(this, '{"key": "value"}'), writeDelay);
17 | 		setTimeout(this.end.bind(this), endDelay);
18 | 	}
19 | 	start() {
20 | 		var options = {
21 | 			port: 443,
22 | 			hostname: this.url.replace('https://', ''),
23 | 			path: '/services/apexrest/LongTxn',
24 | 			method: 'POST',
25 | 			headers: {
26 | 				'Content-Type': 'application/json',
27 | 				'Authorization': 'OAuth ' + this.accessToken,
28 | 				'Transfer-Encoding': 'chunked'
29 | 			}
30 | 		};
31 | 
32 | 		this.req = https.request(options);
33 | 		this.req.setTimeout(150000, (err) => {
34 | 			console.error('Timed out');
35 | 		});
36 | 		this.req.on('response', (response) => {
37 | 			response.on('data', function (chunk) {
38 | 				console.log('response: ' + chunk);
39 | 			});
40 | 		})
41 | 		this.req.on('error', (error) => {
42 | 			console.log('error', error)
43 | 		})
44 | 		this.req.flushHeaders();
45 | 	}
46 | 	write(data) {
47 | 		console.log('writing data', data);
48 | 		this.req.write(data, 'UTF-8', (err, result) => {
49 | 			if (err) {
50 | 				return console.error(err);
51 | 			}
52 | 		})
53 | 	}
54 | 	end() {
55 | 		console.log('ending request');
56 | 		this.req.end((err, result) => {
57 | 			if (err) {
58 | 				return console.error(err);
59 | 			}
60 | 		})
61 | 	}
62 | }
63 | 
64 | var instances = [];
65 | 
66 | var conn = new jsforce.Connection({
67 | 	loginUrl : config.url
68 | });
69 | conn.login(config.username, config.password, function(err, userInfo) {
70 | 	if (err) {
71 | 		return console.error(err);
72 | 	}
73 | 	for (var i = 0; i < config.numberOfInstances; i++) {
74 | 		instances.push(new Instance(conn.accessToken, conn.instanceUrl));
75 | 	}
76 | });
77 | 


--------------------------------------------------------------------------------
/nodejs/stress_regular_rest.js:
--------------------------------------------------------------------------------
 1 | /* This script attempts to create a large number of concurrent requests */
 2 | var fs = require('fs');
 3 | var jsforce = require('jsforce');
 4 | var config = JSON.parse(fs.readFileSync('config.json').toString());
 5 | var https = require('https');
 6 | 
 7 | var writeDelay = 20000;
 8 | var endDelay = 40000;
 9 | 
10 | class Instance {
11 | 	constructor(accessToken, url) {
12 | 		this.accessToken = accessToken;
13 | 		this.url = url;
14 | 
15 | 		this.start();
16 | 		setTimeout(this.write.bind(this, '{"Name" : "Express Logistics and Transport"}'), writeDelay);
17 | 		setTimeout(this.end.bind(this), endDelay);
18 | 	}
19 | 	start() {
20 | 		var options = {
21 | 			port: 443,
22 | 			hostname: this.url.replace('https://', ''),
23 | 			path: '/services/data/v38.0/sobjects/Account/',
24 | 			method: 'POST',
25 | 			headers: {
26 | 				'Content-Type': 'application/json',
27 | 				'Authorization': 'OAuth ' + this.accessToken,
28 | 				'Transfer-Encoding': 'chunked'
29 | 			}
30 | 		};
31 | 
32 | 		this.req = https.request(options);
33 | 		this.req.setTimeout(150000, (err) => {
34 | 			console.error('Timed out');
35 | 		});
36 | 		this.req.on('response', (response) => {
37 | 			response.on('data', function (chunk) {
38 | 				console.log('response: ' + chunk);
39 | 			});
40 | 		})
41 | 		this.req.on('error', (error) => {
42 | 			console.log('error', error)
43 | 		})
44 | 		this.req.flushHeaders();
45 | 	}
46 | 	write(data) {
47 | 		console.log('writing data', data);
48 | 		this.req.write(data, 'UTF-8', (err, result) => {
49 | 			if (err) {
50 | 				return console.error(err);
51 | 			}
52 | 		})
53 | 	}
54 | 	end() {
55 | 		console.log('ending request');
56 | 		this.req.end((err, result) => {
57 | 			if (err) {
58 | 				return console.error(err);
59 | 			}
60 | 		})
61 | 	}
62 | }
63 | 
64 | var instances = [];
65 | 
66 | var conn = new jsforce.Connection({
67 | 	loginUrl : config.url
68 | });
69 | conn.login(config.username, config.password, function(err, userInfo) {
70 | 	if (err) {
71 | 		return console.error(err);
72 | 	}
73 | 	for (var i = 0; i < config.numberOfInstances; i++) {
74 | 		instances.push(new Instance(conn.accessToken, conn.instanceUrl));
75 | 	}
76 | });
77 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | SFDC-ConcurrentPerOrgLongTxn-Reproduction
 2 | =========
 3 | This project contains code to reproduce the seemingly newly introduced "ConcurrentPerOrgLongTxn" exception using a custom APEX REST-service.
 4 | 
 5 | Usage
 6 | ------
 7 | 1. Clone repository
 8 | 2. Determine what SF org you want to use for testing
 9 | 3. Deploy the apex/LongTxn.cls class to your testing-org
10 | 4. Enter your user credentials (with API access and access to the LongTxn class) in nodejs/config.json
11 | 5. Run 'npm install' in the nodejs/ directory to fetch all dependencies
12 | 6. Run nodejs/monitor.js in a terminal which you keep visible
13 | 7. Run nodejs/stress.js in a secondary terminal
14 | 8. After a few seconds the monitor.js script will spew a number of "ConcurrentPerOrgLongTxn" errors
15 | 
16 | Background
17 | ------
18 | - It seems that Salesforce has introduced a new exception type "ConcurrentPerOrgLongTxn" which is more or less equal to "ConcurrentPerOrgLongApex" - not documented anywhere but confirmed by first-line support that it should be treated as being equal
19 | - We have run across this exception using a mobile application with ~400 concurrent users talking to a number of custom REST-services - this has been running beautifully for years - the number of users nor the salesforce org nor the devices used nor the connectivity has changed significantly recently
20 | 
21 | Problem
22 | ------
23 | - It seems that the new exception takes into account the time since the start of a HTTP request for determining the execution time of a synchronous request, rather than the actual start of processing (ie. when the body has been fully received)
24 | - Potentially sending the body of the HTTP request to the SFDC platform can take more than a few seconds - causing these kind of jobs to count towards the concurrent apex limit (10 synchronous processes running longer than 5 seconds) - this will cause other processes to fail - in this case monitor.js will fail
25 | - The included stress.js script attempts to simulate this behaviour by instantly sending the request headers in 20 concurrent requests, but having a delay between sending the actual HTTP request body (20 seconds) and finally the HTTP request end (40 seconds) zero-byte
26 | 
27 | Notes
28 | ------
29 | - The included stress.js script uses 'Transfer-Encoding: chunked' header in order to have some control over when a request ends - the same behaviour will occur if a content-length is specified and the amount of bytes specified is not reached
30 | 


--------------------------------------------------------------------------------
/nodejs/stress_vf.js:
--------------------------------------------------------------------------------
 1 | /* This script attempts to create a large number of concurrent requests through VF remorting - we have not seen this break... */
 2 | var fs = require('fs');
 3 | var jsforce = require('jsforce');
 4 | var config = JSON.parse(fs.readFileSync('config.json').toString());
 5 | var https = require('https');
 6 | 
 7 | var writeDelay1 = 0;
 8 | var writeDelay2 = 30000;
 9 | var endDelay = 50000;
10 | 
11 | // Fill these variables based on a VF remorting request to the LongTxn controller - ie. captured request to /apexremote from the page included in the /apex folder
12 | var csrf = '';
13 | var vid = '';
14 | var cookie = '';
15 | var vf_url = 'c.eu11.visual.force.com';
16 | 
17 | 
18 | class Instance {
19 | 	constructor(csrf, vid, tid, cookie, url, vf_url) {
20 | 		this.csrf = csrf;
21 | 		this.vid = vid;
22 | 		this.cookie = cookie;
23 | 		this.url = url;
24 | 		this.vf_url = vf_url;
25 | 		this.tid = tid;
26 | 		this.payload1 = '{"action":"LongTxn","method":"remoteGet","data":null,"type":"rpc","tid":'+this.tid+',"ctx":{"csrf":"'+this.csrf+'"';
27 | 		this.payload2 = ',"vid":"'+this.vid+'","ns":"","ver":38}}';
28 | 		this.start();
29 | 		setTimeout(this.write.bind(this, this.payload1), writeDelay1);
30 | 		setTimeout(this.write.bind(this, this.payload2), writeDelay2);
31 | 		setTimeout(this.end.bind(this), endDelay);
32 | 	}
33 | 	start() {
34 | 		var options = {
35 | 			port: 443,
36 | 			hostname: this.vf_url,
37 | 			path: '/apexremote',
38 | 			method: 'POST',
39 | 			headers: {
40 | 				'Content-Type': 'application/json',
41 | 				'Transfer-Encoding': 'chunked',
42 | 				'Host': this.vf_url,
43 | 				'Origin': 'https://' + this.vf_url,
44 | 				'Referer': 'https://' + this.vf_url + '/apex/Test',
45 | 				'X-Requested-With': 'XMLHttpRequest',
46 | 				'X-User-Agent': 'X-User-Agent',
47 | 				'Cookie': this.cookie
48 | 			}
49 | 		};
50 | 
51 | 		this.req = https.request(options);
52 | 		this.req.setTimeout(150000, (err) => {
53 | 			console.error('Timed out');
54 | 		});
55 | 		this.req.on('response', (response) => {
56 | 			response.on('data', function (chunk) {
57 | 				console.log('response: ' + chunk);
58 | 			});
59 | 		})
60 | 		this.req.on('error', (error) => {
61 | 			console.log('error', error)
62 | 		})
63 | 		this.req.flushHeaders();
64 | 	}
65 | 	write(data) {
66 | 		console.log('writing data', data);
67 | 		this.req.write(data, 'UTF-8', (err, result) => {
68 | 			if (err) {
69 | 				return console.error(err);
70 | 			}
71 | 		})
72 | 	}
73 | 	end() {
74 | 		console.log('ending request');
75 | 		this.req.end((err, result) => {
76 | 			if (err) {
77 | 				return console.error(err);
78 | 			}
79 | 		})
80 | 	}
81 | }
82 | 
83 | var instances = [];
84 | 
85 | var conn = new jsforce.Connection({
86 | 	loginUrl : config.url
87 | });
88 | 
89 | for (var i = 0; i < config.numberOfInstances; i++) {
90 | 	instances.push(new Instance(csrf, vid, i + 2, cookie, conn.instanceUrl, vf_url));
91 | }
92 | 


--------------------------------------------------------------------------------