82 |
--------------------------------------------------------------------------------
/examples/qcs_deployment/jwt_timing_tests.js:
--------------------------------------------------------------------------------
1 | // Bring in config file with values for the qcs const below, template is in the package
2 | const config = require('./qct_config.json');
3 |
4 | // This example uses just https, you may want to use a package for requests
5 | const https = require('https');
6 |
7 | // Required for jwt generation
8 | const fs = require('fs');
9 | const uid = require('uid-safe');
10 | const jwt = require('jsonwebtoken');
11 |
12 | // Configure for your Qlik Cloud deployment
13 | const qcs = {
14 | targetTenant: '5ie54a4r13glxs5', // the tenant that you wish to configure, e.g. name for name.eu.qlikcloud.com
15 | region: config.region, // Qlik Cloud region URL e.g. name.eu.qlikcloud.com becomes eu.qlikcloud.com
16 | regionClientId: config.regionClientId, // our oauth client id, generated in MyQlik, e.g. '12345678912345678'
17 | regionClientSecret: config.regionClientSecret // our oauth client secret, generated in MyQlik, e.g. 'eac0dswfec23ewfweweg2g2vsebw77e1cce'
18 | }
19 |
20 | // To help with the JWT decoding step
21 | function parseJwt (token) {
22 | return JSON.parse(Buffer.from(token.split('.')[1], 'base64').toString());
23 | }
24 |
25 | // Config for IdP
26 | const idpSettings = {
27 | description: "Auth for my app or portal",
28 | }
29 |
30 | // JWT: Configure JWT IdP payload with the groups we want to seed
31 | const jwtPayload = {
32 | jti: uid.sync(32), // 32 bytes random string
33 | sub: 'mydomain\\yournamehere',
34 | subType: 'user',
35 | name: 'Your Name Here',
36 | email: 'yournamehere@example.com',
37 | email_verified: true,
38 | groups: ['Admin', 'Finance', 'Marketing', 'Sales'],
39 | };
40 |
41 | // JWT: Identify keys
42 | const jwtPrivateKey = fs.readFileSync('./certs/nameap_privatekey.pem');
43 | const jwtPublicKey = fs.readFileSync('./certs/nameap_publickey.cer');
44 |
45 | // JWT: Provide signing options
46 | // kid and issuer have to match with the IDP config and the
47 | // audience has to be qlik.api/jwt-login-session
48 | const jwtSigningOptions = {
49 | keyid: 'myapporportal',
50 | algorithm: 'RS256',
51 | issuer: 'euqlikcloud',
52 | expiresIn: '5m',
53 | notBefore: '47s',
54 | audience: 'qlik.api/login/jwt-session'
55 | };
56 |
57 | // Prepare our oauth cred package for future auth requests
58 | const oauthClientData = JSON.stringify({
59 | client_id: qcs.regionClientId,
60 | client_secret: qcs.regionClientSecret,
61 | grant_type: "client_credentials"
62 | })
63 |
64 | // Prepare target tenant URL
65 | const targetTenantUrl = qcs.targetTenant + '.' + qcs.region;
66 |
67 | // Function to handle our requests
68 | function httpsRequest(params, postBody) {
69 | return new Promise(function (resolve, reject) {
70 | var req = https.request(params, function (res) {
71 |
72 | var body = [];
73 | res.on('data', function (chunk) {
74 | body.push(chunk);
75 | });
76 | res.on('end', function () {
77 | try {
78 | body = JSON.parse(Buffer.concat(body).toString());
79 | } catch (e) {
80 | // Here to catch empty responses and play them back nicely
81 | body = 'Status ' + res.statusCode + 'r/n/' + body;
82 | }
83 | resolve(body);
84 | });
85 | });
86 | req.on('error', function (err) {
87 | reject(err);
88 | });
89 | if (postBody) {
90 | req.write(postBody);
91 | }
92 | req.end();
93 | });
94 | }
95 |
96 | async function testJwtTiming() {
97 |
98 | // ***************************
99 | // 1 - Get access token for target tenant, for use will all future requests
100 | var data = await httpsRequest({
101 | hostname: targetTenantUrl,
102 | port: 443,
103 | path: '/oauth/token',
104 | method: 'POST',
105 | headers: {
106 | 'Content-Type': 'application/json'
107 | }
108 | }, oauthClientData);
109 | // For the demo, log out the response
110 | console.log(Math.floor(Date.now()/1000), 'Output from step 1: ' + JSON.stringify(data));
111 | // Save access token for next request
112 | let targetTenantToken = data.access_token;
113 |
114 | // ***************************
115 | // 2 - Get tenant id
116 | var data = await httpsRequest({
117 | hostname: targetTenantUrl,
118 | port: 443,
119 | path: '/api/v1/tenants',
120 | method: 'GET',
121 | headers: {
122 | 'Content-Type': 'application/json',
123 | 'Authorization': 'Bearer ' + targetTenantToken
124 | }
125 | });
126 | // For the demo, log out the response
127 | console.log(Math.floor(Date.now()/1000), 'Output from step 2: ' + JSON.stringify(data));
128 |
129 | // Keep a copy of the tenant id, we'll need it shortly
130 | const tenantId = data.data[0].id;
131 |
132 | const testClockToleranceSec = 47;
133 |
134 | // ***************************
135 | // 3 - Configure JWT IdP on the tenant
136 | // Prepare IdP configuration for the JWT IdP post with the tenant ID and config
137 | const idpConfiguration = JSON.stringify({
138 | tenantIds: [
139 | tenantId
140 | ],
141 | provider: "external",
142 | protocol: "jwtAuth",
143 | interactive: false,
144 | active: true,
145 | description: idpSettings.description,
146 | clockToleranceSec: testClockToleranceSec,
147 | options: {
148 | jwtLoginEnabled: true,
149 | issuer: jwtSigningOptions.issuer,
150 | staticKeys: [
151 | {
152 | kid: jwtSigningOptions.keyid,
153 | pem: jwtPublicKey.toString()
154 | }
155 | ]
156 | }
157 | });
158 |
159 | // For the demo, log out the IdP config we're going to send
160 | console.log(Math.floor(Date.now()/1000), 'Output from step 3a: ' + idpConfiguration);
161 |
162 | // Send request to create JWT IdP
163 | var data = await httpsRequest({
164 | hostname: targetTenantUrl,
165 | port: 443,
166 | path: '/api/v1/identity-providers',
167 | method: 'POST',
168 | headers: {
169 | 'Content-Type': 'application/json',
170 | 'Authorization': 'Bearer ' + targetTenantToken
171 | }
172 | }, idpConfiguration);
173 |
174 | // Get the JWT IdP id
175 | const idpId = data.id;
176 |
177 | // Get the created ts
178 | const idpCreated = Math.floor(new Date(data.created) / 1000);
179 |
180 | // For the demo, log out the response
181 | console.log(Math.floor(Date.now()/1000), 'Output from step 3b, IdP created at:', idpCreated, 'with config:', JSON.stringify(data));
182 |
183 | // ***************************
184 | // 4 - Send our JWT request and seed groups
185 | // Built our JWT with the groups we want to seed
186 | const seedToken = jwt.sign(jwtPayload, jwtPrivateKey, jwtSigningOptions);
187 |
188 | const testTimeJwtNbf = parseJwt(seedToken).nbf;
189 |
190 | // For the demo, log out the token we're going to use to seed groups
191 | console.log(Math.floor(Date.now()/1000), 'Output from step 4a:', seedToken, parseJwt(seedToken));
192 |
193 | // Send the request
194 | var data = await httpsRequest({
195 | hostname: targetTenantUrl,
196 | port: 443,
197 | path: '/login/jwt-session',
198 | method: 'POST',
199 | headers: {
200 | 'Content-Type': 'application/json',
201 | 'Authorization': 'Bearer ' + seedToken
202 | }
203 | });
204 |
205 | // For the demo, log out the response
206 | console.log(Math.floor(Date.now()/1000), 'Output from step 4b: ' + JSON.stringify(data));
207 |
208 | // 5 - Delete the IdP
209 | // Send the request
210 | var data = await httpsRequest({
211 | hostname: targetTenantUrl,
212 | port: 443,
213 | path: '/api/v1/identity-providers/' + idpId,
214 | method: 'DELETE',
215 | headers: {
216 | 'Content-Type': 'application/json',
217 | 'Authorization': 'Bearer ' + targetTenantToken
218 | }
219 | });
220 |
221 | // For the demo, log out the response
222 | console.log(Math.floor(Date.now()/1000), 'Output from step 5: ' + JSON.stringify(data));
223 |
224 | // For the test
225 | console.log(Math.floor(Date.now()/1000),'Test resulted in nbf of',testTimeJwtNbf,'and server time of',idpCreated,'which was a delta of',testTimeJwtNbf-idpCreated,'seconds.');
226 |
227 | if (testTimeJwtNbf > idpCreated + testClockToleranceSec) {
228 | console.log(Math.floor(Date.now()/1000),'Error: System time +',testClockToleranceSec,'was lesser than nbf');
229 | }
230 | }
231 |
232 | // Go test
233 | testJwtTiming();
--------------------------------------------------------------------------------
/examples/qcs_deployment/package-lock.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "qcs_deployment",
3 | "version": "1.0.0",
4 | "lockfileVersion": 2,
5 | "requires": true,
6 | "packages": {
7 | "": {
8 | "name": "qcs_deployment",
9 | "version": "1.0.0",
10 | "license": "ISC",
11 | "dependencies": {
12 | "fetch": "^1.1.0",
13 | "jsonwebtoken": "^9.0.0",
14 | "uid-safe": "^2.1.5"
15 | }
16 | },
17 | "node_modules/biskviit": {
18 | "version": "1.0.1",
19 | "resolved": "https://registry.npmjs.org/biskviit/-/biskviit-1.0.1.tgz",
20 | "integrity": "sha512-VGCXdHbdbpEkFgtjkeoBN8vRlbj1ZRX2/mxhE8asCCRalUx2nBzOomLJv8Aw/nRt5+ccDb+tPKidg4XxcfGW4w==",
21 | "dependencies": {
22 | "psl": "^1.1.7"
23 | },
24 | "engines": {
25 | "node": ">=1.0.0"
26 | }
27 | },
28 | "node_modules/buffer-equal-constant-time": {
29 | "version": "1.0.1",
30 | "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
31 | "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="
32 | },
33 | "node_modules/ecdsa-sig-formatter": {
34 | "version": "1.0.11",
35 | "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
36 | "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
37 | "dependencies": {
38 | "safe-buffer": "^5.0.1"
39 | }
40 | },
41 | "node_modules/encoding": {
42 | "version": "0.1.12",
43 | "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
44 | "integrity": "sha512-bl1LAgiQc4ZWr++pNYUdRe/alecaHFeHxIJ/pNciqGdKXghaTCOwKkbKp6ye7pKZGu/GcaSXFk8PBVhgs+dJdA==",
45 | "dependencies": {
46 | "iconv-lite": "~0.4.13"
47 | }
48 | },
49 | "node_modules/fetch": {
50 | "version": "1.1.0",
51 | "resolved": "https://registry.npmjs.org/fetch/-/fetch-1.1.0.tgz",
52 | "integrity": "sha512-5O8TwrGzoNblBG/jtK4NFuZwNCkZX6s5GfRNOaGtm+QGJEuNakSC/i2RW0R93KX6E0jVjNXm6O3CRN4Ql3K+yA==",
53 | "dependencies": {
54 | "biskviit": "1.0.1",
55 | "encoding": "0.1.12"
56 | }
57 | },
58 | "node_modules/iconv-lite": {
59 | "version": "0.4.24",
60 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
61 | "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
62 | "dependencies": {
63 | "safer-buffer": ">= 2.1.2 < 3"
64 | },
65 | "engines": {
66 | "node": ">=0.10.0"
67 | }
68 | },
69 | "node_modules/jsonwebtoken": {
70 | "version": "9.0.0",
71 | "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
72 | "integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==",
73 | "dependencies": {
74 | "jws": "^3.2.2",
75 | "lodash": "^4.17.21",
76 | "ms": "^2.1.1",
77 | "semver": "^7.3.8"
78 | },
79 | "engines": {
80 | "node": ">=12",
81 | "npm": ">=6"
82 | }
83 | },
84 | "node_modules/jwa": {
85 | "version": "1.4.1",
86 | "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
87 | "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==",
88 | "dependencies": {
89 | "buffer-equal-constant-time": "1.0.1",
90 | "ecdsa-sig-formatter": "1.0.11",
91 | "safe-buffer": "^5.0.1"
92 | }
93 | },
94 | "node_modules/jws": {
95 | "version": "3.2.2",
96 | "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
97 | "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
98 | "dependencies": {
99 | "jwa": "^1.4.1",
100 | "safe-buffer": "^5.0.1"
101 | }
102 | },
103 | "node_modules/lodash": {
104 | "version": "4.17.21",
105 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
106 | "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
107 | },
108 | "node_modules/lru-cache": {
109 | "version": "6.0.0",
110 | "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
111 | "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
112 | "dependencies": {
113 | "yallist": "^4.0.0"
114 | },
115 | "engines": {
116 | "node": ">=10"
117 | }
118 | },
119 | "node_modules/ms": {
120 | "version": "2.1.3",
121 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
122 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
123 | },
124 | "node_modules/psl": {
125 | "version": "1.8.0",
126 | "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
127 | "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
128 | },
129 | "node_modules/random-bytes": {
130 | "version": "1.0.0",
131 | "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz",
132 | "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==",
133 | "engines": {
134 | "node": ">= 0.8"
135 | }
136 | },
137 | "node_modules/safe-buffer": {
138 | "version": "5.2.1",
139 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
140 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
141 | "funding": [
142 | {
143 | "type": "github",
144 | "url": "https://github.com/sponsors/feross"
145 | },
146 | {
147 | "type": "patreon",
148 | "url": "https://www.patreon.com/feross"
149 | },
150 | {
151 | "type": "consulting",
152 | "url": "https://feross.org/support"
153 | }
154 | ]
155 | },
156 | "node_modules/safer-buffer": {
157 | "version": "2.1.2",
158 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
159 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
160 | },
161 | "node_modules/semver": {
162 | "version": "7.5.4",
163 | "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
164 | "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
165 | "dependencies": {
166 | "lru-cache": "^6.0.0"
167 | },
168 | "bin": {
169 | "semver": "bin/semver.js"
170 | },
171 | "engines": {
172 | "node": ">=10"
173 | }
174 | },
175 | "node_modules/uid-safe": {
176 | "version": "2.1.5",
177 | "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz",
178 | "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==",
179 | "dependencies": {
180 | "random-bytes": "~1.0.0"
181 | },
182 | "engines": {
183 | "node": ">= 0.8"
184 | }
185 | },
186 | "node_modules/yallist": {
187 | "version": "4.0.0",
188 | "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
189 | "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
190 | }
191 | },
192 | "dependencies": {
193 | "biskviit": {
194 | "version": "1.0.1",
195 | "resolved": "https://registry.npmjs.org/biskviit/-/biskviit-1.0.1.tgz",
196 | "integrity": "sha512-VGCXdHbdbpEkFgtjkeoBN8vRlbj1ZRX2/mxhE8asCCRalUx2nBzOomLJv8Aw/nRt5+ccDb+tPKidg4XxcfGW4w==",
197 | "requires": {
198 | "psl": "^1.1.7"
199 | }
200 | },
201 | "buffer-equal-constant-time": {
202 | "version": "1.0.1",
203 | "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
204 | "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="
205 | },
206 | "ecdsa-sig-formatter": {
207 | "version": "1.0.11",
208 | "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
209 | "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
210 | "requires": {
211 | "safe-buffer": "^5.0.1"
212 | }
213 | },
214 | "encoding": {
215 | "version": "0.1.12",
216 | "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
217 | "integrity": "sha512-bl1LAgiQc4ZWr++pNYUdRe/alecaHFeHxIJ/pNciqGdKXghaTCOwKkbKp6ye7pKZGu/GcaSXFk8PBVhgs+dJdA==",
218 | "requires": {
219 | "iconv-lite": "~0.4.13"
220 | }
221 | },
222 | "fetch": {
223 | "version": "1.1.0",
224 | "resolved": "https://registry.npmjs.org/fetch/-/fetch-1.1.0.tgz",
225 | "integrity": "sha512-5O8TwrGzoNblBG/jtK4NFuZwNCkZX6s5GfRNOaGtm+QGJEuNakSC/i2RW0R93KX6E0jVjNXm6O3CRN4Ql3K+yA==",
226 | "requires": {
227 | "biskviit": "1.0.1",
228 | "encoding": "0.1.12"
229 | }
230 | },
231 | "iconv-lite": {
232 | "version": "0.4.24",
233 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
234 | "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
235 | "requires": {
236 | "safer-buffer": ">= 2.1.2 < 3"
237 | }
238 | },
239 | "jsonwebtoken": {
240 | "version": "9.0.0",
241 | "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
242 | "integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==",
243 | "requires": {
244 | "jws": "^3.2.2",
245 | "lodash": "^4.17.21",
246 | "ms": "^2.1.1",
247 | "semver": "^7.3.8"
248 | }
249 | },
250 | "jwa": {
251 | "version": "1.4.1",
252 | "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz",
253 | "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==",
254 | "requires": {
255 | "buffer-equal-constant-time": "1.0.1",
256 | "ecdsa-sig-formatter": "1.0.11",
257 | "safe-buffer": "^5.0.1"
258 | }
259 | },
260 | "jws": {
261 | "version": "3.2.2",
262 | "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz",
263 | "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==",
264 | "requires": {
265 | "jwa": "^1.4.1",
266 | "safe-buffer": "^5.0.1"
267 | }
268 | },
269 | "lodash": {
270 | "version": "4.17.21",
271 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
272 | "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
273 | },
274 | "lru-cache": {
275 | "version": "6.0.0",
276 | "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
277 | "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
278 | "requires": {
279 | "yallist": "^4.0.0"
280 | }
281 | },
282 | "ms": {
283 | "version": "2.1.3",
284 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
285 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
286 | },
287 | "psl": {
288 | "version": "1.8.0",
289 | "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
290 | "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
291 | },
292 | "random-bytes": {
293 | "version": "1.0.0",
294 | "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz",
295 | "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ=="
296 | },
297 | "safe-buffer": {
298 | "version": "5.2.1",
299 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
300 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
301 | },
302 | "safer-buffer": {
303 | "version": "2.1.2",
304 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
305 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
306 | },
307 | "semver": {
308 | "version": "7.5.4",
309 | "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
310 | "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
311 | "requires": {
312 | "lru-cache": "^6.0.0"
313 | }
314 | },
315 | "uid-safe": {
316 | "version": "2.1.5",
317 | "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz",
318 | "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==",
319 | "requires": {
320 | "random-bytes": "~1.0.0"
321 | }
322 | },
323 | "yallist": {
324 | "version": "4.0.0",
325 | "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
326 | "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
327 | }
328 | }
329 | }
330 |
--------------------------------------------------------------------------------
/examples/qcs_deployment/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "qcs_deployment",
3 | "version": "1.0.0",
4 | "description": "Create a tenant, configure the tenant, and deploy apps and content",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "echo \"Error: no test specified\" && exit 1"
8 | },
9 | "author": "",
10 | "license": "ISC",
11 | "dependencies": {
12 | "fetch": "^1.1.0",
13 | "jsonwebtoken": "^9.0.0",
14 | "uid-safe": "^2.1.5"
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/examples/qcs_deployment/qct_config_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "licenseKey": "eyJhbGciOiJ...",
3 | "sourceTenant": "name",
4 | "region": "eu.qlikcloud.com",
5 | "regionClientId": "1ef2caed0dbe0a6...",
6 | "regionClientSecret": "eac0d78212fd61e3e..."
7 | }
--------------------------------------------------------------------------------
/examples/qcs_deployment/resources/Template_Finance.qvf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/withdave/qlik/930874175e114138aa4ec560880bfa2d3ccad23d/examples/qcs_deployment/resources/Template_Finance.qvf
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/00_Main.qvs:
--------------------------------------------------------------------------------
1 | SET ThousandSep=',';
2 | SET DecimalSep='.';
3 | SET MoneyThousandSep=',';
4 | SET MoneyDecimalSep='.';
5 | SET MoneyFormat='$#,##0.00;-$#,##0.00';
6 | SET TimeFormat='hh:mm:ss TT';
7 | SET DateFormat='DD/MM/YYYY';
8 | SET TimestampFormat='DD/MM/YYYY hh:mm:ss[.fff]';
9 | SET FirstWeekDay=6;
10 | SET BrokenWeeks=1;
11 | SET ReferenceDay=0;
12 | SET FirstMonthOfYear=1;
13 | SET CollationLocale='en-US';
14 | SET CreateSearchIndexOnReload=1;
15 | SET MonthNames='Jan;Feb;Mar;Apr;May;Jun;Jul;Aug;Sep;Oct;Nov;Dec';
16 | SET LongMonthNames='January;February;March;April;May;June;July;August;September;October;November;December';
17 | SET DayNames='Mon;Tue;Wed;Thu;Fri;Sat;Sun';
18 | SET LongDayNames='Monday;Tuesday;Wednesday;Thursday;Friday;Saturday;Sunday';
19 | SET NumericalAbbreviation='3:k;6:M;9:G;12:T;15:P;18:E;21:Z;24:Y;-3:m;-6:μ;-9:n;-12:p;-15:f;-18:a;-21:z;-24:y';
20 |
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/01_Config.qvs:
--------------------------------------------------------------------------------
1 | // Configure shared variables
2 |
3 | SET vConn_QVD = 'lib://file_stackoverflow';
4 | SET vConn_QVD_Data = '$(vConn_QVD)/datascience.stackexchange.com';
5 | SET vConn_QVD_Math = '$(vConn_QVD)/math.stackexchange.com';
6 | SET vConn_QVD_Server = '$(vConn_QVD)/serverfault.com';
7 |
8 | // Set number of iterations to test for
9 | SET vTest_Iterations = 10;
10 |
11 | // Set context for test
12 | SET vTest_Mode = 'Encrypted QVD + QVF';
13 |
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/02_sTimerEnd().qvs:
--------------------------------------------------------------------------------
1 | // Create a function to collect the log timestamp and append it to the timer log table
2 |
3 | SUB sTimerEnd(vSub_TestName)
4 |
5 | // Set the time now
6 | LET vSub_Now = now();
7 |
8 | TRACE >> Event [$(vSub_TestName)] recorded with start [$(vLog_Time)] and end [$(vSub_Now)].;
9 |
10 | // Concatenate onto the logging table
11 | Concatenate(Logging)
12 | LOAD
13 | rowno() AS [Log ID],
14 | '$(vSub_Now)' AS [Log Timestamp],
15 | '$(vSub_TestName)' AS [Log Event],
16 | '$(vLog_Time)' AS [Log Event Start],
17 | '$(vSub_Now)' AS [Log Event End],
18 | ROUND(('$(vSub_Now)'-'$(vLog_Time)')*24*60*60,0.001) AS [Log Event Duration (s)],
19 | '$(vTest_Mode)' AS [Log Mode]
20 | AutoGenerate 1;
21 |
22 | // Now reset the log time variable
23 | LET vLog_Time = now();
24 |
25 | END SUB;
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/03_TimerLog & Start.qvs:
--------------------------------------------------------------------------------
1 | // Create an empty table to append log records
2 | // Log table will eventually hold:
3 | // - Log Timestamp
4 | // - Log Event
5 | // - Log Event Start
6 | // - Log Event End
7 | // - Log Event Duration
8 | // - Log ID (autogenerated)
9 | Logging:
10 | LOAD
11 | null() AS [Log ID]
12 | AutoGenerate 0;
13 |
14 | // Initiate the timer
15 | LET vLog_Time = now();
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/04_Badges.qvs:
--------------------------------------------------------------------------------
1 | // Pick which data set
2 | LET vConn_Test_Data = '$(vConn_QVD_Server)';
3 | LET vConn_Test_File = 'Badges';
4 |
5 | // Create table
6 | LET vConn_Test_Table = lower('$(vConn_Test_File)');
7 |
8 | // Loop over test
9 | FOR iter_Test = 0 to vTest_Iterations-1
10 |
11 | /////////// Raw load
12 | [$(vConn_Test_File)]:
13 | LOAD
14 | Id,
15 | "UserId",
16 | Name,
17 | "Date",
18 | "Class",
19 | TagBased
20 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).xml]
21 | (XmlSimple, table is [$(vConn_Test_Table)/row]);
22 |
23 | Call sTimerEnd('$(vConn_Test_File) raw load [$(vConn_Test_Data)])');
24 |
25 | /////////// Badge raw store & drop
26 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File).qvd] (qvd);
27 | DROP TABLE [$(vConn_Test_File)];
28 |
29 | Call sTimerEnd('$(vConn_Test_File) raw store & drop [$(vConn_Test_Data)])');
30 |
31 |
32 | /////////// QVD optimised load
33 | [$(vConn_Test_File)]:
34 | LOAD
35 | *
36 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd] (qvd);
37 |
38 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load [$(vConn_Test_Data)])');
39 |
40 | /////////// Badge second store & drop
41 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_copy.qvd] (qvd);
42 | DROP TABLE [$(vConn_Test_File)];
43 |
44 | Call sTimerEnd('$(vConn_Test_File) QVD store and drop [$(vConn_Test_Data)])');
45 |
46 | /////////// Unoptimised load and store
47 | [$(vConn_Test_File)]:
48 | LOAD
49 | Id AS [Badge ID],
50 | "UserId" AS [Badge User ID],
51 | Name AS [Badge Name],
52 | Name & ' (' & [Class] & ')' AS [Badge Name & Class],
53 | Timestamp("Date") AS [Badge Timestamp],
54 | Date(Floor("Date")) AS [Badge Date],
55 | "Class" AS [Badge Class],
56 | IF(TagBased,'Yes','No') AS [Badge Tag Based]
57 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
58 | (qvd);
59 |
60 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 1 (load) [$(vConn_Test_Data)])');
61 |
62 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_ult1.qvd] (qvd);
63 | DROP TABLE [$(vConn_Test_File)];
64 |
65 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 1 (store) [$(vConn_Test_Data)])');
66 |
67 | /////////// Unoptimised load and store #2
68 | [$(vConn_Test_File)]:
69 | LOAD
70 | *,
71 | Hash128(Id) AS IdHash
72 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
73 | (qvd)
74 | WHERE WildMatch(Name,'*a*') OR WildMatch(Id,'*1*');
75 |
76 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 2 (load) [$(vConn_Test_Data)])');
77 |
78 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_ult2.qvd] (qvd);
79 | DROP TABLE [$(vConn_Test_File)];
80 |
81 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 2 (store) [$(vConn_Test_Data)])');
82 |
83 | /////////// Optimised load and resident then store
84 | [$(vConn_Test_File)_Temp]:
85 | LOAD
86 | *
87 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
88 | (qvd);
89 |
90 | [$(vConn_Test_File)]:
91 | NoConcatenate
92 | LOAD
93 | Id AS [Badge ID],
94 | "UserId" AS [Badge User ID],
95 | Name AS [Badge Name],
96 | Name & ' (' & [Class] & ')' AS [Badge Name & Class],
97 | Timestamp("Date") AS [Badge Timestamp],
98 | Date(Floor("Date")) AS [Badge Date],
99 | "Class" AS [Badge Class],
100 | IF(TagBased,'Yes','No') AS [Badge Tag Based]
101 | RESIDENT [$(vConn_Test_File)_Temp];
102 |
103 | DROP TABLE [$(vConn_Test_File)_Temp];
104 |
105 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load & transform 1 (load) [$(vConn_Test_Data)])');
106 |
107 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_olt1.qvd] (qvd);
108 | DROP TABLE [$(vConn_Test_File)];
109 |
110 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load & transform 1 (store) [$(vConn_Test_Data)])');
111 |
112 | Next iter_Test;
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/05_Post History.qvs:
--------------------------------------------------------------------------------
1 | // Pick which data set
2 | LET vConn_Test_Data = '$(vConn_QVD_Server)';
3 | LET vConn_Test_File = 'PostHistory';
4 |
5 | // Create table
6 | LET vConn_Test_Table = lower('$(vConn_Test_File)');
7 |
8 | // Loop over test
9 | FOR iter_Test = 0 to vTest_Iterations-1
10 |
11 | /////////// Raw load
12 | [$(vConn_Test_File)]:
13 | LOAD
14 | Id,
15 | PostHistoryTypeId,
16 | PostId,
17 | RevisionGUID,
18 | CreationDate,
19 | "UserId",
20 | "Text",
21 | "Comment",
22 | UserDisplayName
23 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).xml]
24 | (XmlSimple, table is [$(vConn_Test_Table)/row]);
25 |
26 | Call sTimerEnd('$(vConn_Test_File) raw load [$(vConn_Test_Data)])');
27 |
28 | /////////// Badge raw store & drop
29 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File).qvd] (qvd);
30 | DROP TABLE [$(vConn_Test_File)];
31 |
32 | Call sTimerEnd('$(vConn_Test_File) raw store & drop [$(vConn_Test_Data)])');
33 |
34 |
35 | /////////// QVD optimised load
36 | [$(vConn_Test_File)]:
37 | LOAD
38 | *
39 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd] (qvd);
40 |
41 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load [$(vConn_Test_Data)])');
42 |
43 | /////////// Badge second store & drop
44 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_copy.qvd] (qvd);
45 | DROP TABLE [$(vConn_Test_File)];
46 |
47 | Call sTimerEnd('$(vConn_Test_File) QVD store and drop [$(vConn_Test_Data)])');
48 |
49 | /////////// Unoptimised load and store
50 | [$(vConn_Test_File)]:
51 | LOAD
52 | Id AS [Post History ID],
53 | PostHistoryTypeId AS [Post History Type ID],
54 | PostId AS [Post ID],
55 | AutoNumber(RevisionGUID) AS [Post Revision ID],
56 | RevisionGUID AS [Post Revision GUID],
57 | Timestamp(CreationDate) AS [Post Created Timestamp],
58 | Date(Floor(CreationDate)) AS [Post Created Date],
59 | "UserId" AS [Post User ID],
60 | "Text" AS [Post Text],
61 | Hash256([Text]) AS [Post Text Hash],
62 | IF(WildMatch([Text],'*app*','*application*'),'Yes','No') AS [Post Mentions App],
63 | "Comment" AS [Post Comment],
64 | UserDisplayName AS [Post User Name]
65 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
66 | (qvd);
67 |
68 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 1 (load) [$(vConn_Test_Data)])');
69 |
70 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_ult1.qvd] (qvd);
71 | DROP TABLE [$(vConn_Test_File)];
72 |
73 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 1 (store) [$(vConn_Test_Data)])');
74 |
75 | /////////// Unoptimised load and store #2
76 | [$(vConn_Test_File)]:
77 | LOAD
78 | *,
79 | Hash128(Id) AS IdHash
80 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
81 | (qvd)
82 | WHERE WildMatch([Text],'*app*') OR WildMatch(Id,'*1*');
83 |
84 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 2 (load) [$(vConn_Test_Data)])');
85 |
86 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_ult2.qvd] (qvd);
87 | DROP TABLE [$(vConn_Test_File)];
88 |
89 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 2 (store) [$(vConn_Test_Data)])');
90 |
91 | /////////// Optimised load and resident then store
92 | [$(vConn_Test_File)_Temp]:
93 | LOAD
94 | *
95 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
96 | (qvd);
97 |
98 | [$(vConn_Test_File)]:
99 | NoConcatenate
100 | LOAD
101 | Id AS [Post History ID],
102 | PostHistoryTypeId AS [Post History Type ID],
103 | PostId AS [Post ID],
104 | AutoNumber(RevisionGUID) AS [Post Revision ID],
105 | RevisionGUID AS [Post Revision GUID],
106 | Timestamp(CreationDate) AS [Post Created Timestamp],
107 | Date(Floor(CreationDate)) AS [Post Created Date],
108 | "UserId" AS [Post User ID],
109 | "Text" AS [Post Text],
110 | Hash256([Text]) AS [Post Text Hash],
111 | IF(WildMatch([Text],'*app*','*application*'),'Yes','No') AS [Post Mentions App],
112 | "Comment" AS [Post Comment],
113 | UserDisplayName AS [Post User Name]
114 | RESIDENT [$(vConn_Test_File)_Temp];
115 |
116 | DROP TABLE [$(vConn_Test_File)_Temp];
117 |
118 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load & transform 1 (load) [$(vConn_Test_Data)])');
119 |
120 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_olt1.qvd] (qvd);
121 | DROP TABLE [$(vConn_Test_File)];
122 |
123 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load & transform 1 (store) [$(vConn_Test_Data)])');
124 |
125 | Next iter_Test;
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/06_Posts.qvs:
--------------------------------------------------------------------------------
1 | // Pick which data set
2 | LET vConn_Test_Data = '$(vConn_QVD_Server)';
3 | LET vConn_Test_File = 'Posts';
4 |
5 | // Create table
6 | LET vConn_Test_Table = lower('$(vConn_Test_File)');
7 |
8 | // Loop over test
9 | FOR iter_Test = 0 to vTest_Iterations-1
10 |
11 | /////////// Raw load
12 | [$(vConn_Test_File)]:
13 | LOAD
14 | Id,
15 | PostTypeId,
16 | AcceptedAnswerId,
17 | CreationDate,
18 | Score,
19 | ViewCount,
20 | Body,
21 | OwnerUserId,
22 | LastEditorUserId,
23 | LastEditDate,
24 | LastActivityDate,
25 | Title,
26 | "Tags",
27 | AnswerCount,
28 | CommentCount,
29 | FavoriteCount,
30 | ParentId,
31 | ClosedDate,
32 | CommunityOwnedDate,
33 | OwnerDisplayName,
34 | LastEditorDisplayName
35 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).xml]
36 | (XmlSimple, table is [$(vConn_Test_Table)/row]);
37 |
38 | Call sTimerEnd('$(vConn_Test_File) raw load [$(vConn_Test_Data)])');
39 |
40 | /////////// Badge raw store & drop
41 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File).qvd] (qvd);
42 | DROP TABLE [$(vConn_Test_File)];
43 |
44 | Call sTimerEnd('$(vConn_Test_File) raw store & drop [$(vConn_Test_Data)])');
45 |
46 |
47 | /////////// QVD optimised load
48 | [$(vConn_Test_File)]:
49 | LOAD
50 | *
51 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd] (qvd);
52 |
53 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load [$(vConn_Test_Data)])');
54 |
55 | /////////// Badge second store & drop
56 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_copy.qvd] (qvd);
57 | DROP TABLE [$(vConn_Test_File)];
58 |
59 | Call sTimerEnd('$(vConn_Test_File) QVD store and drop [$(vConn_Test_Data)])');
60 |
61 | /////////// Unoptimised load and store
62 | [$(vConn_Test_File)]:
63 | LOAD
64 | Id AS [Post ID],
65 | PostTypeId AS [Post Type ID],
66 | AcceptedAnswerId AS [Post Accepted Answer ID],
67 | Timestamp(CreationDate) AS [Post Created Timestamp],
68 | Date(Floor(CreationDate)) AS [Post Created Date],
69 | Score AS [Post Score],
70 | ViewCount AS [Post View Count],
71 | Body AS [Post Body],
72 | Hash256(Body) AS [Post Body Hash],
73 | OwnerUserId AS [Post Owner User ID],
74 | LastEditorUserId AS [Post Last Editor User ID],
75 | Timestamp(LastEditDate) AS [Post Last Edit Timestamp],
76 | Date(Floor(LastEditDate)) AS [Post Last Edit Date],
77 | Timestamp(LastActivityDate) AS [Post Last Activity Timestamp],
78 | Date(Floor(LastActivityDate)) AS [Post Last Activity Date],
79 | Title AS [Post Title],
80 | "Tags" AS [Post Tags],
81 | AnswerCount AS [Post Answer Count],
82 | CommentCount AS [Post Comment Count],
83 | FavoriteCount AS [Post Favourite Count],
84 | ParentId AS [Post Parent ID],
85 | Timestamp(ClosedDate) AS [Post Closed Timestamp],
86 | Date(Floor(ClosedDate)) AS [Post Closed Date],
87 | Timestamp(CommunityOwnedDate) AS [Post Community Owned Timestamp],
88 | Date(Floor(CommunityOwnedDate)) AS [Post Community Owned Date],
89 | OwnerDisplayName AS [Post Owner Name],
90 | LastEditorDisplayName AS [Post Last Editor Name]
91 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
92 | (qvd);
93 |
94 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 1 (load) [$(vConn_Test_Data)])');
95 |
96 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_ult1.qvd] (qvd);
97 | DROP TABLE [$(vConn_Test_File)];
98 |
99 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 1 (store) [$(vConn_Test_Data)])');
100 |
101 | /////////// Unoptimised load and store #2
102 | [$(vConn_Test_File)]:
103 | LOAD
104 | *,
105 | Hash128(Id) AS IdHash
106 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
107 | (qvd)
108 | WHERE WildMatch([Body],'*app*') OR WildMatch(Id,'*1*');
109 |
110 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 2 (load) [$(vConn_Test_Data)])');
111 |
112 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_ult2.qvd] (qvd);
113 | DROP TABLE [$(vConn_Test_File)];
114 |
115 | Call sTimerEnd('$(vConn_Test_File) QVD unoptimised load & transform 2 (store) [$(vConn_Test_Data)])');
116 |
117 | /////////// Optimised load and resident then store
118 | [$(vConn_Test_File)_Temp]:
119 | LOAD
120 | *
121 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd]
122 | (qvd);
123 |
124 | [$(vConn_Test_File)]:
125 | NoConcatenate
126 | LOAD
127 | Id AS [Post ID],
128 | PostTypeId AS [Post Type ID],
129 | AcceptedAnswerId AS [Post Accepted Answer ID],
130 | Timestamp(CreationDate) AS [Post Created Timestamp],
131 | Date(Floor(CreationDate)) AS [Post Created Date],
132 | Score AS [Post Score],
133 | ViewCount AS [Post View Count],
134 | Body AS [Post Body],
135 | Hash256(Body) AS [Post Body Hash],
136 | OwnerUserId AS [Post Owner User ID],
137 | LastEditorUserId AS [Post Last Editor User ID],
138 | Timestamp(LastEditDate) AS [Post Last Edit Timestamp],
139 | Date(Floor(LastEditDate)) AS [Post Last Edit Date],
140 | Timestamp(LastActivityDate) AS [Post Last Activity Timestamp],
141 | Date(Floor(LastActivityDate)) AS [Post Last Activity Date],
142 | Title AS [Post Title],
143 | "Tags" AS [Post Tags],
144 | AnswerCount AS [Post Answer Count],
145 | CommentCount AS [Post Comment Count],
146 | FavoriteCount AS [Post Favourite Count],
147 | ParentId AS [Post Parent ID],
148 | Timestamp(ClosedDate) AS [Post Closed Timestamp],
149 | Date(Floor(ClosedDate)) AS [Post Closed Date],
150 | Timestamp(CommunityOwnedDate) AS [Post Community Owned Timestamp],
151 | Date(Floor(CommunityOwnedDate)) AS [Post Community Owned Date],
152 | OwnerDisplayName AS [Post Owner Name],
153 | LastEditorDisplayName AS [Post Last Editor Name]
154 | RESIDENT [$(vConn_Test_File)_Temp];
155 |
156 | DROP TABLE [$(vConn_Test_File)_Temp];
157 |
158 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load & transform 1 (load) [$(vConn_Test_Data)])');
159 |
160 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_olt1.qvd] (qvd);
161 | DROP TABLE [$(vConn_Test_File)];
162 |
163 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load & transform 1 (store) [$(vConn_Test_Data)])');
164 |
165 | Next iter_Test;
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/07_Users.qvs:
--------------------------------------------------------------------------------
1 | // Pick which data set
2 | LET vConn_Test_Data = '$(vConn_QVD_Server)';
3 | LET vConn_Test_File = 'Users';
4 |
5 | // Create table
6 | LET vConn_Test_Table = lower('$(vConn_Test_File)');
7 |
8 | // Loop over test
9 | FOR iter_Test = 0 to vTest_Iterations-1
10 |
11 | /////////// Raw load
12 | [$(vConn_Test_File)]:
13 | LOAD
14 | Id,
15 | Reputation,
16 | CreationDate,
17 | DisplayName,
18 | LastAccessDate,
19 | WebsiteUrl,
20 | Location,
21 | AboutMe,
22 | Views,
23 | UpVotes,
24 | DownVotes,
25 | AccountId,
26 | ProfileImageUrl
27 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).xml]
28 | (XmlSimple, table is [$(vConn_Test_Table)/row]);
29 |
30 | Call sTimerEnd('$(vConn_Test_File) raw load [$(vConn_Test_Data)])');
31 |
32 | /////////// Badge raw store & drop
33 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File).qvd] (qvd);
34 | DROP TABLE [$(vConn_Test_File)];
35 |
36 | Call sTimerEnd('$(vConn_Test_File) raw store & drop [$(vConn_Test_Data)])');
37 |
38 |
39 | /////////// QVD optimised load
40 | [$(vConn_Test_File)]:
41 | LOAD
42 | *
43 | FROM [$(vConn_Test_Data)/$(vConn_Test_File).qvd] (qvd);
44 |
45 | Call sTimerEnd('$(vConn_Test_File) QVD optimised load [$(vConn_Test_Data)])');
46 |
47 | /////////// Badge second store & drop
48 | STORE [$(vConn_Test_File)] INTO [$(vConn_Test_Data)/$(vConn_Test_File)_copy.qvd] (qvd);
49 | DROP TABLE [$(vConn_Test_File)];
50 |
51 | Call sTimerEnd('$(vConn_Test_File) QVD store and drop [$(vConn_Test_Data)])');
52 |
53 | Next iter_Test;
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/08_Store Logging.qvs:
--------------------------------------------------------------------------------
1 | // Store the logging to disk
2 | LET vLog_Timestamp = timestamp(now(),'YYYYMMDD_hhmmss');
3 | STORE Logging INTO [$(vConn_QVD)/Logging_$(vLog_Timestamp).qvd] (qvd);
4 | LET vLog_Timestamp = ;
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/09_exit script.qvs:
--------------------------------------------------------------------------------
1 | exit script;
--------------------------------------------------------------------------------
/examples/qs_encryption_qvd_performance/Encrypted QVD Performance.qvf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/withdave/qlik/930874175e114138aa4ec560880bfa2d3ccad23d/examples/qs_encryption_qvd_performance/Encrypted QVD Performance.qvf
--------------------------------------------------------------------------------
/examples/qs_flexible_pivot.qvf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/withdave/qlik/930874175e114138aa4ec560880bfa2d3ccad23d/examples/qs_flexible_pivot.qvf
--------------------------------------------------------------------------------
/examples/qs_variable_load/MyVariables.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/withdave/qlik/930874175e114138aa4ec560880bfa2d3ccad23d/examples/qs_variable_load/MyVariables.xlsx
--------------------------------------------------------------------------------
/examples/qs_variable_load/qs_variable_load.qvf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/withdave/qlik/930874175e114138aa4ec560880bfa2d3ccad23d/examples/qs_variable_load/qs_variable_load.qvf
--------------------------------------------------------------------------------
/examples/recno_rowno_speed.qvf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/withdave/qlik/930874175e114138aa4ec560880bfa2d3ccad23d/examples/recno_rowno_speed.qvf
--------------------------------------------------------------------------------
/snippets/Server Disk Space v1.qvf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/withdave/qlik/930874175e114138aa4ec560880bfa2d3ccad23d/snippets/Server Disk Space v1.qvf
--------------------------------------------------------------------------------
/snippets/bat_DriveSpace.bat:
--------------------------------------------------------------------------------
1 | @echo off
2 | :: Batch file to log out a local drive report to a .log file in the same directory
3 | :: Log files are named based on the date and time
4 |
5 | :: Specify which servers to attempt to return disk space for (delimit with spaces)
6 | :: Enter hostname, and ensure the account running this script has local or domain admin rights
7 | set SERVER_LIST=server
8 |
9 | :: Set output date time (YYYYMMDD_hhmmss)
10 | set LOG_TIMESTAMP=%date:~-4,4%%date:~-7,2%%date:~0,2%_%time:~0,2%%time:~3,2%%time:~6,2%
11 | set LOG_DATE=%date:~-4,4%%date:~-7,2%%date:~0,2%
12 | set LOG_TIME=%time:~0,2%%time:~3,2%%time:~6,2%
13 |
14 | :: Specify output prefix - this uses an absolute path, if you prefer to use relative then ensure the scheduler includes a "start in" path
15 | set LOG_LOCATION=C:\DriveReport\DriveReport_%LOG_TIMESTAMP%.log
16 |
17 | :: Create empty output file
18 | >nul copy nul %LOG_LOCATION%
19 |
20 | :: Loop over each server to return stats and append to log file
21 | echo ServerName,LogDate,LogTime,Drive,Size,FreeSpace>>%LOG_LOCATION%
22 | for %%i in (%SERVER_LIST%) do (
23 |
24 | for /f "tokens=1,2,3" %%a in ('wmic /node:"%%i" LogicalDisk Where DriveType^="3" Get DeviceID^,Size^,FreeSpace^|find ":"') do @echo %%i,%LOG_DATE%,%LOG_TIME%,%%a,%%c,%%b>>%LOG_LOCATION%
25 |
26 | )
27 |
--------------------------------------------------------------------------------
/snippets/bat_ServiceCheck.bat:
--------------------------------------------------------------------------------
1 | @Echo Off
2 | :: ServiceCheck.bat
3 | :: Accepts a service name, if it's running, exits. If it's not running, attempts to start it and creates a log file.
4 | :: Uplift to check and retry?
5 |
6 | Set ServiceName=%~1
7 | ::Set ServiceName=QlikSenseProxyService
8 |
9 | :: Get date in yyyyMMdd_HHmm format to use with file name.
10 | FOR /f "usebackq" %%i IN (`PowerShell ^(Get-Date^).ToString^('yyyy-MM-dd'^)`) DO SET LogDate=%%i
11 |
12 | SC queryex "%ServiceName%"|Find "STATE"|Find /v "RUNNING">Nul&&(
13 | echo %ServiceName% not running
14 | echo Start %ServiceName%
15 |
16 | Net start "%ServiceName%">nul||(
17 | Echo "%ServiceName%" wont start
18 | exit /b 1
19 | )
20 |
21 | echo "%ServiceName%" started
22 |
23 | :: Now log out to a file so we have some sort of history
24 | echo ### Service [%ServiceName%] not running on %LogDate% & echo %Time% Attempting to start service.>>"%~dp0ServiceCheck_%ServiceName%_%LogDate%.log"
25 | exit /b 0
26 |
27 | )||(
28 | :: All OK, let's just write to console and exit
29 | echo "%ServiceName%" working
30 | exit /b 0
31 | )
32 |
--------------------------------------------------------------------------------
/snippets/node_gen_date_range.js:
--------------------------------------------------------------------------------
1 | /*
2 | To do:
3 | - Handle start date correctly
4 | - Wrap end date to last day
5 | */
6 |
7 | function generateDateArray(timestamp) {
8 | var currentDate = new Date();
9 | var currentDate = currentDate.setDate(currentDate.getDate() - 2);
10 | const startDate = new Date(timestamp);
11 | var iterDate = new Date(timestamp);
12 | const datesArray = [];
13 |
14 | // Set the start date's time to match the provided timestamp
15 | startDate.setHours(new Date(timestamp).getHours());
16 | startDate.setMinutes(new Date(timestamp).getMinutes());
17 | startDate.setSeconds(new Date(timestamp).getSeconds());
18 | startDate.setMilliseconds(new Date(timestamp).getMilliseconds());
19 |
20 | iterDate = new Date(JSON.parse(JSON.stringify(startDate)));
21 | iterDate.setHours(24,59,59,999);
22 | datesArray.push((new Date(startDate)).toISOString() + '/' + (new Date(iterDate)).toISOString());
23 |
24 |
25 | while (startDate < currentDate) {
26 | //console.log(startDate + currentDate)
27 | startDate.setDate(startDate.getDate() + 1);
28 | //startDate.setHours(1);
29 | startDate.setHours(1,0,0,0);
30 |
31 |
32 | iterDate = new Date(JSON.parse(JSON.stringify(startDate)));
33 | iterDate.setHours(24,59,59,999);
34 |
35 | datesArray.push((new Date(startDate)).toISOString() + '/' + (new Date(iterDate)).toISOString());
36 | }
37 |
38 | // Add the current timestamp
39 | var today = new Date();
40 | var today = today.setHours(1,0,0,0);
41 | datesArray.push((new Date(today)).toISOString() + '/' + (new Date()).toISOString());
42 |
43 | return datesArray;
44 | }
45 |
46 | // Example usage
47 | const inputTimestamp = 1684262549000;
48 | const dateArray = generateDateArray(inputTimestamp);
49 |
50 | console.log(dateArray);
--------------------------------------------------------------------------------
/snippets/q_all_fields.md:
--------------------------------------------------------------------------------
1 | # Load all fields in a data model
2 |
3 | Script loads all fields in a data model into a single field. Useful for custom pivot sheets.
4 |
5 | ```
6 | // Let's keep things tidy in the UI
7 | Set HidePrefix = '%';
8 |
9 | // Load a table of field names
10 | // First iter over all tables
11 | FOR iter_tables=0 to NoOfTables() - 1
12 |
13 | // Next iter over all fields in this table
14 | FOR iter_fields=1 to NoOfFields(TableName($(iter_tables)))
15 |
16 | Island_Fields:
17 | LOAD
18 | TableName($(iter_tables)) AS %TableName,
19 | FieldName($(iter_fields),TableName($(iter_tables))) AS %FieldName
20 | AutoGenerate 1;
21 |
22 | NEXT iter_fields;
23 | NEXT iter_tables;
24 | ```
25 |
26 | ## To then reference specific fields in a pivot table
27 |
28 | This can be used on the show column property of a pivot table.
29 |
30 | ```
31 | SubStringCount('|' & Concat(distinct %FieldName, '|') & '|', '|Expression1|')
32 | ```
33 |
--------------------------------------------------------------------------------
/snippets/q_colours.md:
--------------------------------------------------------------------------------
1 | # Qlik colour list
2 |
3 | ## Green white & grey (QS 2020)
4 |
5 |
6 | | Colour | HEX | RGB |
7 | |--- |--- |--- |
8 | | Green | 009845 | 0,152,69 |
9 | | White | ffffff | 255,255,255 |
10 | | Light Grey (Alternative) | dddddd | 221,221,221 |
11 | | Dark Grey (Excluded) | a9a9a9 | 169,169,169 |
12 |
--------------------------------------------------------------------------------
/snippets/q_gitignore:
--------------------------------------------------------------------------------
1 | ## GitIgnore built from own experience and using samples from https://github.com/github/gitignore
2 | ## DC - 20190920
3 |
4 | ## Qlik exclusions
5 | # Ignore all Qlik data files
6 | *.qvd
7 |
8 |
9 | ## Windows exclusions
10 | # Windows thumbnail cache files
11 | Thumbs.db
12 | Thumbs.db:encryptable
13 | ehthumbs.db
14 | ehthumbs_vista.db
15 |
16 | # Dump file
17 | *.stackdump
18 |
19 | # Folder config file
20 | [Dd]esktop.ini
21 |
22 | # Recycle Bin used on file shares
23 | $RECYCLE.BIN/
24 |
25 | # Windows Installer files
26 | *.cab
27 | *.msi
28 | *.msix
29 | *.msm
30 | *.msp
31 |
32 | # Windows shortcuts
33 | *.lnk
34 |
35 | ## Dropbox exclusions
36 | # Dropbox settings and caches
37 | .dropbox
38 | .dropbox.attr
39 | .dropbox.cache
40 |
41 | ## MacOS exclusions
42 | # General
43 | .DS_Store
44 | .AppleDouble
45 | .LSOverride
46 |
47 | # Icon must end with two \r
48 | Icon
49 |
50 |
51 | # Thumbnails
52 | ._*
53 |
54 | # Files that might appear in the root of a volume
55 | .DocumentRevisions-V100
56 | .fseventsd
57 | .Spotlight-V100
58 | .TemporaryItems
59 | .Trashes
60 | .VolumeIcon.icns
61 | .com.apple.timemachine.donotpresent
62 |
63 | # Directories potentially created on remote AFP share
64 | .AppleDB
65 | .AppleDesktop
66 | Network Trash Folder
67 | Temporary Items
68 | .apdisk
69 |
70 | # Finally I want to exclude any packages I might make
71 | *.zip
72 |
--------------------------------------------------------------------------------
/snippets/q_groupby_performance.md:
--------------------------------------------------------------------------------
1 | # Group By Performance
2 |
3 | This script explores whether it's faster to aggregate data using a straight group by, or if there's a benefit to sorting the data first.
4 |
5 | Tests:
6 | * Order by, group by (two resident loads)
7 | * Group by (one resident load)
8 |
9 | ## Script
10 | ```
11 | ///$tab sLog
12 | Log:
13 | Load
14 | null() AS TestName
15 | AutoGenerate 0;
16 |
17 | Sub sLog(vTest,vStart,vFinish)
18 |
19 | TRACE >> Log [$(vTest)] started $(vStart) finished $(vFinish).;
20 |
21 | Concatenate(Log)
22 | LOAD
23 | '$(vTest)' AS TestName,
24 | '$(vStart)' AS TestStart,
25 | '$(vFinish)'AS TestFinish
26 | AutoGenerate 1;
27 |
28 | END SUB;
29 |
30 |
31 | ///$tab Load Sample Data
32 | // Load data from QVD
33 | Source:
34 | LOAD
35 | // TransLineID,
36 | // TransID,
37 | Number,
38 | Dimension1,
39 | // Dimension2,
40 | Dimension3,
41 | // Dimension4,
42 | Expression1
43 | // Expression2,
44 | // Expression3
45 | FROM [lib://dir_root_azurefiles/me/RandomData.qvd]
46 | (qvd);
47 |
48 | // Set number of iterations to run
49 | SET vIter = 5;
50 |
51 | ///$tab Order by, Group by
52 | For i=0 to vIter
53 |
54 | // Group by letter on a sum
55 | LET vTestStart = timestamp(now(),'hh:mm:ss');
56 |
57 | // Load order by
58 | OrderBy:
59 | LOAD
60 | Dimension1,
61 | Expression1
62 | RESIDENT Source
63 | ORDER BY
64 | Dimension1;
65 |
66 | GroupBy:
67 | LOAD
68 | Dimension1,
69 | SUM(Expression1) AS Exp1
70 | RESIDENT OrderBy
71 | GROUP BY
72 | Dimension1;
73 |
74 | DROP Table OrderBy, GroupBy;
75 |
76 | Call sLog('Dimension1 - O&G','$(vTestStart)',timestamp(now(),'hh:mm:ss'));
77 |
78 | // Group by a large dimension
79 | LET vTestStart = timestamp(now(),'hh:mm:ss');
80 |
81 | // Load order by
82 | OrderBy:
83 | LOAD
84 | Dimension1,
85 | Expression1
86 | RESIDENT Source
87 | ORDER BY
88 | Dimension1;
89 |
90 | GroupBy:
91 | LOAD
92 | Dimension1,
93 | SUM(Expression1) AS Exp1
94 | RESIDENT OrderBy
95 | GROUP BY
96 | Dimension1;
97 |
98 | DROP Table OrderBy, GroupBy;
99 |
100 | Call sLog('Dimension3 - O&G','$(vTestStart)',timestamp(now(),'hh:mm:ss'));
101 |
102 | // Group by number on a sum
103 | LET vTestStart_1 = timestamp(now(),'hh:mm:ss');
104 |
105 | // Load order by
106 | OrderBy:
107 | LOAD
108 | Number,
109 | Expression1
110 | RESIDENT Source
111 | ORDER BY
112 | Number;
113 |
114 | GroupBy:
115 | LOAD
116 | Number,
117 | SUM(Expression1) AS Exp1
118 | RESIDENT OrderBy
119 | GROUP BY
120 | Number;
121 |
122 | DROP Table OrderBy, GroupBy;
123 |
124 | Call sLog('Number - O&G','$(vTestStart)',timestamp(now(),'hh:mm:ss'));
125 |
126 |
127 | Next i;
128 |
129 |
130 | ///$tab Group by
131 | For i=0 to vIter
132 |
133 | // Group by letter on a sum
134 | LET vTestStart = timestamp(now(),'hh:mm:ss');
135 |
136 | Temp_1:
137 | LOAD
138 | Dimension1,
139 | SUM(Expression1) AS Exp1
140 | RESIDENT Source
141 | GROUP BY
142 | Dimension1;
143 |
144 | DROP Table Temp_1;
145 |
146 | Call sLog('Dimension1 - G','$(vTestStart)',timestamp(now(),'hh:mm:ss'));
147 |
148 | // Group by a large dimension
149 | LET vTestStart = timestamp(now(),'hh:mm:ss');
150 |
151 | Temp_1:
152 | LOAD
153 | Dimension3,
154 | SUM(Expression1) AS Exp1
155 | RESIDENT Source
156 | GROUP BY
157 | Dimension3;
158 |
159 | DROP Table Temp_1;
160 |
161 | Call sLog('Dimension3 - G','$(vTestStart)',timestamp(now(),'hh:mm:ss'));
162 |
163 | // Group by number on a sum
164 | LET vTestStart_1 = timestamp(now(),'hh:mm:ss');
165 |
166 | Temp_1:
167 | LOAD
168 | Number,
169 | SUM(Expression1) AS Exp1
170 | RESIDENT Source
171 | GROUP BY
172 | Number;
173 |
174 | DROP Table Temp_1;
175 |
176 | Call sLog('Number - G','$(vTestStart)',timestamp(now(),'hh:mm:ss'));
177 |
178 |
179 | Next i;
180 |
181 |
182 | ///$tab Generate Data
183 | // This is here just for reference
184 | exit script;
185 | // Generate some random data
186 | // This is roughly based off of the Qlik Ctrl+O+O default script, but with a bit more variety
187 | SET vRecordCount = 5000000;
188 |
189 | Transactions:
190 | Load
191 | IterNo() as TransLineID,
192 | RecNo() as TransID,
193 | mod(RecNo(),26)+1 as Number,
194 | chr(Floor(26*Rand())+65) as Dimension1,
195 | chr(Floor(26*Rand())+97) as Dimension2,
196 | chr(Floor(26*Rand())+pick(round(rand())+1,65,97))
197 | &chr(Floor(26*Rand())+pick(round(rand())+1,65,97))
198 | &chr(Floor(26*Rand())+pick(round(rand())+1,65,97))
199 | &chr(Floor(26*Rand())+pick(round(rand())+1,65,97)) as Dimension3,
200 | // Hash128(Rand()) as Dimension4,
201 | round(1000000*Rand(),0.01) as Expression1,
202 | Round(1000*Rand()*Rand()) as Expression2,
203 | Round(Rand()*Rand()*Rand(),0.00001) as Expression3
204 | Autogenerate $(vRecordCount);
205 |
206 | // Add comments to the dimension fields
207 | Comment Field Dimension1 With "Random upper-case letter";
208 | Comment Field Dimension2 With "Random lower-case letter";
209 | Comment Field Dimension3 With "Random four letter string";
210 | Comment Field Dimension4 With "Random string (hash128)";
211 | Comment Field Expression1 With "Random value between 0 and 1000000 (2dp)";
212 | Comment Field Expression2 With "Random value between 0 and 1000 (0dp)";
213 | Comment Field Expression3 With "Random value between 0 and 1 (5dp)";
214 |
215 | STORE Transactions INTO [lib://dir_root_qlikdatastore/me/RandomData.qvd] (qvd);
216 | DROP TABLE Transactions;
217 |
218 | ```
219 |
--------------------------------------------------------------------------------
/snippets/q_load_dynamic_date_fields.md:
--------------------------------------------------------------------------------
1 |
2 | # Load a source file with changing field names and crosstable
3 | For use where the data has columns named after dates, and the dates change each week. Likely to be a cleaner way of doing this!
4 |
5 | Data Example - Qlik Inline
6 | ```
7 | LOAD * INLINE [
8 | F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11, F12, F13, F14, F15, F16
9 | Resource Name, Project Name, Assignment Bill Rate (USD), 29/03/2020, 05/04/2020, 12/04/2020, 19/04/2020, 26/04/2020, 03/05/2020, 10/05/2020, 17/05/2020, 24/05/2020, 31/05/2020, 07/06/2020, 14/06/2020, 21/06/2020
10 | Person A, Project 1, 100.00, -, -, -, -, -, -, -, 500.00, -, -, -, -, -
11 | Person B, Project 2, 100.00, 400.00, 0.00, 0.00, 0.00, 500.00, -, -, -, -, -, -, -, -
12 | Person C, Project 3, 100.00, -, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, -, 650.00, -
13 | Person A, Project 4, 100.00, -, -, -, -, 900.00, -, -, -, -, -, -, -, -
14 | Person B, Project 5, 100.00, 350.00, -, -, -, -, -, 750.00, -, -, -, -, -, -
15 | ];
16 | ```
17 |
18 | Data Example - Tabular
19 | | Resource Name | Project Name | Assignment Bill Rate (USD) | 29/03/2020 | 05/04/2020 | 12/04/2020 | 19/04/2020 | 26/04/2020 | 03/05/2020 | 10/05/2020 | 17/05/2020 | 24/05/2020 | 31/05/2020 | 07/06/2020 | 14/06/2020 | 21/06/2020 |
20 | |---------------|--------------|----------------------------|------------|------------|------------|------------|------------|------------|------------|------------|------------|------------|------------|------------|------------|
21 | | Person A | Project 1 | 100.00 | - | - | - | - | - | - | - | 500.00 | - | - | - | - | - |
22 | | Person B | Project 2 | 100.00 | 400.00 | 0.00 | 0.00 | 0.00 | 500.00 | - | - | - | - | - | - | - | - |
23 | | Person C | Project 3 | 100.00 | - | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | 0.00 | - | 650.00 | - |
24 | | Person A | Project 4 | 100.00 | - | - | - | - | 900.00 | - | - | - | - | - | - | - | - |
25 | | Person B | Project 5 | 100.00 | 350.00 | - | - | - | - | - | 750.00 | - | - | - | - | - | - |
26 |
27 |
28 | Qlik load script
29 | ```
30 | // Load the table definition
31 | // Can be specified as A/B/C or with *
32 | Table_Definition:
33 | FIRST 1
34 | LOAD
35 | *
36 | FROM [lib://DLs/TLExport.xlsx]
37 | (ooxml, no labels, table is Sheet1);
38 |
39 | // Load and print field counts
40 | LET vDefinition_Count = NoOfFields('Table_Definition');
41 | TRACE >> Found $(vDefinition_Count) fields.;
42 |
43 | // Create an empty field definition
44 | SET vDefinition_Fields = '';
45 |
46 | // Iterate over the table to build the field definition
47 | FOR vIter = 1 TO vDefinition_Count
48 |
49 | LET vLoop_FieldValue = Peek(FieldName(vIter,'Table_Definition'),0,'Table_Definition');
50 | LET vLoop_FieldValue = IF(IsNum('$(vLoop_FieldValue)'),NUM('$(vLoop_FieldValue)','00000'),'$(vLoop_FieldValue)');
51 | LET vDefinition_Fields = '$(vDefinition_Fields)' & chr(91) & '$(vLoop_FieldValue)' & chr(93) & ',';
52 |
53 | Next vIter;
54 |
55 | // Clean the trailing ',', drop source table and trace out
56 | LET vDefinition_Fields = LEFT('$(vDefinition_Fields)',LEN('$(vDefinition_Fields)')-1);
57 | TRACE >> $(vDefinition_Fields);
58 | DROP TABLE Table_Definition;
59 |
60 |
61 | // Now load the new table with our custom definition
62 | MyData:
63 | CrossTable('Date','Revenue',3)
64 | LOAD
65 | $(vDefinition_Fields)
66 | FROM [lib://DLs/TLExport.xlsx]
67 | (ooxml, embedded labels, table is Sheet1);
68 | ```
69 |
--------------------------------------------------------------------------------
/snippets/q_random_data.md:
--------------------------------------------------------------------------------
1 | # Generate some random data
2 |
3 | ```
4 | // Generate some random data
5 | // This is roughly based off of the Qlik Ctrl+O+O default script, but with a bit more variety
6 | SET vRecordCount = 50000;
7 |
8 | Transactions:
9 | Load
10 | IterNo() as TransLineID,
11 | RecNo() as TransID,
12 | mod(RecNo(),26)+1 as Number,
13 | chr(Floor(26*Rand())+65) as Dimension1,
14 | chr(Floor(26*Rand())+97) as Dimension2,
15 | chr(Floor(26*Rand())+pick(round(rand())+1,65,97))
16 | &chr(Floor(26*Rand())+pick(round(rand())+1,65,97))
17 | &chr(Floor(26*Rand())+pick(round(rand())+1,65,97))
18 | &chr(Floor(26*Rand())+pick(round(rand())+1,65,97)) as Dimension3,
19 | Hash128(Rand()) as Dimension4,
20 | round(1000000*Rand(),0.01) as Expression1,
21 | Round(1000*Rand()*Rand()) as Expression2,
22 | Round(Rand()*Rand()*Rand(),0.00001) as Expression3
23 | Autogenerate $(vRecordCount);
24 |
25 | // Add comments to describe each field
26 | Comment Field Dimension1 With "Random upper-case letter";
27 | Comment Field Dimension2 With "Random lower-case letter";
28 | Comment Field Dimension3 With "Random four letter string";
29 | Comment Field Dimension4 With "Random string (hash128)";
30 | Comment Field Expression1 With "Random value between 0 and 1000000 (2dp)";
31 | Comment Field Expression2 With "Random value between 0 and 1000 (0dp)";
32 | Comment Field Expression3 With "Random value between 0 and 1 (5dp)";
33 | ```
34 |
--------------------------------------------------------------------------------
/snippets/q_variable_creator.md:
--------------------------------------------------------------------------------
1 | # Variable generator
2 |
3 | Simple qvs to create a number of variables for testing APIs and client.
4 |
5 | ```
6 | // Snippet to produce 1000 variables (all quite long, all with an integer at the start to make them unique)
7 |
8 | For i=0 to 1000
9 |
10 | LET [v_$(i)] = 'Var' & $(i) & ': This is a wonderfully long variable which doesnt fit into the bla bLorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industrys standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.';
11 |
12 | Next i;
13 | ```
14 |
--------------------------------------------------------------------------------
/snippets/q_version_tab.qvs:
--------------------------------------------------------------------------------
1 | /*
2 | ********************* INFORMATION *********************
3 |
4 | CREATED BY: withdave
5 | CREATED DATE: 19/03/2020
6 | PURPOSE: RAW Extractor for Teradata T3 datamarts
7 |
8 | ********************* CHANGE LOG *********************
9 | */
10 |
11 | // Update this version control table after each edit. Avoid using the ; delimiter in your change log
12 | Island_VersionControl:
13 | LOAD
14 | *
15 | INLINE [
16 | %Ver_Version; %Ver_Date; %Ver_Author; %Ver_Change
17 | 1; 19/03/2020; withdave; Initial version
18 | 2; 21/04/2020; withdave; Changed store file names and added mapping for region from contact
19 | 3; 28/04/2020; withdave; Added transformed contact file
20 | ] (delimiter is ';');
21 |
22 |
23 | // Do not edit the section below as this loads and sets the version variable in the app using the table above
24 | // Identify the maximum version
25 | Temp_Version:
26 | LOAD
27 | Max(%Ver_Version) AS Temp_Max_Version
28 | RESIDENT Island_VersionControl;
29 |
30 | // Set the version variable
31 | LET vVersion = Peek('Temp_Max_Version',0,'Temp_Version');
32 |
33 | // Drop the temporary table
34 | DROP TABLE Temp_Version;
35 |
36 |
--------------------------------------------------------------------------------
/snippets/qcs_audit_delete_data_connections.md:
--------------------------------------------------------------------------------
1 | # Auditing and deleting data connections by type
2 |
3 | It is not possible to disable specific data connection types on QSE SaaS.
4 |
5 | Instead, we could automate the platform to proactively search for and remove data connections that are not approved for use.
6 |
7 | Process:
8 | - Return data connections
9 | - Delete those which shouldn't be present
10 | - Store information on the deleted connections and owners
11 |
12 | ## Script
13 |
14 | The script grabs all data connections, creates a new object containing the type we're after, then blindly deletes the connections from the object.
15 |
16 | It also stores a list of the connections it's about to delete as a CSV into the same directory prior to deleting them.
17 |
18 | ```
19 | # This script doesn't do error handling or tracking, it simply 'does'
20 | # Set the connection type to delete
21 | $connectionToDeleteType = 'File_AmazonS3Connector';
22 |
23 | # Grab all non-datafiles connections
24 | $dataConnections = qlik raw get v1/data-connections --query noDatafiles=true | ConvertFrom-Json
25 |
26 | # Create a new object containing only those with the connection type we're after
27 | $connectionsToDelete = Foreach ($dataConnection in $dataConnections) {
28 | If ($dataConnection.datasourceID -eq $connectionToDeleteType) {
29 | New-Object PSObject $dataConnection;
30 | }
31 | }
32 |
33 | # Store this object down into a CSV before we try to run deletes
34 | $connectionsToDelete | Export-Csv -Path ".\ConnectionsToDelete_$($connectionToDeleteType)_$(get-date -f yyyyMMdd_hhmmss).csv" -NoTypeInformation
35 |
36 | # Create a new object containing only those with the connection type we're after
37 | Foreach ($connectionToDelete in $connectionsToDelete) {
38 | If ($connectionToDelete.datasourceID -eq $connectionToDeleteType) {
39 | qlik raw delete v1/data-connections/$($connectionToDelete.id)
40 | }
41 | }
42 |
43 | ```
44 |
45 | ## Script to generate dummy data connections on S3
46 |
47 | This script generates 100 dummy S3 data connections to test the audit script above. Change the connection owner and space ID prior to running.
48 |
49 | ```
50 | for ($i=1; $i -lt 101; $i++) {
51 |
52 | $ownerID = "