├── .github └── workflows │ └── publish.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── __tests__ ├── sqlSyntax.test.js └── substitutionTemplates.test.js ├── broker.js ├── index.js ├── iotMock.js ├── iotSql ├── README.md ├── applyActions.js ├── applySqlSelect.js ├── applySqlWhere.js ├── eval.js ├── parseSql.js ├── sqlFunctions.js ├── substitutionTemplates.js └── whereParser.js ├── package-lock.json ├── package.json ├── ruleHandler.js ├── shadowService.js ├── testData.js └── util.js /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Build-Publish 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | test-and-lint: 9 | runs-on: [ubuntu-latest] 10 | 11 | steps: 12 | - name: Checkout 🛎 13 | uses: actions/checkout@v2 14 | 15 | - name: Setup node env 🏗 16 | uses: actions/setup-node@v1 17 | with: 18 | node-version: '16' 19 | registry-url: https://registry.npmjs.org/ 20 | 21 | - name: Install dependencies 👨🏻‍💻 22 | uses: bahmutov/npm-install@v1 23 | 24 | - name: Publish package 25 | run: npm publish 26 | env: 27 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | 63 | # idea 64 | .idea/ 65 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 👍🎉 First off, thanks for taking the time to contribute! 🎉👍 2 | 3 | Pull requests are very welcome! Make sure your patches are well tested. Ideally create a topic branch for every separate change you make. For example: 4 | 5 | 1. Fork the repo 6 | 2. Create your feature branch (git checkout -b my-new-feature) 7 | 3. Commit your changes (git commit -am 'Added some feature') 8 | 4. Push to the branch (git push origin my-new-feature) 9 | 5. Create new Pull Request -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 mitipi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # serverless-iot-offline 2 | [![serverless](http://public.serverless.com/badges/v3.svg)](http://www.serverless.com) 3 | [![npm](https://img.shields.io/npm/v/serverless-iot-offline.svg)](https://www.npmjs.com/package/serverless-iot-offline) 4 | [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](#contributing) 5 | 6 | Serverless plugin that emulates AWS IoT service. Manages topic subscriptions, lifecycle events, thing shadow management and rule engine with limited SQL syntax support. 7 | 8 | ## Prerequisites 9 | 10 | [Redis](https://redis.io/) installed. 11 | Serverless framework 1.x 12 | 13 | ## Installation 14 | Add serverless-iot-offline to your project: 15 | `npm install --save-dev serverless-iot-offline` 16 | Then inside yur `serverless.yml` file add following entry to the plugins section: `serverless-iot-offline`. If there is no plugin section you will need to add it to the file. 17 | Example: 18 | ```yaml 19 | plugins: 20 | - serverless-iot-offline 21 | ``` 22 | 23 | or if you are using `serverless-offline` plugin: 24 | ```yaml 25 | plugins: 26 | - serverless-iot-offline 27 | - serverless-offline 28 | ``` 29 | 30 | > If you are using `serverless-offline` `v5.12.1` and below, use `serverless-iot-offline@0.1.4` for comaptibility. 31 | 32 | ## Usage and command line options 33 | 34 | Make sure `redis-server` is started. 35 | If you are using `serverless-offline` you can run: 36 | `sls offline start` 37 | Otherwise run: 38 | `sls iot start` 39 | 40 | CLI options: 41 | ```bash 42 | --port -p # Port to listen on. Default: 1883 43 | --httpPort -h # Port for WebSocket connections. Default: 1884 44 | --noStart -n # Prevent Iot broker (Mosca MQTT brorker) from being started (if you already have one) 45 | --skipCacheValidation -c # Tells the plugin to skip require cache invalidation. A script reloading tool like Nodemon might then be needed (same as serverless-offline) 46 | ``` 47 | 48 | Above options could be added through `serverless.yml` file: 49 | ```yaml 50 | custom: 51 | iot: 52 | start: 53 | port: 1880 54 | redis: 55 | host: 'localhost' 56 | port: 6379 57 | db: 12 58 | # path to initial shadows 59 | # it is used to seed redis database with preconfigured shadows 60 | seedShadows: ./shadows.json 61 | # optional seedPolicies path 62 | seedPolicies: ./policy.json 63 | ``` 64 | Example of `shadows.json` file which will seed redis with 2 shadows: 65 | ```json 66 | { 67 | "thingName1": { 68 | "state": { 69 | "reported": { 70 | "some_prop": "hello" 71 | } 72 | } 73 | }, 74 | "thingName2": { 75 | "state": { 76 | "reported": {} 77 | } 78 | } 79 | } 80 | ``` 81 | 82 | ## Contributing 83 | 84 | Local implementation of AWS IoT service has a minimum of SQL syntax support and primarily we need help with that. 85 | To get a better understanding of what SQL syntax we are supporting see [documentation](./iotSql/README.md) and [testData.js](./testData.js) file. 86 | Checkout [contributing guidelines](./CONTRIBUTING.md). 87 | 88 | ## Credits and inspiration 89 | This plugin was inspired by [Tradle](https://github.com/tradle)'s [serverless-iot-local](https://github.com/tradle/serverless-iot-local) project 90 | 91 | ## Licence 92 | 93 | MIT 94 | -------------------------------------------------------------------------------- /__tests__/sqlSyntax.test.js: -------------------------------------------------------------------------------- 1 | const {parseSelect} = require('../iotSql/parseSql') 2 | const {applySelect} = require('../iotSql/applySqlSelect') 3 | const {applyWhereClause} = require('../iotSql/applySqlWhere') 4 | const {topic, timestamp, clientid, accountid, encode} = require('../iotSql/sqlFunctions') 5 | const {sqlParseTestData} = require('../testData') 6 | 7 | const log = () => {} 8 | const fnName = 'test_function' 9 | 10 | describe('SQL parser', () => { 11 | beforeAll(() => { 12 | process.env.AWS_ACCOUNT_ID = 'test_account' 13 | }) 14 | 15 | afterAll(() => { 16 | delete process.env.AWS_ACCOUNT_ID 17 | }) 18 | 19 | test('should parse and apply SQL correctly', () => { 20 | 21 | sqlParseTestData.forEach((data) => { 22 | const {sql, expected, payload} = data 23 | const parsed = parseSelect(sql) 24 | expect(parsed).toEqual(expected.parsed) 25 | expect(applyWhereClause(payload, parsed.whereClause, log, fnName)).toBe(expected.whereEvaluatesTo) 26 | expect(applySelect({ 27 | select: parsed.select, 28 | payload: JSON.parse(payload), 29 | context: { 30 | topic: (index) => topic(index, parsed.topic), 31 | clientid: () => clientid(parsed.topic), 32 | timestamp: () => timestamp(), 33 | accountid: () => accountid(), 34 | encode: (field, encoding) => encode(payload, field, encoding) 35 | } 36 | })).toEqual(expected.event) 37 | }) 38 | }) 39 | 40 | test('should parse and apply SQL with timestamp function', () => { 41 | const sql = `SELECT state.reported.preferences as pref, timestamp() as curr_time FROM '$aws/things/+/shadow/get/accepted' WHERE (state.reported.preferences.volume > 30 OR state.desired.preferences.volume > 30) AND state.reported.activities.length > 0` 42 | const payload = `{"state": {"reported": {"activities": [{"activityId":1}], "preferences": {"volume":31}}}}` 43 | 44 | const parsed = parseSelect(sql) 45 | expect(parsed).toEqual({ 46 | select: [{alias: 'pref', field: 'state.reported.preferences'}, {alias: 'curr_time', field: 'timestamp()'}], 47 | topic: '$aws/things/+/shadow/get/accepted', 48 | whereClause: '(state.reported.preferences.volume > 30 OR state.desired.preferences.volume > 30) AND state.reported.activities.length > 0' 49 | }) 50 | 51 | expect(applyWhereClause(payload, parsed.whereClause, log, fnName)).toBeTruthy() 52 | const event = applySelect({ 53 | select: parsed.select, 54 | payload: JSON.parse(payload), 55 | context: { 56 | timestamp: () => timestamp() 57 | } 58 | }) 59 | expect(event).toHaveProperty('curr_time') 60 | expect(event).toHaveProperty('pref') 61 | }) 62 | }) 63 | -------------------------------------------------------------------------------- /__tests__/substitutionTemplates.test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { fillSubstitutionTemplates } = require('../iotSql/substitutionTemplates') 4 | 5 | describe('AWS IoT Substitution Templates', () => { 6 | 7 | describe('fillSubstitutionTemplates', () => { 8 | const topicCombinations = [ 9 | { 10 | concrete: 'my/things/MY_DEVICE/shadow/update', 11 | template: '$$aws/things/${topic(3)}/shadow/update', 12 | expected: '$$aws/things/MY_DEVICE/shadow/update' 13 | }, 14 | { 15 | concrete: 'things/MY_DEVICE_001/test/topic', 16 | template: 'my/things/${topic(2)}/test/topic', 17 | expected: 'my/things/MY_DEVICE_001/test/topic' 18 | }, 19 | { 20 | concrete: 'things/MY_DEVICE_002/test/topic', 21 | template: '$aws/events/presence/connected/${topic(2)}', 22 | expected: '$aws/events/presence/connected/MY_DEVICE_002', 23 | }, 24 | { 25 | concrete: 'MY_DEVICE_003/test/topic', 26 | template: '$$aws/things/test/${topic(1)}', 27 | expected: '$$aws/things/test/MY_DEVICE_003', 28 | }, 29 | { 30 | concrete: 'part1/part2/part3', 31 | template: '$$aws/${topic(3)}/${topic(2)}/${topic(1)}', 32 | expected: '$$aws/part3/part2/part1', 33 | } 34 | ] 35 | 36 | topicCombinations.forEach(topics => { 37 | it(`${topics.concrete.padEnd(33)} -> ${topics.template.padEnd(42)} = ${topics.expected.padEnd(44)}`, () => { 38 | const substituted = fillSubstitutionTemplates(topics.concrete, topics.template) 39 | expect(substituted).toBe(topics.expected) 40 | }) 41 | }) 42 | }) 43 | 44 | }) 45 | -------------------------------------------------------------------------------- /broker.js: -------------------------------------------------------------------------------- 1 | const mosca = require('mosca') 2 | const redis = require('redis') 3 | const ascoltatore = { 4 | type: 'redis', 5 | redis, 6 | db: 12, 7 | port: 6379, 8 | return_buffers: true, // to handle binary payloads 9 | host: 'localhost' 10 | } 11 | 12 | const moscaSettings = { 13 | // port: 1883, 14 | backend: ascoltatore, 15 | persistence: { 16 | factory: mosca.persistence.Redis 17 | } 18 | } 19 | 20 | // fired when the mqtt server is ready 21 | function setup (log, opts) { 22 | log(`IOT broker is up and running on: ws://localhost:${opts.http.port}/mqqt`) 23 | } 24 | 25 | function createAWSLifecycleEvent ({type, clientId, topics}) { 26 | // http://docs.aws.amazon.com/iot/latest/developerguide/life-cycle-events.html#subscribe-unsubscribe-events 27 | const event = { 28 | clientId, 29 | timestamp: Date.now(), 30 | eventType: type, 31 | sessionIdentifier: '00000000-0000-0000-0000-000000000000', 32 | principalIdentifier: '000000000000/ABCDEFGHIJKLMNOPQRSTU:some-user/ABCDEFGHIJKLMNOPQRSTU:some-user' 33 | } 34 | 35 | if (topics) { 36 | event.topics = topics 37 | } 38 | 39 | return event 40 | } 41 | 42 | function createBroker (opts, log) { 43 | opts = Object.assign({}, moscaSettings, opts) 44 | const server = new mosca.Server(opts) 45 | server.on('ready', () => setup(log, opts)) 46 | 47 | // fired when a message is received 48 | server.on('published', function (packet, client) { 49 | 50 | const presence = packet.topic.match(/^\$SYS\/.*\/(new|disconnect)\/clients$/) 51 | if (presence) { 52 | const clientId = packet.payload 53 | const type = presence[1] === 'new' ? 'connected' : 'disconnected' 54 | server.publish({ 55 | topic: `$aws/events/presence/${type}/${clientId}`, 56 | payload: JSON.stringify(createAWSLifecycleEvent({ 57 | type, 58 | clientId 59 | })) 60 | }) 61 | } 62 | 63 | const subscription = packet.topic.match(/^\$SYS\/.*\/new\/(subscribes|unsubscribes)$/) 64 | if (subscription) { 65 | const type = subscription[1] === 'subscribes' ? 'subscribed' : 'unsubscribed' 66 | const {clientId, topic} = JSON.parse(packet.payload) 67 | server.publish({ 68 | topic: `$aws/events/subscriptions/${type}/${clientId}`, 69 | payload: JSON.stringify(createAWSLifecycleEvent({ 70 | type, 71 | clientId, 72 | topics: [topic] 73 | })) 74 | }) 75 | } 76 | }) 77 | 78 | return server 79 | } 80 | 81 | module.exports = createBroker 82 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash') 4 | const createBroker = require('./broker') 5 | const createShadowService = require('./shadowService') 6 | const iotMock = require('./iotMock') 7 | const ruleHandler = require('./ruleHandler') 8 | const {seedShadows, seedPolicies} = require('./util') 9 | 10 | const defaultOpts = { 11 | host: 'localhost', 12 | location: '.', 13 | port: 1883, 14 | httpPort: 1884, 15 | noStart: false, 16 | skipCacheInvalidation: false 17 | } 18 | 19 | class ServerlessIotPlugin { 20 | constructor (serverless, options) { 21 | this.serverless = serverless 22 | this.log = serverless.cli.log.bind(serverless.cli) 23 | this.service = serverless.service 24 | this.options = options 25 | this.provider = 'aws' 26 | this.topics = {} 27 | 28 | this.commands = { 29 | iot: { 30 | commands: { 31 | start: { 32 | usage: 'Start local Iot broker.', 33 | lifecycleEvents: ['startHandler'], 34 | options: { 35 | host: { 36 | usage: 'host name to listen on. Default: localhost', 37 | // match serverless-offline option shortcuts 38 | shortcut: 'o' 39 | }, 40 | port: { 41 | usage: 'MQTT port to listen on. Default: 1883', 42 | shortcut: 'p' 43 | }, 44 | httpPort: { 45 | usage: 'http port for client connections over WebSockets. Default: 1884', 46 | shortcut: 'h' 47 | }, 48 | skipCacheInvalidation: { 49 | usage: 'Tells the plugin to skip require cache invalidation. A script reloading tool like Nodemon might then be needed', 50 | shortcut: 'c', 51 | }, 52 | } 53 | } 54 | } 55 | } 56 | } 57 | 58 | this.hooks = { 59 | 'iot:start:startHandler': this.startHandler.bind(this), 60 | 'before:offline:start:init': this.startHandler.bind(this), 61 | 'before:offline:start': this.startHandler.bind(this), 62 | 'before:offline:start:end': this.endHandler.bind(this), 63 | } 64 | } 65 | 66 | startHandler () { 67 | this.options = _.merge({}, defaultOpts, _.get(this.service, 'custom.iot', {}), this.options) 68 | 69 | this.mqttBroker = createBroker({ 70 | host: this.options.host, 71 | port: this.options.port, 72 | http: { 73 | host: this.options.host, 74 | port: this.options.httpPort, 75 | bundle: true 76 | } 77 | }, this.log) 78 | 79 | const {client, redisClient} = createShadowService(this.options, this.log) 80 | seedShadows(this.options.seedShadows, redisClient) 81 | seedPolicies(this.options.seedPolicies, redisClient) 82 | iotMock(client, redisClient) 83 | ruleHandler(this.options, this.service, this.serverless, this.log) 84 | } 85 | 86 | endHandler () { 87 | this.log('Stopping Iot broker') 88 | this.mqttBroker.close() 89 | } 90 | } 91 | 92 | module.exports = ServerlessIotPlugin 93 | -------------------------------------------------------------------------------- /iotMock.js: -------------------------------------------------------------------------------- 1 | const AWSMock = require('aws-sdk-mock') 2 | const path = require('path') 3 | 4 | module.exports = (client, redisClient) => { 5 | AWSMock.mock('IotData', 'publish', (params, callback) => { 6 | const {topic, payload} = params 7 | client.publish(topic, payload, callback) 8 | }) 9 | 10 | AWSMock.mock('IotData', 'getThingShadow', (params, callback) => { 11 | redisClient.get(params.thingName, (err, result) => { 12 | callback(err || null, {payload: result}) 13 | }) 14 | }) 15 | 16 | AWSMock.mock('IotData', 'updateThingShadow', (params, callback) => { 17 | client.publish(`$aws/things/${params.thingName}/shadow/update`, params.payload, callback) 18 | }) 19 | 20 | AWSMock.mock('Iot', 'createPolicy', (params, callback) => { 21 | redisClient.get(params.policyName, (err, data) => { 22 | if (err || data) { 23 | callback(data ? {code: 'ResourceAlreadyExistsException'} : err) 24 | } else { 25 | const policy = { 26 | versions: { 27 | '1': { 28 | policyDocument: params.policyDocument, 29 | targets: [], 30 | timestamp: (new Date()).getTime() 31 | } 32 | }, 33 | defaultVersion: '1' 34 | } 35 | redisClient.set(params.policyName, JSON.stringify(policy), (err) => { 36 | if (err) { 37 | callback(err) 38 | } else { 39 | callback(null, { 40 | policyDocument: params.policyDocument, 41 | policyName: params.policyName, 42 | policyVersionId: 1, 43 | policyArn: '' 44 | }) 45 | } 46 | }) 47 | } 48 | }) 49 | }) 50 | 51 | AWSMock.mock('Iot', 'listTargetsForPolicy', (params, callback) => { 52 | redisClient.get(params.policyName, (err, res) => { 53 | if (err || !res) { 54 | callback(err || {message: `Policy ${params.policyName} does not exists`}) 55 | } else { 56 | const policy = JSON.parse(res) 57 | callback(null, {targets: policy.versions[policy.defaultVersion].targets}) 58 | } 59 | }) 60 | }) 61 | 62 | AWSMock.mock('Iot', 'detachPolicy', (params, callback) => { 63 | redisClient.get(params.policyName, (err, res) => { 64 | if (err || !res) { 65 | callback(err || {message: `Policy ${params.policyName} does not exists`}) 66 | } else { 67 | const policy = JSON.parse(res) 68 | const principal = `${process.env.AWS_ACCOUNT_ID}:${params.target}` 69 | const targets = policy.versions[policy.defaultVersion].targets 70 | const index = targets.indexOf(principal) 71 | if (index !== -1) { 72 | targets.splice(index, 1) 73 | } 74 | 75 | redisClient.set(params.policyName, JSON.stringify(policy), (err) => { 76 | if (err) { 77 | callback(err) 78 | } else { 79 | callback(null) 80 | } 81 | }) 82 | } 83 | }) 84 | }) 85 | 86 | AWSMock.mock('Iot', 'attachPolicy', (params, callback) => { 87 | redisClient.get(params.policyName, (err, res) => { 88 | if (err || !res) { 89 | callback(err || {message: `Policy ${params.policyName} does not exists`}) 90 | } else { 91 | const policy = JSON.parse(res) 92 | const principal = `${process.env.AWS_ACCOUNT_ID}:${params.target}` 93 | const targets = policy.versions[policy.defaultVersion].targets 94 | if (!targets.find((target) => target === principal)) { 95 | targets.push(principal) 96 | } 97 | 98 | redisClient.set(params.policyName, JSON.stringify(policy), (err) => { 99 | if (err) { 100 | callback(err) 101 | } else { 102 | callback(null) 103 | } 104 | }) 105 | } 106 | }) 107 | }) 108 | } 109 | -------------------------------------------------------------------------------- /iotSql/README.md: -------------------------------------------------------------------------------- 1 | ## IoT SQL 2 | 3 | In AWS IoT, rules are defined using an SQL-like syntax. SQL statements are composed of three types of clauses: 4 | - SELECT - Required. Extracts information from the incoming payload and performs transformations. 5 | - FROM - Required. The MQTT topic filter from which the rule receives messages. 6 | - WHERE - Optional. Adds conditional logic that determines if a rule is evaluated and its actions are executed. 7 | 8 | `iotSql` folder consists of methods for parsing SQL statements, applying transformations to incoming payload and executing additional actions. 9 | There are methods for each step of the process. 10 | - Methods for parsing SQL statement: 11 | - `parseSql.js` - split up SQL statement to select, from and where parts 12 | - `whereParser.js` - tokenize WHERE part of SQL statement 13 | 14 | - Methods for applying transformations to incoming payload using previously parsed SQL: 15 | - `applySqlWhere.js` - build and apply where clause condition based on payload. This condition determines whether lambda function will execute or not. 16 | - `applySqlSelect.js` - builds `event` for lambda function. 17 | - `sqlFunctions.js` - local implementation of [AWS Iot SQL functions](https://docs.aws.amazon.com/iot/latest/developerguide/iot-sql-functions.html) 18 | - `eval.js` - executes select statement in context of SQL functions. 19 | 20 | - Methods for executing actions: 21 | - `applyActions.js` - execute additional action defined on Iot rule. 22 | -------------------------------------------------------------------------------- /iotSql/applyActions.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk') 2 | const _ = require('lodash') 3 | let log 4 | 5 | /** 6 | * Applies actions defined for iot rules in serverless.yml 7 | * @example 8 | * // dynamoDBv2 action - writes event payload into a table 9 | * dynamoDBv2: 10 | * roleArn: "some_role_arn or reference to an arn" 11 | * putItem: 12 | * tableName: SomeTable 13 | */ 14 | module.exports.applyActions = (actions, payload, _log) => { 15 | log = _log 16 | actions.forEach((action) => { 17 | if (action.DynamoDBv2) { 18 | handleDynamoDBV2Action(_.get(action, 'DynamoDBv2.PutItem.TableName'), payload) 19 | } 20 | }) 21 | } 22 | 23 | const handleDynamoDBV2Action = (tableName, payload) => { 24 | if (!tableName) { 25 | return log('DynamoDBv2 error: table name not defined') 26 | } 27 | 28 | const offlineOptions = { 29 | region: 'localhost', 30 | endpoint: 'http://localhost:8000' 31 | } 32 | 33 | const docClient = new AWS.DynamoDB.DocumentClient(offlineOptions) 34 | const params = { 35 | TableName: tableName, 36 | Item: payload 37 | } 38 | 39 | docClient.put(params, (err) => { 40 | if (err) { 41 | log('DynamoDBv2 action error', err) 42 | } else { 43 | log('DynamoDBv2 action successful') 44 | } 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /iotSql/applySqlSelect.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | const evalInContext = require('./eval') 3 | 4 | const brace = new Buffer('{')[0] 5 | const bracket = new Buffer('[')[0] 6 | const doubleQuote = new Buffer('"')[0] 7 | 8 | // to avoid stopping here when Stop on Caught Exceptions is on 9 | const maybeParseJSON = val => { 10 | switch (val[0]) { 11 | case brace: 12 | case bracket: 13 | case doubleQuote: 14 | try { 15 | return JSON.parse(val) 16 | } catch (err) { 17 | } 18 | } 19 | 20 | return val 21 | } 22 | 23 | const applySelect = ({select, payload, context}) => { 24 | let event = {} 25 | const json = maybeParseJSON(payload) 26 | 27 | // iterate over select parsed array 28 | // ex. [{alias: 'serialNumber', field: 'topic(2)'}, {field: 'state.reported.preferences.*'}] 29 | for (let part of select) { 30 | let {alias, field} = part 31 | 32 | if (field === '*') { 33 | // if select part has alias, add that alias as property and assign payload as value 34 | if (alias) { 35 | event[alias] = json 36 | } else { 37 | // else add whole payload to event 38 | event = _.merge(event, json) 39 | } 40 | // check if field is sqlFunction 41 | } else if (Object.keys(context).some((sqlFunc) => (new RegExp(`${sqlFunc}\\((.*)\\)`).test(field)))) { 42 | // execute sqlFunction 43 | event[alias || field.replace(/\(()\)/, '')] = evalInContext(field, context) 44 | } else { 45 | // event is some property on shadow 46 | let propPath = field.split('.') 47 | let prop = propPath[propPath.length - 1] 48 | if (prop === '*') { 49 | propPath = propPath.slice(0, -1) 50 | if (alias) { 51 | event[alias] = _.get(json, propPath.join('.')) 52 | } else { 53 | event = _.merge(event, _.get(json, propPath.join('.'))) 54 | } 55 | } else { 56 | event[alias || prop] = _.get(json, propPath.join('.')) 57 | } 58 | } 59 | } 60 | 61 | return event 62 | } 63 | 64 | module.exports = {applySelect} 65 | -------------------------------------------------------------------------------- /iotSql/applySqlWhere.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | const WhereParser = require('./whereParser') 3 | 4 | const conjunctionMap = { 5 | 'OR': '||', 6 | 'AND': '&&', 7 | 'NOT': '!' 8 | } 9 | 10 | const operatorMap = { 11 | '<>': '!==', 12 | '=': '===' 13 | } 14 | 15 | const valueMap = { 16 | 'TRUE': 'true', 17 | 'FALSE': 'false' 18 | } 19 | 20 | const applyWhereClause = (message, clause, log, fnName) => { 21 | if (!clause) { 22 | return true 23 | } 24 | 25 | const parser = new WhereParser() 26 | const whereClauseAst = parser.parse(clause) 27 | let payload 28 | let condition = '' 29 | try { 30 | payload = JSON.parse(message) 31 | } catch (e) { 32 | log(`Topic payload is not JSON, skipping SQL WHERE clause for function: ${fnName}. Error: ${e}`) 33 | return false 34 | } 35 | 36 | whereClauseAst.forEach((block) => { 37 | if (Array.isArray(block)) { 38 | condition += '(' 39 | block.forEach((item) => { 40 | condition += buildCondition(item) 41 | }) 42 | condition += ')' 43 | } else { 44 | condition += `${buildCondition(block)}` 45 | } 46 | }) 47 | 48 | try { 49 | const result = eval(condition) 50 | return result 51 | } catch (e) { 52 | log(`Error executing WHERE clause for function: ${fnName}. Error: ${e}`) 53 | return false 54 | } 55 | } 56 | 57 | const buildCondition = (item) => { 58 | if (item.type === 'conjunction') { 59 | return ` ${conjunctionMap[item.value]} ` 60 | } else { 61 | return `(_.get(payload, '${item.key}', {}) ${operatorMap[item.operator] || item.operator} ${valueMap[item.value] || item.value})` 62 | } 63 | } 64 | 65 | module.exports = {applyWhereClause} 66 | -------------------------------------------------------------------------------- /iotSql/eval.js: -------------------------------------------------------------------------------- 1 | const evalInContext = (expr, context) => { 2 | let [func, fields] = expr.match(/(\w+)\((.*)\)/).slice(1, 3); 3 | fields = fields 4 | ? fields.split(",").map((f) => f.trim().replace(/['"]+/g, "")) 5 | : []; 6 | 7 | try { 8 | return context[func](...fields); 9 | } catch (err) { 10 | debugger; 11 | console.log(`failed to evaluate: ${expr}`); 12 | throw err; 13 | } 14 | }; 15 | module.exports = evalInContext; 16 | -------------------------------------------------------------------------------- /iotSql/parseSql.js: -------------------------------------------------------------------------------- 1 | const SQL_REGEX = /^SELECT (.*) FROM '([^']+)'/ 2 | const SELECT_PART_REGEX = /^(.*?)(?: as (.*))?$/i 3 | const FIELDS_REGEX = /((\w+[\n\r\s]*\([^)]*\))|([^\n\r\s(,]+))([\n\r\s]+as[\n\r\s]+\w*)?/g 4 | const WHERE_REGEX = /WHERE (.*)/ 5 | 6 | const parseSelect = sql => { 7 | const [select, topic] = sql.match(SQL_REGEX).slice(1) 8 | const [whereClause] = (sql.match(WHERE_REGEX) || []).slice(1) 9 | 10 | return { 11 | select: select.match(FIELDS_REGEX).map(parseSelectPart), 12 | topic, 13 | whereClause 14 | } 15 | } 16 | 17 | const parseSelectPart = part => { 18 | const [field, alias] = part.match(SELECT_PART_REGEX).slice(1) 19 | return { 20 | field, 21 | alias 22 | } 23 | } 24 | 25 | module.exports = {parseSelect} 26 | -------------------------------------------------------------------------------- /iotSql/sqlFunctions.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | 3 | module.exports = { 4 | topic: (index, topicUrl) => (typeof index !== 'undefined') ? topicUrl.split('/')[(index - 1)] : topicUrl, 5 | clientid: (topicUrl) => { 6 | if (/^\$aws\/events/.test(topicUrl)) { 7 | return topicUrl.slice(topicUrl.lastIndexOf('/') + 1) 8 | } else { 9 | return '' 10 | } 11 | }, 12 | timestamp: () => (new Date()).getTime(), 13 | accountid: () => process.env.AWS_ACCOUNT_ID, 14 | encode: (message, field, encoding) => { 15 | if (encoding !== "base64") { 16 | throw new Error( 17 | "AWS Iot SQL encode() function only supports base64 as an encoding" 18 | ); 19 | } 20 | if (field === "*") { 21 | return Buffer.from(message).toString("base64"); 22 | } 23 | 24 | let payload; 25 | try { 26 | payload = JSON.parse(message); 27 | } catch (e) { 28 | console.log(e); 29 | } 30 | 31 | const value = _.get(payload, field); 32 | if (!value) { 33 | throw new Error( 34 | `Failed to evaluate encode(${field}, 'base64'): Cannot find ${field} in payload` 35 | ); 36 | } 37 | return Buffer.from(value.toString()).toString("base64"); 38 | }, 39 | } 40 | -------------------------------------------------------------------------------- /iotSql/substitutionTemplates.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | function fillSubstitutionTemplates (concreteTopic, templatedTopic) { 4 | const substitutionTemplateRegExp = /(?:\$\{topic\()(\d{1,2})(?:\)\})/ 5 | while (substitutionTemplateRegExp.test(templatedTopic)) { 6 | const substitutionIndex = templatedTopic.match(substitutionTemplateRegExp)[1] - 1 7 | const split = templatedTopic.split('/') 8 | const placeholderIndex = split.findIndex(topicPiece => substitutionTemplateRegExp.test(topicPiece)) 9 | 10 | const substitutionVariable = concreteTopic.split('/')[substitutionIndex] 11 | split[placeholderIndex] = substitutionVariable 12 | templatedTopic = split.join('/') 13 | } 14 | return templatedTopic 15 | } 16 | 17 | module.exports = { 18 | fillSubstitutionTemplates 19 | } 20 | -------------------------------------------------------------------------------- /iotSql/whereParser.js: -------------------------------------------------------------------------------- 1 | const array_contains = function (haystack, needle) { 2 | return haystack.indexOf(needle) !== -1 3 | } 4 | let WhereParser = function () { 5 | } 6 | WhereParser.prototype = { 7 | blockOpen: '(', 8 | blockClose: ')', 9 | escapeOpen: '\'', 10 | escapeClose: '\'', 11 | sentinels: ['and', 'or', 'not'], 12 | operators: ['=', '<', '>', '+', '-', '*', '/', '%'], 13 | textEscape: ['\''], 14 | parse: function (query) { 15 | return this.parse_where(query) 16 | }, 17 | parse_where: function (clause) { 18 | let blocks = this.parse_blocks(clause) 19 | let phrases = this.parse_compound_phrases(blocks, []) 20 | let object = this 21 | return phrases.map(function mapFunction (value) { 22 | if (Array.isArray(value)) { 23 | return value.map(mapFunction) 24 | } else { 25 | if (array_contains(object.sentinels, value.toLowerCase())) { 26 | return { 27 | type: 'conjunction', 28 | value: value 29 | } 30 | } else { 31 | return object.parse_discriminant(value) 32 | } 33 | } 34 | }) 35 | }, 36 | parse_discriminant: function (text) { 37 | let key = '' 38 | let operator = '' 39 | let value = '' 40 | let ch 41 | 42 | for (let lcv = 0; lcv < text.length; lcv++) { 43 | ch = text[lcv] 44 | 45 | if (array_contains(this.operators, ch)) { 46 | operator += ch 47 | continue 48 | } 49 | if (operator !== '') { 50 | value += ch 51 | } else { 52 | key += ch 53 | } 54 | } 55 | return { 56 | type: 'expression', 57 | key: key, 58 | operator: operator, 59 | value: value 60 | } 61 | }, 62 | parse_blocks: function (parseableText) { 63 | let ch 64 | let env = [] 65 | let stack = [] 66 | let textMode = false 67 | let text = '' 68 | let root = env 69 | for (let lcv = 0; lcv < parseableText.length; lcv++) { 70 | ch = parseableText[lcv] 71 | if (textMode) { 72 | text += ch 73 | if (ch === this.escapeClose) textMode = false 74 | continue 75 | } 76 | if (ch === this.escapeOpen) { 77 | text += ch 78 | textMode = true 79 | continue 80 | } 81 | if (ch === this.blockOpen) { 82 | if (text.trim() !== '') env.push(text) 83 | let newEnvironment = [] 84 | env.push(newEnvironment) 85 | stack.push(this.env || env) 86 | env = newEnvironment 87 | text = '' 88 | continue 89 | } 90 | if (ch === this.blockClose) { 91 | if (text.trim() !== '') env.push(text) 92 | env = stack.pop() 93 | text = '' 94 | continue 95 | } 96 | text += ch 97 | } 98 | if (text.trim() !== '') env.push(text) 99 | return root 100 | }, 101 | parse_compound_phrases: function (data, result) { 102 | let ob = this 103 | data.forEach(function (item) { 104 | let theType = Array.isArray(item) ? 'array' : typeof item 105 | if (theType === 'array') { 106 | let results = ob.parse_compound_phrases(item, []) 107 | result.push(results) 108 | } else if (theType === 'string') { 109 | result = result.concat(ob.parse_compound_phrase(item)).filter(function (item) { 110 | return item !== '' 111 | }) 112 | } 113 | }) 114 | return result 115 | }, 116 | parse_compound_phrase: function (clause) { 117 | let inText = false 118 | let escape = '' 119 | let current = '' 120 | let results = [''] 121 | let ch 122 | for (let lcv = 0; lcv < clause.length; lcv++) { 123 | ch = clause[lcv] 124 | if (inText) { 125 | results[results.length - 1] += current + ch 126 | current = '' 127 | if (ch === escape) inText = false 128 | } else { 129 | if (array_contains(this.textEscape, ch)) { 130 | inText = true 131 | escape = ch 132 | } 133 | if (ch !== ' ') { 134 | current += ch 135 | if (array_contains(this.sentinels, current.toLowerCase())) { 136 | results.push(current) 137 | results.push('') 138 | current = '' 139 | } 140 | } else { 141 | results[results.length - 1] += current 142 | current = '' 143 | } 144 | } 145 | } 146 | if (current !== '') results[results.length - 1] += current 147 | if (results[results.length - 1] === '') results.pop() 148 | return results 149 | } 150 | } 151 | WhereParser.prototype.constructor = WhereParser 152 | module.exports = WhereParser 153 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverless-iot-offline", 3 | "version": "1.1.0", 4 | "description": "Start AWS IoT service offline. Manages topic subscriptions, lifecycle events, thing shadow management and rule engine with limited SQL syntax support.", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "jest" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/mitipi/serverless-iot-offline.git" 12 | }, 13 | "keywords": [ 14 | "serverless", 15 | "iot" 16 | ], 17 | "author": "Nenad Panic", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/mitipi/serverless-iot-offline/issues" 21 | }, 22 | "homepage": "https://github.com/mitipi/serverless-iot-offline#readme", 23 | "dependencies": { 24 | "aws-sdk": "^2.1242.0", 25 | "aws-sdk-mock": "^5.8.0", 26 | "lodash": "^4.17.21", 27 | "mosca": "^2.8.3", 28 | "mqtt": "^4.3.7", 29 | "mqtt-match": "^3.0.0", 30 | "redis": "^4.3.1" 31 | }, 32 | "peerDependencies": { 33 | "serverless-offline": "^6.1.0" 34 | }, 35 | "jest": { 36 | "testMatch": [ 37 | "**/__tests__/*.js" 38 | ] 39 | }, 40 | "devDependencies": { 41 | "jest": "^29.2.2" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /ruleHandler.js: -------------------------------------------------------------------------------- 1 | const functionHelper = require('serverless-offline/src/functionHelper') 2 | const createLambdaContext = require('serverless-offline/src/LambdaContext') 3 | const path = require('path') 4 | const {parseSelect} = require('./iotSql/parseSql') 5 | const {applySelect} = require('./iotSql/applySqlSelect') 6 | const {applyWhereClause} = require('./iotSql/applySqlWhere') 7 | const {applyActions} = require('./iotSql/applyActions') 8 | const { fillSubstitutionTemplates } = require('./iotSql/substitutionTemplates') 9 | const mqtt = require('mqtt') 10 | const mqttMatch = require('mqtt-match') 11 | const _ = require('lodash') 12 | const {topic, accountid, clientid, timestamp, encode} = require('./iotSql/sqlFunctions') 13 | 14 | /** 15 | * Searches serverless.yml for functions configurations. 16 | * Creates mqtt client which connects to Iot broker and subscribes to topics that are defined in discovered functions. 17 | * Links discovered functions to topic and calls them when their topic gets published. 18 | */ 19 | module.exports = (slsOptions, slsService, serverless, log) => { 20 | const {location} = slsOptions 21 | const topicsToFunctionsMap = {} 22 | const republishMap = {} 23 | 24 | Object.keys(slsService.functions).forEach(key => { 25 | const fun = getFunction(key, slsService) 26 | const servicePath = path.join(serverless.config.servicePath, location) 27 | const funOptions = functionHelper.getFunctionOptions(fun, key, servicePath) 28 | 29 | if (!fun.environment) { 30 | fun.environment = {} 31 | } 32 | 33 | if (!(fun.events && fun.events.length)) { 34 | return 35 | } 36 | 37 | fun.events.forEach(event => { 38 | if (!event.iot) return 39 | 40 | const {iot} = event 41 | const {sql, actions} = iot 42 | 43 | const parsed = parseSelect(sql) 44 | const topicMatcher = parsed.topic 45 | if (!topicsToFunctionsMap[topicMatcher]) { 46 | topicsToFunctionsMap[topicMatcher] = [] 47 | } 48 | 49 | topicsToFunctionsMap[topicMatcher].push({ 50 | fn: fun, 51 | name: key, 52 | options: funOptions, 53 | select: parsed.select, 54 | whereClause: parsed.whereClause, 55 | actions 56 | }) 57 | }) 58 | }) 59 | 60 | // search in resources for Iot rule events 61 | const resources = _.get(slsService, 'resources.Resources', {}) 62 | Object.keys(resources).forEach(key => { 63 | const ruleConf = _.get(slsService.resources.Resources[key], 'Properties.TopicRulePayload') 64 | 65 | if (!ruleConf || ruleConf.RuleDisabled === "true") { 66 | return 67 | } 68 | 69 | registerLambdaRules(key, ruleConf) 70 | registerRepublishRules(key, ruleConf) 71 | 72 | function registerLambdaRules (key, ruleConf) { 73 | const actions = [] 74 | const sql = ruleConf.Sql 75 | const parsed = parseSelect(sql) 76 | const topicMatcher = parsed.topic 77 | let fun 78 | let funName 79 | let funOptions 80 | 81 | if (!topicsToFunctionsMap[topicMatcher]) { 82 | topicsToFunctionsMap[topicMatcher] = [] 83 | } 84 | 85 | ruleConf.Actions.forEach((action) => { 86 | if (action.Lambda) { 87 | const awsFunName = action.Lambda.FunctionArn['Fn::GetAtt'][0] 88 | funName = _.lowerFirst(awsFunName).replace('LambdaFunction', '') 89 | fun = getFunction(funName, slsService) 90 | const servicePath = path.join(serverless.config.servicePath, location) 91 | funOptions = functionHelper.getFunctionOptions(fun, key, servicePath) 92 | 93 | if (!fun.environment) { 94 | fun.environment = {} 95 | } 96 | } else { 97 | actions.push(action) 98 | } 99 | }) 100 | 101 | if (fun && funName && funOptions) { 102 | topicsToFunctionsMap[topicMatcher].push({ 103 | fn: fun, 104 | name: funName, 105 | options: funOptions, 106 | select: parsed.select, 107 | whereClause: parsed.whereClause, 108 | actions 109 | }) 110 | } else { 111 | delete topicsToFunctionsMap[topicMatcher] 112 | } 113 | } 114 | 115 | function registerRepublishRules (key, ruleConf) { 116 | ruleConf.Actions 117 | .filter(action => action.Republish) 118 | .forEach(republishAction => { 119 | const topicMatcher = parseSelect(ruleConf.Sql).topic 120 | 121 | if (!republishMap[topicMatcher]) { 122 | republishMap[topicMatcher] = [] 123 | } 124 | const cleanedRepublishTopic = cleanRepublishTopic(republishAction.Republish.Topic) 125 | republishMap[topicMatcher].push({ 126 | republishTopic: cleanedRepublishTopic 127 | }) 128 | }) 129 | 130 | function cleanRepublishTopic(topic) { 131 | return topic.replace(/^\${2}/, '$') 132 | } 133 | } 134 | }) 135 | 136 | const client = mqtt.connect(`ws://${slsOptions.host}:${slsOptions.httpPort}/mqqt`) 137 | client.on('connect', () => { 138 | log('Rule engine connected to IOT broker') 139 | for (let topicMatcher in topicsToFunctionsMap) { 140 | client.subscribe(topicMatcher) 141 | } 142 | for (let topicMatcher in republishMap) { 143 | client.subscribe(topicMatcher) 144 | } 145 | }) 146 | 147 | client.on('message', (topicUrl, message) => { 148 | const functionMatches = Object.keys(topicsToFunctionsMap).filter(topicMatcher => mqttMatch(topicMatcher, topicUrl)) 149 | if (functionMatches.length > 0) { 150 | functionMatches.forEach(triggerLambdaRules) 151 | } 152 | 153 | const republishMatches = Object.keys(republishMap).filter(topicMatcher => mqttMatch(topicMatcher, topicUrl)) 154 | if (republishMatches.length > 0) { 155 | republishMatches.forEach(topicMatcher => { 156 | triggerRepublishRules(topicMatcher, topicUrl, message) 157 | }) 158 | } 159 | 160 | function triggerLambdaRules (topicMatcher) { 161 | let functions = topicsToFunctionsMap[topicMatcher] 162 | functions.forEach(fnInfo => { 163 | const {fn, name, options, select, whereClause, actions} = fnInfo 164 | const requestId = Math.random().toString().slice(2) 165 | 166 | if (applyWhereClause(message, whereClause, log, name)) { 167 | let handler // The lambda function 168 | const event = applySelect({ 169 | select, 170 | payload: message, 171 | context: { 172 | topic: (index) => topic(index, topicUrl), 173 | clientid: () => clientid(topicUrl), 174 | timestamp: () => timestamp(), 175 | accountid: () => accountid(), 176 | encode: (field, encoding) => encode(message, field, encoding) 177 | } 178 | }) 179 | 180 | if (actions && actions.length) { 181 | applyActions(actions, event, log) 182 | } 183 | 184 | try { 185 | process.env = _.extend({}, slsService.provider.environment, slsService.functions[name].environment, process.env) 186 | handler = functionHelper.createHandler(options, slsOptions) 187 | } catch (err) { 188 | log(`Error while loading ${name}: ${err.stack}, ${requestId}`) 189 | return 190 | } 191 | 192 | const lambdaContext = createLambdaContext(fn) 193 | try { 194 | handler(event, lambdaContext, lambdaContext.done) 195 | } catch (error) { 196 | log(`Uncaught error in your '${name}' handler: ${error.stack}, ${requestId}`) 197 | } 198 | } 199 | }) 200 | } 201 | 202 | function triggerRepublishRules(topicMatcher, topicUrl, originalMessage) { 203 | if (republishMap[topicMatcher]) { 204 | republishMap[topicMatcher].forEach(republishRule => { 205 | const republishTopic = fillSubstitutionTemplates(topicUrl, republishRule.republishTopic) 206 | client.publish(republishTopic, originalMessage, () => { 207 | console.log(`Republished from: "${topicUrl}", to: "${republishTopic}"`) 208 | }) 209 | }) 210 | } 211 | 212 | } 213 | }) 214 | } 215 | 216 | const getFunction = (key, slsService) => { 217 | const fun = slsService.getFunction(key) 218 | if (!fun.timeout) { 219 | fun.timeout = slsService.provider.timeout 220 | } 221 | 222 | return fun 223 | } 224 | -------------------------------------------------------------------------------- /shadowService.js: -------------------------------------------------------------------------------- 1 | const mqtt = require('mqtt') 2 | const redis = require('redis') 3 | const {getTopics, removeNulledProps, errObj} = require('./util') 4 | const _ = require('lodash') 5 | 6 | let client 7 | let redisClient 8 | let log 9 | 10 | module.exports = (options, _log) => { 11 | client = mqtt.connect(`ws://${options.host}:${options.httpPort}/mqqt`) 12 | redisClient = redis.createClient() 13 | log = _log 14 | 15 | client.on('connect', () => { 16 | client.subscribe('$aws/things/+/shadow/update') 17 | client.subscribe('$aws/things/+/shadow/delete') 18 | client.subscribe('$aws/things/+/shadow/get') 19 | log('Shadow service connected to IOT broker.') 20 | }) 21 | 22 | client.on('message', (topic, message) => { 23 | const serialNumber = topic.split('/')[2] 24 | const action = topic.split('/')[4] 25 | switch (action) { 26 | case 'update': 27 | handleUpdateTopic(serialNumber, JSON.parse(message.toString())) 28 | break 29 | case 'get': 30 | log(`Retrieving shadow for '${serialNumber}'`) 31 | handleGetTopic(serialNumber) 32 | break 33 | case 'delete': 34 | log(`Deleting shadow for '${serialNumber}'`) 35 | handleDeleteTopic(serialNumber) 36 | } 37 | }) 38 | 39 | return {client, redisClient} 40 | } 41 | 42 | const handleUpdateTopic = (serialNumber, shadow) => { 43 | redisClient.get(serialNumber, (err, result) => { 44 | if (err) { 45 | return publishError(err) 46 | } 47 | if (shadow.state.desired) { 48 | if (!result) { 49 | return publishError('Can\'t update desired state for non existing thing.') 50 | } 51 | return updateShadowDesired(serialNumber, JSON.parse(result), shadow) 52 | } else { 53 | return updateShadow(serialNumber, (result ? JSON.parse(result) : null), shadow) 54 | } 55 | }) 56 | } 57 | 58 | const updateShadow = (serialNumber, prevShadow, shadow) => { 59 | log(`Updating shadow reported state for '${serialNumber}'`) 60 | const newShadow = _.merge(prevShadow, shadow) 61 | updateThingDb(serialNumber, newShadow).then((version) => { 62 | publishAccepted(newShadow, serialNumber, version) 63 | publishDocuments(prevShadow, serialNumber, newShadow, version) 64 | }).catch((err) => publishError(err, serialNumber)) 65 | } 66 | 67 | const updateShadowDesired = (serialNumber, prevShadow, desiredShadow) => { 68 | log(`Updating shadow desired state for '${serialNumber}'`) 69 | const newShadow = _.merge(prevShadow, desiredShadow) 70 | updateThingDb(serialNumber, newShadow).then((version) => { 71 | publishAccepted(desiredShadow, serialNumber, version) 72 | publishDelta(desiredShadow, serialNumber, version) 73 | publishDocuments(prevShadow, serialNumber, newShadow, version) 74 | }).catch((err) => publishError(err, serialNumber)) 75 | } 76 | 77 | const publishAccepted = (shadow, serialNumber, version) => { 78 | client.publish(getTopics(serialNumber).updateAccepted, JSON.stringify(Object.assign(shadow, { 79 | timestamp: new Date().getTime(), 80 | version 81 | }))) 82 | } 83 | 84 | const publishDelta = (shadow, serialNumber, version) => { 85 | client.publish(getTopics(serialNumber).updateDelta, JSON.stringify({ 86 | timestamp: new Date().getTime(), 87 | version, 88 | state: shadow.state.desired 89 | })) 90 | } 91 | 92 | const publishDocuments = (prevShadow, serialNumber, currentShadow, version) => { 93 | client.publish(getTopics(serialNumber).updateDocuments, JSON.stringify({ 94 | previous: Object.assign({}, prevShadow, {version: --version}), 95 | current: Object.assign({}, currentShadow, {version}), 96 | timestamp: new Date().getTime() 97 | })) 98 | } 99 | 100 | const publishError = (err, serialNumber) => { 101 | client.publish(getTopics(serialNumber).updateRejected, JSON.stringify(errObj(err.toString()))) 102 | } 103 | 104 | const updateThingDb = (serialNumber, shadow, isDelete) => { 105 | return new Promise((resolve, reject) => { 106 | const newShadow = removeNulledProps(shadow) 107 | redisClient.set(serialNumber, JSON.stringify(newShadow), (err) => { 108 | if (err) { 109 | return reject(err) 110 | } 111 | 112 | if (isDelete) { 113 | redisClient.set(`${serialNumber}Version`, '1', (err, version) => { 114 | if (err) { 115 | return reject(err) 116 | } 117 | return resolve(version) 118 | }) 119 | } else { 120 | redisClient.incr(`${serialNumber}Version`, (err, version) => { 121 | if (err) { 122 | return reject(err) 123 | } 124 | return resolve(version) 125 | }) 126 | } 127 | }) 128 | }) 129 | } 130 | 131 | const handleGetTopic = (serialNumber) => { 132 | redisClient.get(serialNumber, (err, result) => { 133 | if (err || !result) { 134 | return client.publish(getTopics(serialNumber).getRejected, JSON.stringify(errObj(err ? err.toString() : 'No shadow stored for that serial number'))) 135 | } 136 | const shadow = JSON.parse(result) 137 | redisClient.get(`${serialNumber}Version`, (err, version) => { 138 | return client.publish(getTopics(serialNumber).getAccepted, JSON.stringify(Object.assign({}, shadow, { 139 | version, 140 | timestamp: new Date().getTime() 141 | }))) 142 | }) 143 | }) 144 | } 145 | 146 | const handleDeleteTopic = (serialNumber) => { 147 | updateThingDb(serialNumber, null, true).then((version) => { 148 | client.publish(getTopics(serialNumber).deleteAccepted, JSON.stringify({ 149 | version, 150 | timestamp: new Date().getTime() 151 | })) 152 | }).catch((err) => client.publish(getTopics(serialNumber).deleteRejected, JSON.stringify(errObj(err.toString())))) 153 | } 154 | -------------------------------------------------------------------------------- /testData.js: -------------------------------------------------------------------------------- 1 | module.exports.sqlParseTestData = [ 2 | { 3 | sql: `SELECT topic(3) as deviceId, * FROM '$aws/things/sn:123/shadow/update' WHERE NOT state.reported.client.confirmed`, 4 | payload: `{"state": {"reported": {"client": {"confirmed": false}}}}`, 5 | expected: { 6 | parsed: { 7 | select: [{alias: 'deviceId', field: 'topic(3)'}, {field: '*'}], 8 | topic: '$aws/things/sn:123/shadow/update', 9 | whereClause: 'NOT state.reported.client.confirmed' 10 | }, 11 | whereEvaluatesTo: true, 12 | event: { 13 | deviceId: 'sn:123', 14 | state: {reported: {client: {confirmed: false}}} 15 | } 16 | } 17 | }, 18 | { 19 | sql: `SELECT state.reported.preferences.*, state.reported.activity as activityId FROM '$aws/things/+/shadow/update/accepted' WHERE state.reported.preferences.volume > 30`, 20 | payload: `{"state": {"reported": {"preferences": {"volume": 31}, "activity": 1}}}`, 21 | expected: { 22 | parsed: { 23 | select: [{field: 'state.reported.preferences.*'}, {field: 'state.reported.activity', alias: 'activityId'}], 24 | topic: '$aws/things/+/shadow/update/accepted', 25 | whereClause: 'state.reported.preferences.volume > 30' 26 | }, 27 | whereEvaluatesTo: true, 28 | event: { 29 | volume: 31, 30 | activityId: 1 31 | } 32 | } 33 | }, 34 | { 35 | sql: `SELECT state.desired.client FROM '$aws/things/+/shadow/update/accepted' WHERE state.desired.client = TRUE`, 36 | payload: `{"state": {"desired": {"client": true}}}`, 37 | expected: { 38 | parsed: { 39 | select: [{field: 'state.desired.client'}], 40 | topic: '$aws/things/+/shadow/update/accepted', 41 | whereClause: 'state.desired.client = TRUE' 42 | }, 43 | whereEvaluatesTo: true, 44 | event: { 45 | client: true 46 | } 47 | } 48 | }, 49 | { 50 | sql: `SELECT *, topic(), topic(2), topic(3), topic(4) FROM '$aws/things/+/shadow/update' WHERE state.reported.mode = 'STAND_BY'`, 51 | payload: `{"state": {"reported": {"mode": "STAND_BY"}}}`, 52 | expected: { 53 | parsed: { 54 | select: [ 55 | {field: '*'}, 56 | {field: 'topic()'}, 57 | {field: 'topic(2)'}, 58 | {field: 'topic(3)'}, 59 | {field: 'topic(4)'} 60 | ], 61 | topic: '$aws/things/+/shadow/update', 62 | whereClause: `state.reported.mode = 'STAND_BY'` 63 | }, 64 | whereEvaluatesTo: true, 65 | event: { 66 | state: {reported: {mode: 'STAND_BY'}}, 67 | topic: '$aws/things/+/shadow/update', 68 | 'topic(2)': 'things', 69 | 'topic(3)': '+', 70 | 'topic(4)': 'shadow' 71 | } 72 | } 73 | }, 74 | { 75 | sql: `SELECT clientid(), accountid() as account FROM '$aws/events/presence/connected/test_client'`, 76 | payload: `{"event": "connected"}`, 77 | expected: { 78 | parsed: { 79 | select: [{field: 'clientid()'}, {field: 'accountid()', alias: 'account'}], 80 | topic: '$aws/events/presence/connected/test_client' 81 | }, 82 | whereEvaluatesTo: true, 83 | event: { 84 | account: 'test_account', 85 | clientid: 'test_client' 86 | } 87 | } 88 | }, 89 | { 90 | sql: `SELECT encode(*, 'base64') as encodedPayload FROM '$aws/things/sn:123/shadow/update'`, 91 | payload: `{"state": {"reported": {"mode": "STAND_BY"}}}`, 92 | expected: { 93 | parsed: { 94 | select: [{ field: "encode(*, 'base64')", alias: "encodedPayload" }], 95 | topic: "$aws/things/sn:123/shadow/update", 96 | }, 97 | whereEvaluatesTo: true, 98 | event: { 99 | encodedPayload: 100 | "eyJzdGF0ZSI6IHsicmVwb3J0ZWQiOiB7Im1vZGUiOiAiU1RBTkRfQlkifX19", 101 | }, 102 | }, 103 | }, 104 | { 105 | sql: `SELECT encode(state.reported.mode, 'base64') as encodedPayload FROM '$aws/things/sn:123/shadow/update'`, 106 | payload: `{"state": {"reported": {"mode": "STAND_BY"}}}`, 107 | expected: { 108 | parsed: { 109 | select: [ 110 | { 111 | field: "encode(state.reported.mode, 'base64')", 112 | alias: "encodedPayload", 113 | }, 114 | ], 115 | topic: "$aws/things/sn:123/shadow/update", 116 | }, 117 | whereEvaluatesTo: true, 118 | event: { 119 | encodedPayload: "U1RBTkRfQlk=", 120 | }, 121 | }, 122 | } 123 | ] 124 | -------------------------------------------------------------------------------- /util.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const fs = require('fs') 3 | 4 | const removeNulledProps = (shadow) => { 5 | let obj = JSON.parse(JSON.stringify(shadow)) 6 | Object.keys(obj).forEach(key => { 7 | if (obj[key] && typeof obj[key] === 'object') { 8 | removeNulledProps(obj[key]) 9 | } else if (obj[key] == null) { 10 | delete obj[key] 11 | } 12 | }); 13 | return obj 14 | } 15 | 16 | module.exports.removeNulledProps = removeNulledProps 17 | 18 | module.exports.getTopics = (serialNumber) => { 19 | return { 20 | updateAccepted: `$aws/things/${serialNumber}/shadow/update/accepted`, 21 | updateDocuments: `$aws/things/${serialNumber}/shadow/update/documents`, 22 | updateDelta: `$aws/things/${serialNumber}/shadow/update/delta`, 23 | updateRejected: `$aws/things/${serialNumber}/shadow/update/rejected`, 24 | getAccepted: `$aws/things/${serialNumber}/shadow/get/accepted`, 25 | getRejected: `$aws/things/${serialNumber}/shadow/get/rejected`, 26 | deleteAccepted: `$aws/things/${serialNumber}/shadow/delete/accepted`, 27 | deleteRejected: `$aws/things/${serialNumber}/shadow/delete/rejected` 28 | } 29 | } 30 | 31 | module.exports.seedShadows = (seedPath, redisClient) => { 32 | const location = path.join(process.cwd(), seedPath) 33 | fs.exists(location, (exists) => { 34 | if (exists) { 35 | const shadows = require(location) 36 | Object.keys(shadows).forEach((serialNumber) => { 37 | redisClient.set(serialNumber, JSON.stringify(shadows[serialNumber])) 38 | redisClient.set(`${serialNumber}Version`, '1') 39 | }) 40 | } 41 | }) 42 | } 43 | 44 | module.exports.seedPolicies = (seedPath, redisClient) => { 45 | if(!seedPath) return 46 | const location = path.join(process.cwd(), seedPath) 47 | fs.exists(location, (exists) => { 48 | if (exists) { 49 | const policies = require(location) 50 | Object.keys(policies).forEach((policyName) => { 51 | redisClient.set(policyName, JSON.stringify(policies[policyName])) 52 | }) 53 | } 54 | }) 55 | } 56 | 57 | module.exports.errObj = (message) => { 58 | return { 59 | code: 500, 60 | message, 61 | timestamp: new Date().getTime(), 62 | clientToken: 'token' 63 | } 64 | } 65 | --------------------------------------------------------------------------------