├── LICENSE ├── README.md ├── index.coffee ├── package.json ├── script ├── bootstrap └── test └── src └── luigi.coffee /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2015 Houzz Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hubot Luigi Script 2 | 3 | Interact with luigi central scheduler, to get job status, worker status, etc. 4 | 5 | 6 | ## Installation 7 | 8 | Add the package `hubot-luigi` entry to the `packages.json` file 9 | (you may need to create this file). 10 | 11 | "dependencies": { 12 | "hubot-luigi": "1.0.x" 13 | } 14 | 15 | Run the following command to make sure the module is installed. 16 | 17 | #npm install hubot-luigi 18 | 19 | To enable the script, add the `hubot-luigi` entry to the `external-scripts.json` 20 | file (you may need to create this file). 21 | 22 | ["hubots-luigi"] 23 | 24 | 25 | ## Configuration 26 | * HUBOT_LUIGI_ENDPOINT - specify luigi scheduler api endpoint, like 'http://localhost:8082/api/' 27 | 28 | ## Usage Examples 29 | ### stats 30 | user1> hubot luigi stats 31 | hubot> 8 jobs running, 1315 jobs pending, 76 jobs failed, 5737 jobs disabled 32 | 33 | 34 | ### show 35 | user1> hubot luigi show pending 36 | hubot> Wed Sep 02 2015 13:27:36 GMT-0700 (PDT) Task(param1=value) p=50 37 | Wed Sep 02 2015 13:27:36 GMT-0700 (PDT) WrapperTask(run=all_tasks) p=50 38 | Wed Sep 02 2015 13:27:36 GMT-0700 (PDT) HadoopHourlyJob(jar=process_logs.jar, time=2015-08-30T00) p=100 39 | Wed Sep 02 2015 13:27:36 GMT-0700 (PDT) ScpTask(file=filename.csv, target=server) p=10 40 | 41 | Show can be run with any task status, including pending, running, disabled, failed. If more than 20 42 | tasks are to be shown, it will truncate the list and state how many more there are. 43 | 44 | 45 | ### search 46 | user1> hubot luigi search Hadoop 47 | hubot> DONE Sun Aug 30 2015 00:12:21 GMT-0700 (PDT) HadoopHourlyJob(jar=process_logs.jar, time=2015-08-30T01) p=100 48 | DONE Sun Aug 30 2015 00:12:21 GMT-0700 (PDT) HadoopHourlyJob(jar=process_logs.jar, time=2015-08-30T00) p=100 49 | DONE Sun Aug 30 2015 00:12:21 GMT-0700 (PDT) HadoopHourlyJob(jar=process_logs.jar, time=2015-08-30T02) p=100 50 | DONE Sun Aug 30 2015 01:12:18 GMT-0700 (PDT) HadoopHourlyJob(jar=analytics_reports.jar, time=2015-08-30T00) p=15 51 | 52 | Search matches the query against task ids and is case sensitive. Results are truncated as in show. 53 | 54 | 55 | ### resources 56 | user1> hubot luigi resources 57 | hubot> impala: 3/4 58 | hadoop_xl: 2/2 59 | mysql_access: 0/5 60 | 61 | This will only show resources that are defined in luigi.cfg and not ones that are created in response 62 | to task scheduling. 63 | 64 | 65 | ### refresh resources 66 | user1> hubot luigi refreshresources 67 | hubot> impala: 3/2 68 | hadoop_xl: 2/3 69 | mysql_access: 0/4 70 | 71 | This causes the scheduler to dynamically reload the resource constraints from luigi.cfg. The current 72 | resource usage is then displayed, which can violate the new constraints until existing jobs have had 73 | a chance to finish. 74 | 75 | 76 | ### workers 77 | user1> hubot luigi workers 78 | hubot> 781608491 AllTasksFrontFill(days=1, end_time=2015-09-02T13:30) [10] 1 running, 637 pending 79 | 377433553 AllTasksFrontFill(days=1, end_time=2015-09-02T13:00) [10] 6 running, 486 pending 80 | 042179238 AllTasksFrontFill(days=1, end_time=2015-09-01T16:00) [10] 0 running, 46 pending 81 | 208362896 AllTasksFrontFill(days=5, end_time=2015-09-01T03:00) [10] 2 running, 496 pending 82 | 83 | ### worker 84 | user1> hubot luigi worker 208362896 85 | hubot> 208362896 AllTasksFrontFill(days=5, end_time=2015-09-01T03:00) [10] 3 running, 496 pending, 478 unique pending 86 | 87 | running tasks: NumberedTask(n=7), NumberedTask(n=18), NumberedTask(n=3) 88 | -------------------------------------------------------------------------------- /index.coffee: -------------------------------------------------------------------------------- 1 | fs = require 'fs' 2 | path = require 'path' 3 | 4 | module.exports = (robot, scripts) -> 5 | scriptsPath = path.resolve(__dirname, 'src') 6 | fs.exists scriptsPath, (exists) -> 7 | if exists 8 | for script in fs.readdirSync(scriptsPath) 9 | if scripts? and '*' not in scripts 10 | robot.loadFile(scriptsPath, script) if script in scripts 11 | else 12 | robot.loadFile(scriptsPath, script) 13 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hubot-luigi", 3 | "description": "A hubot script that interact with luigi scheduler", 4 | "version": "1.2.0", 5 | "author": "Kyle Sun ", 6 | "license": "APACHE 2.0", 7 | "keywords": [ 8 | "hubot", 9 | "hubot-scripts", 10 | "luigi" 11 | ], 12 | "repository": { 13 | "type": "git", 14 | "url": "git://github.com/Houzz/hubot-luigi.git" 15 | }, 16 | "bugs": { 17 | "url": "https://github.com/Houzz/hubot-luigi/issues" 18 | }, 19 | "dependencies": {}, 20 | "peerDependencies": {}, 21 | "devDependencies": {}, 22 | "main": "index.coffee", 23 | "scripts": { 24 | "test": "grunt test" 25 | }, 26 | "homepage": "https://github.com/Houzz/hubot-luigi" 27 | } 28 | -------------------------------------------------------------------------------- /script/bootstrap: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Make sure everything is development forever 4 | export NODE_ENV=development 5 | 6 | # Load environment specific environment variables 7 | if [ -f .env ]; then 8 | source .env 9 | fi 10 | 11 | if [ -f .env.${NODE_ENV} ]; then 12 | source .env.${NODE_ENV} 13 | fi 14 | 15 | npm install 16 | 17 | # Make sure coffee and mocha are on the path 18 | export PATH="node_modules/.bin:$PATH" 19 | -------------------------------------------------------------------------------- /script/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # bootstrap environment 4 | source script/bootstrap 5 | 6 | mocha --compilers coffee:coffee-script 7 | -------------------------------------------------------------------------------- /src/luigi.coffee: -------------------------------------------------------------------------------- 1 | # Description: 2 | # 3 | # Viewing luigi stats 4 | # 5 | # Commands: 6 | # hubot luigi stats - Show overall stats 7 | # hubot luigi show - List RUNNING/PENDING/DONE tasks 8 | # hubot luigi search - Search task by task id 9 | # hubot luigi resources - Luigi resource summary 10 | # hubot luigi refresh resources - Luigi refresh resources from disk 11 | # hubot luigi workers - Luigi worker summary 12 | # hubot luigi worker - Show worker details 13 | # hubot luigi blockers - Show worker details 14 | # 15 | # Configuration: 16 | # HUBOT_LUIGI_ENDPOINT - luigi scheduler api endpoint, like 'http://localhost:8082/api/' 17 | # HUBOT_LUIGI_BLOCKERS_CRONTAB - crontab to run blocker alerts 18 | # HUBOT_LUIGI_BLOCKERS_THRESHOLD - alert threshold 19 | # HUBOT_LUIGI_BLOCKERS_ROOM - the room to post to for blocker alert 20 | # 21 | # URLS: 22 | # https://github.com/spotify/luigi/ 23 | # 24 | # Author: 25 | # interskh 26 | 27 | 28 | luigiApiEndpoint = process.env.HUBOT_LUIGI_ENDPOINT 29 | 30 | luigiBlockersCrontab = process.env.HUBOT_LUIGI_BLOCKERS_CRONTAB 31 | luigiBlockersThreshold = process.env.HUBOT_LUIGI_BLOCKERS_THRESHOLD 32 | luigiBlockersAlertRoom = process.env.HUBOT_LUIGI_BLOCKERS_ROOM 33 | 34 | module.exports = (robot) -> 35 | 36 | blockersAlert = -> 37 | console.log "running cronjob - getting luigi blockers stats" 38 | robot.http(luigiApiEndpoint + "blockers") 39 | .query(data: JSON.stringify({priority_sum: true, min_blocked: parseInt(luigiBlockersThreshold, 10), limit: 10})) 40 | .get() (err, res, body) -> 41 | try 42 | ret = JSON.parse body 43 | if ret.response.length > 0 44 | results = [] 45 | for w in ret.response 46 | results.push(w.blocked + " " + w.display_name) 47 | console.log "luigi blockers \n" + results.join("\n") 48 | robot.messageRoom luigiBlockersAlertRoom, "hi team, there seems to be some slackers in the pipeline: \n" + results.join("\n") 49 | catch error 50 | console.log body 51 | console.log error 52 | 53 | if luigiBlockersCrontab 54 | console.log("enabling luigi blockers") 55 | cronJob = require('cron').CronJob 56 | tz = 'America/Los_Angeles' 57 | try 58 | new cronJob(luigiBlockersCrontab, blockersAlert, null, true, tz) 59 | catch error 60 | console.log error 61 | 62 | robot.respond /luigi statu?s(\s*)$/i, (msg) -> 63 | callLuigiTaskList msg, "RUNNING", (res) -> 64 | running = numberOfTask(res) 65 | callLuigiTaskList msg, "PENDING", (res) -> 66 | pending = numberOfTask(res) 67 | callLuigiTaskList msg, "FAILED", (res) -> 68 | failed = numberOfTask(res) 69 | callLuigiTaskList msg, "DISABLED", (res) -> 70 | disabled = numberOfTask(res) 71 | msg.send running + " jobs running, " + pending + " jobs pending, " + failed + " jobs failed, " + disabled + " jobs disabled" 72 | 73 | robot.respond /luigi show( all)? (.*)(\s*)/i, (msg) -> 74 | status = msg.match[2].toUpperCase() 75 | callLuigiTaskList msg, status, (res) -> 76 | results = [] 77 | for t in sortTask(res) 78 | results.push(formatTask(t[0], t[1])) 79 | sendLimitedResult(msg, results, 20) 80 | 81 | robot.respond /luigi search (.*)(\s*)/i, (msg) -> 82 | callLuigiTaskSearch msg, msg.match[1], (res) -> 83 | results = [] 84 | counts = {} 85 | for status, d of res 86 | counts[status] = Object.keys(res[status]).length 87 | for t in sortTask(d) 88 | results.push(status + " " + formatTask(t[0], t[1])) 89 | if results.length > 0 90 | counts_message = "summary: " 91 | for status, count of counts 92 | counts_message += count + " " + status + ", " 93 | msg.send counts_message 94 | sendLimitedResult(msg, results, 20) 95 | 96 | robot.respond /luigi resources(\s*)/i, (msg) -> 97 | callLuigiResources msg, (res) -> 98 | results = [] 99 | for r in sortResource(res) 100 | resource = r[0] 101 | d = res[resource] 102 | results.push(resource + " : " + d.used + "/" + d.total) 103 | msg.send results.join("\n") 104 | 105 | robot.respond /luigi workers(\s*)/i, (msg) -> 106 | callLuigiWorkers msg, (res) -> 107 | results = [] 108 | for w in res 109 | results.push(w.salt + " " + w.first_task + " [" + w.workers + "] " + w.num_running + " running, " + w.num_pending + " pending") 110 | msg.send results.join("\n") 111 | 112 | robot.respond /luigi worker (.*)(\s*)/i, (msg) -> 113 | search = msg.match[1] 114 | callLuigiWorkers msg, (res) -> 115 | for w in res 116 | if w.salt == search 117 | msg.send(w.salt + " " + w.first_task + " [" + w.workers + "] " + w.num_running + " running, " + w.num_pending + " pending, " + w.num_uniques + " uniq pending") 118 | if w.num_running > 0 119 | tasks = [] 120 | for task_id, task of w.running 121 | tasks.push(task_id) 122 | msg.send("running tasks: " + tasks.join(", ")) 123 | 124 | robot.respond /luigi refresh resources(\s*)/i, (msg) -> 125 | callLuigiUpdateResources msg, (res) -> 126 | callLuigiResources msg, (res) -> 127 | results = [] 128 | for r in sortResource(res) 129 | resource = r[0] 130 | d = res[resource] 131 | results.push(resource + " : " + d.used + "/" + d.total) 132 | msg.send results.join("\n") 133 | 134 | robot.respond /luigi blockers(\s*)/i, (msg) -> 135 | callLuigiBlockers msg, (res) -> 136 | results = [] 137 | for w in res 138 | results.push(w.blocked + " " + w.display_name) 139 | msg.send results.join("\n") 140 | 141 | sendLimitedResult = (msg, results, n=0) -> 142 | if results.length > 0 143 | if n > 0 144 | if results.length > n 145 | msg.send results.slice(0, n).join("\n") + "... and " + (results.length - n) + " more" 146 | else 147 | msg.send results.slice(0, n).join("\n") 148 | else 149 | msg.send results.join("\n") 150 | 151 | callLuigiTaskList = (msg, jobType, cb) -> 152 | msg.http(luigiApiEndpoint + "task_list") 153 | .query(data: JSON.stringify({status: jobType, upstream_status: ""})) 154 | .get() (err, res, body) -> 155 | try 156 | ret = JSON.parse body 157 | cb ret.response 158 | catch error 159 | console.log body 160 | console.log error 161 | cb {} 162 | 163 | numberOfTask = (res) -> 164 | if res.hasOwnProperty("num_tasks") 165 | return res.num_tasks 166 | else 167 | return Object.keys(res).length 168 | 169 | callLuigiTaskSearch = (msg, str, cb) -> 170 | msg.http(luigiApiEndpoint + "task_search") 171 | .query(data: JSON.stringify({task_str: str})) 172 | .get() (err, res, body) -> 173 | try 174 | ret = JSON.parse body 175 | cb ret.response 176 | catch error 177 | console.log body 178 | console.log error 179 | cb {} 180 | 181 | callLuigiResources= (msg, cb) -> 182 | msg.http(luigiApiEndpoint + "resources") 183 | .get() (err, res, body) -> 184 | try 185 | ret = JSON.parse body 186 | cb ret.response 187 | catch error 188 | console.log body 189 | console.log error 190 | cb {} 191 | 192 | callLuigiUpdateResources= (msg, cb) -> 193 | msg.http(luigiApiEndpoint + "update_resources") 194 | .get() (err, res, body) -> 195 | try 196 | ret = JSON.parse body 197 | cb ret.response 198 | catch error 199 | console.log body 200 | console.log error 201 | cb {} 202 | 203 | callLuigiWorkers= (msg, cb) -> 204 | msg.http(luigiApiEndpoint + "worker_list") 205 | .get() (err, res, body) -> 206 | try 207 | ret = JSON.parse body 208 | cb ret.response 209 | catch error 210 | console.log body 211 | console.log error 212 | cb {} 213 | 214 | callLuigiBlockers = (msg, cb) -> 215 | msg.http(luigiApiEndpoint + "blockers") 216 | .query(data: JSON.stringify({priority_sum: true, min_blocked: 101, limit: 10})) 217 | .get() (err, res, body) -> 218 | try 219 | ret = JSON.parse body 220 | cb ret.response 221 | catch error 222 | console.log body 223 | console.log error 224 | cb {} 225 | 226 | sortTask = (taskDict) -> 227 | # tasks in taskDict should be in the same status 228 | sortable = [] 229 | for task_id, task of taskDict 230 | status = task.status 231 | sortable.push([task_id, task]) 232 | if status == 'RUNNING' 233 | sortable.sort (a,b) -> a[1].time_running - b[1].time_running 234 | else 235 | sortable.sort (a,b) -> a[1].start_time - b[1].start_time 236 | 237 | sortResource = (resourcesDict) -> 238 | sortable = [] 239 | for resource, d of resourcesDict 240 | if d.total == 0 241 | n = 0 242 | else if d.used == 0 243 | n = 0.1 / d.total 244 | else 245 | n = d.used / d.total 246 | sortable.push([resource, n]) 247 | sortable.sort (a,b) -> a[1] - b[1] 248 | 249 | formatTask = (task_id, task) -> 250 | if task.status == 'RUNNING' 251 | formatTime(task.time_running) + " " + task_id + " p=" + task.priority 252 | else 253 | formatTime(task.start_time) + " " + task_id + " p=" + task.priority 254 | 255 | formatTime = (ts) -> 256 | new Date(Math.floor(ts*1000)).toLocaleString() 257 | --------------------------------------------------------------------------------