├── .gitignore ├── README.md ├── gulpfile.js ├── package.json ├── src ├── constants │ └── global.constant.ts ├── domain │ ├── default-arguments.ts │ ├── glue-job.ts │ ├── glue-trigger.ts │ └── support-files.ts ├── helpers │ ├── aws.helper.ts │ ├── glue.helper.ts │ └── serverless.helper.ts ├── index.ts ├── interfaces │ ├── bucket-options.interface.ts │ ├── create-bucket-config.interface.ts │ ├── default-arguments.interface.ts │ ├── glue-job.interface.ts │ ├── glue-plugin-config.interce.ts │ ├── glue-trigger-action.interface.ts │ ├── glue-trigger.interface.ts │ └── support-files.interface.ts ├── schemas │ └── glue.schema.ts ├── services │ └── serverless.service.ts └── utils │ ├── cloud-formation.utils.ts │ └── string.utils.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/node,linux,macos,windows,visualstudio,jetbrains+all 3 | # Edit at https://www.gitignore.io/?templates=node,linux,macos,windows,visualstudio,jetbrains+all 4 | 5 | ### JetBrains+all ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | .idea/**/usage.statistics.xml 13 | .idea/**/dictionaries 14 | .idea/**/shelf 15 | 16 | #avoid openapi.json 17 | openapi.json 18 | 19 | # Generated files 20 | .idea/**/contentModel.xml 21 | 22 | # Sensitive or high-churn files 23 | .idea/**/dataSources/ 24 | .idea/**/dataSources.ids 25 | .idea/**/dataSources.local.xml 26 | .idea/**/sqlDataSources.xml 27 | .idea/**/dynamic.xml 28 | .idea/**/uiDesigner.xml 29 | .idea/**/dbnavigator.xml 30 | 31 | # Gradle 32 | .idea/**/gradle.xml 33 | .idea/**/libraries 34 | 35 | # Gradle and Maven with auto-import 36 | # When using Gradle or Maven with auto-import, you should exclude module files, 37 | # since they will be recreated, and may cause churn. Uncomment if using 38 | # auto-import. 39 | # .idea/modules.xml 40 | # .idea/*.iml 41 | # .idea/modules 42 | # *.iml 43 | # *.ipr 44 | 45 | # CMake 46 | cmake-build-*/ 47 | 48 | # Mongo Explorer plugin 49 | .idea/**/mongoSettings.xml 50 | 51 | # File-based project format 52 | *.iws 53 | 54 | # IntelliJ 55 | out/ 56 | 57 | # mpeltonen/sbt-idea plugin 58 | .idea_modules/ 59 | 60 | # JIRA plugin 61 | atlassian-ide-plugin.xml 62 | 63 | # Cursive Clojure plugin 64 | .idea/replstate.xml 65 | 66 | # Crashlytics plugin (for Android Studio and IntelliJ) 67 | com_crashlytics_export_strings.xml 68 | crashlytics.properties 69 | crashlytics-build.properties 70 | fabric.properties 71 | 72 | # Editor-based Rest Client 73 | .idea/httpRequests 74 | 75 | # Android studio 3.1+ serialized cache file 76 | .idea/caches/build_file_checksums.ser 77 | 78 | ### JetBrains+all Patch ### 79 | # Ignores the whole .idea folder and all .iml files 80 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 81 | 82 | .idea/ 83 | 84 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 85 | 86 | *.iml 87 | modules.xml 88 | .idea/misc.xml 89 | *.ipr 90 | 91 | # Sonarlint plugin 92 | .idea/sonarlint 93 | 94 | # Sonar Scanner plugin 95 | .scannerwork 96 | 97 | ### Linux ### 98 | *~ 99 | 100 | # temporary files which can be created if a process still has a handle open of a deleted file 101 | .fuse_hidden* 102 | 103 | # KDE directory preferences 104 | .directory 105 | 106 | # Linux trash folder which might appear on any partition or disk 107 | .Trash-* 108 | 109 | # .nfs files are created when an open file is removed but is still being accessed 110 | .nfs* 111 | 112 | ### macOS ### 113 | # General 114 | .DS_Store 115 | .AppleDouble 116 | .LSOverride 117 | 118 | # Icon must end with two \r 119 | Icon 120 | 121 | # Thumbnails 122 | ._* 123 | 124 | # Files that might appear in the root of a volume 125 | .DocumentRevisions-V100 126 | .fseventsd 127 | .Spotlight-V100 128 | .TemporaryItems 129 | .Trashes 130 | .VolumeIcon.icns 131 | .com.apple.timemachine.donotpresent 132 | 133 | # Directories potentially created on remote AFP share 134 | .AppleDB 135 | .AppleDesktop 136 | Network Trash Folder 137 | Temporary Items 138 | .apdisk 139 | 140 | ### Node ### 141 | # Logs 142 | logs 143 | *.log 144 | npm-debug.log* 145 | yarn-debug.log* 146 | yarn-error.log* 147 | lerna-debug.log* 148 | 149 | # Diagnostic reports (https://nodejs.org/api/report.html) 150 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 151 | 152 | # Runtime access 153 | pids 154 | *.pid 155 | *.seed 156 | *.pid.lock 157 | 158 | # Directory for instrumented libs generated by jscoverage/JSCover 159 | lib 160 | lib-cov 161 | 162 | # Coverage directory used by tools like istanbul 163 | coverage 164 | *.lcov 165 | 166 | # nyc test coverage 167 | .nyc_output 168 | 169 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 170 | .grunt 171 | 172 | # Bower dependency directory (https://bower.io/) 173 | bower_components 174 | 175 | # node-waf configuration 176 | .lock-wscript 177 | 178 | # Compiled binary addons (https://nodejs.org/api/addons.html) 179 | build/Release 180 | 181 | # Dependency directories 182 | node_modules/ 183 | jspm_packages/ 184 | 185 | # TypeScript v1 declaration files 186 | typings/ 187 | 188 | # TypeScript cache 189 | *.tsbuildinfo 190 | 191 | # Optional npm cache directory 192 | .npm 193 | 194 | # Optional eslint cache 195 | .eslintcache 196 | 197 | # Optional REPL history 198 | .node_repl_history 199 | 200 | # Output of 'npm pack' 201 | *.tgz 202 | 203 | # Yarn Integrity file 204 | .yarn-integrity 205 | 206 | # dotenv environment variables file 207 | .env 208 | .env.test 209 | 210 | # parcel-bundler cache (https://parceljs.org/) 211 | .cache 212 | 213 | # next.js build output 214 | .next 215 | 216 | # nuxt.js build output 217 | .nuxt 218 | 219 | # rollup.js default build output 220 | dist/ 221 | 222 | # Uncomment the public line if your project uses Gatsby 223 | # https://nextjs.org/blog/next-9-1#public-directory-support 224 | # https://create-react-app.dev/docs/using-the-public-folder/#docsNav 225 | # public 226 | 227 | # Storybook build outputs 228 | .out 229 | .storybook-out 230 | 231 | # vuepress build output 232 | .vuepress/dist 233 | 234 | # Serverless directories 235 | .serverless/ 236 | 237 | # FuseBox cache 238 | .fusebox/ 239 | 240 | # DynamoDB Local files 241 | .dynamodb/ 242 | 243 | # Temporary folders 244 | tmp/ 245 | temp/ 246 | 247 | ### Windows ### 248 | # Windows thumbnail cache files 249 | Thumbs.db 250 | Thumbs.db:encryptable 251 | ehthumbs.db 252 | ehthumbs_vista.db 253 | 254 | # Dump file 255 | *.stackdump 256 | 257 | # Folder config file 258 | [Dd]esktop.ini 259 | 260 | # Recycle Bin used on file shares 261 | $RECYCLE.BIN/ 262 | 263 | # Windows Installer files 264 | *.cab 265 | *.msi 266 | *.msix 267 | *.msm 268 | *.msp 269 | 270 | # Windows shortcuts 271 | *.lnk 272 | 273 | ### VisualStudio ### 274 | ## Ignore Visual Studio temporary files, build results, and 275 | ## files generated by popular Visual Studio add-ons. 276 | ## 277 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 278 | 279 | # User-specific files 280 | *.rsuser 281 | *.suo 282 | *.user 283 | *.userosscache 284 | *.sln.docstates 285 | 286 | # User-specific files (MonoDevelop/Xamarin Studio) 287 | *.userprefs 288 | 289 | # Mono auto generated files 290 | mono_crash.* 291 | 292 | # Build results 293 | [Dd]ebug/ 294 | [Dd]ebugPublic/ 295 | [Rr]elease/ 296 | [Rr]eleases/ 297 | x64/ 298 | x86/ 299 | [Aa][Rr][Mm]/ 300 | [Aa][Rr][Mm]64/ 301 | bld/ 302 | [Bb]in/ 303 | [Oo]bj/ 304 | [Ll]og/ 305 | 306 | # Visual Studio 2015/2017 cache/options directory 307 | .vs/ 308 | # Uncomment if you have tasks that create the project's static files in wwwroot 309 | #wwwroot/ 310 | 311 | # Visual Studio 2017 auto generated files 312 | Generated\ Files/ 313 | 314 | # MSTest test Results 315 | [Tt]est[Rr]esult*/ 316 | [Bb]uild[Ll]og.* 317 | 318 | # NUnit 319 | *.VisualState.xml 320 | TestResult.xml 321 | nunit-*.xml 322 | 323 | # Build Results of an ATL Project 324 | [Dd]ebugPS/ 325 | [Rr]eleasePS/ 326 | dlldata.c 327 | 328 | # Benchmark Results 329 | BenchmarkDotNet.Artifacts/ 330 | 331 | # .NET Core 332 | project.lock.json 333 | project.fragment.lock.json 334 | artifacts/ 335 | 336 | # StyleCop 337 | StyleCopReport.xml 338 | 339 | # Files built by Visual Studio 340 | *_i.c 341 | *_p.c 342 | *_h.h 343 | *.ilk 344 | *.obj 345 | *.iobj 346 | *.pch 347 | *.pdb 348 | *.ipdb 349 | *.pgc 350 | *.pgd 351 | *.rsp 352 | *.sbr 353 | *.tlb 354 | *.tli 355 | *.tlh 356 | *.tmp 357 | *.tmp_proj 358 | *_wpftmp.csproj 359 | *.vspscc 360 | *.vssscc 361 | .builds 362 | *.pidb 363 | *.svclog 364 | *.scc 365 | 366 | # Chutzpah Test files 367 | _Chutzpah* 368 | 369 | # Visual C++ cache files 370 | ipch/ 371 | *.aps 372 | *.ncb 373 | *.opendb 374 | *.opensdf 375 | *.sdf 376 | *.cachefile 377 | *.VC.db 378 | *.VC.VC.opendb 379 | 380 | # Visual Studio profiler 381 | *.psess 382 | *.vsp 383 | *.vspx 384 | *.sap 385 | 386 | # Visual Studio Trace Files 387 | *.e2e 388 | 389 | # TFS 2012 Local Workspace 390 | $tf/ 391 | 392 | # Guidance Automation Toolkit 393 | *.gpState 394 | 395 | # ReSharper is a .NET coding add-in 396 | _ReSharper*/ 397 | *.[Rr]e[Ss]harper 398 | *.DotSettings.user 399 | 400 | # JustCode is a .NET coding add-in 401 | .JustCode 402 | 403 | # TeamCity is a build add-in 404 | _TeamCity* 405 | 406 | # DotCover is a Code Coverage Tool 407 | *.dotCover 408 | 409 | # AxoCover is a Code Coverage Tool 410 | .axoCover/* 411 | !.axoCover/settings.json 412 | 413 | # Visual Studio code coverage results 414 | *.coverage 415 | *.coveragexml 416 | 417 | # NCrunch 418 | _NCrunch_* 419 | .*crunch*.local.xml 420 | nCrunchTemp_* 421 | 422 | # MightyMoose 423 | *.mm.* 424 | AutoTest.Net/ 425 | 426 | # Web workbench (sass) 427 | .sass-cache/ 428 | 429 | # Installshield output folder 430 | [Ee]xpress/ 431 | 432 | # DocProject is a documentation generator add-in 433 | DocProject/buildhelp/ 434 | DocProject/Help/*.HxT 435 | DocProject/Help/*.HxC 436 | DocProject/Help/*.hhc 437 | DocProject/Help/*.hhk 438 | DocProject/Help/*.hhp 439 | DocProject/Help/Html2 440 | DocProject/Help/html 441 | 442 | # Click-Once directory 443 | publish/ 444 | 445 | # Publish Web Output 446 | *.[Pp]ublish.xml 447 | *.azurePubxml 448 | # Note: Comment the next line if you want to checkin your web deploy settings, 449 | # but database connection strings (with potential passwords) will be unencrypted 450 | *.pubxml 451 | *.publishproj 452 | 453 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 454 | # checkin your Azure Web App publish settings, but sensitive information contained 455 | # in these scripts will be unencrypted 456 | PublishScripts/ 457 | 458 | # NuGet Packages 459 | *.nupkg 460 | # NuGet Symbol Packages 461 | *.snupkg 462 | # The packages folder can be ignored because of Package Restore 463 | **/[Pp]ackages/* 464 | # except build/, which is used as an MSBuild target. 465 | !**/[Pp]ackages/build/ 466 | # Uncomment if necessary however generally it will be regenerated when needed 467 | #!**/[Pp]ackages/repositories.config 468 | # NuGet v3's project.json files produces more ignorable files 469 | *.nuget.props 470 | *.nuget.targets 471 | 472 | # Microsoft Azure Build Output 473 | csx/ 474 | *.build.csdef 475 | 476 | # Microsoft Azure Emulator 477 | ecf/ 478 | rcf/ 479 | 480 | # Windows Store app package directories and files 481 | AppPackages/ 482 | BundleArtifacts/ 483 | Package.StoreAssociation.xml 484 | _pkginfo.txt 485 | *.appx 486 | *.appxbundle 487 | *.appxupload 488 | package-lock.json 489 | 490 | # Visual Studio cache files 491 | # files ending in .cache can be ignored 492 | *.[Cc]ache 493 | # but keep track of directories ending in .cache 494 | !?*.[Cc]ache/ 495 | 496 | # Others 497 | ClientBin/ 498 | ~$* 499 | *.dbmdl 500 | *.dbproj.schemaview 501 | *.jfm 502 | *.pfx 503 | *.publishsettings 504 | orleans.codegen.cs 505 | 506 | # Including strong name files can present a security risk 507 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 508 | #*.snk 509 | 510 | # Since there are multiple workflows, uncomment next line to ignore bower_components 511 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 512 | #bower_components/ 513 | 514 | # RIA/Silverlight projects 515 | Generated_Code/ 516 | 517 | # Backup & report files from converting an old project file 518 | # to a newer Visual Studio version. Backup files are not needed, 519 | # because we have git ;-) 520 | _UpgradeReport_Files/ 521 | Backup*/ 522 | UpgradeLog*.XML 523 | UpgradeLog*.htm 524 | ServiceFabricBackup/ 525 | *.rptproj.bak 526 | 527 | # SQL Server files 528 | *.mdf 529 | *.ldf 530 | *.ndf 531 | 532 | # Business Intelligence projects 533 | *.rdl.data 534 | *.bim.layout 535 | *.bim_*.settings 536 | *.rptproj.rsuser 537 | *- [Bb]ackup.rdl 538 | *- [Bb]ackup ([0-9]).rdl 539 | *- [Bb]ackup ([0-9][0-9]).rdl 540 | 541 | # Microsoft Fakes 542 | FakesAssemblies/ 543 | 544 | # GhostDoc plugin setting file 545 | *.GhostDoc.xml 546 | 547 | # Node.js Tools for Visual Studio 548 | .ntvs_analysis.dat 549 | 550 | # Visual Studio 6 build log 551 | *.plg 552 | 553 | # Visual Studio 6 workspace options file 554 | *.opt 555 | 556 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 557 | *.vbw 558 | 559 | # Visual Studio LightSwitch build output 560 | **/*.HTMLClient/GeneratedArtifacts 561 | **/*.DesktopClient/GeneratedArtifacts 562 | **/*.DesktopClient/ModelManifest.xml 563 | **/*.Server/GeneratedArtifacts 564 | **/*.Server/ModelManifest.xml 565 | _Pvt_Extensions 566 | 567 | # Paket dependency manager 568 | .paket/paket.exe 569 | paket-files/ 570 | 571 | # FAKE - F# Make 572 | .fake/ 573 | 574 | # CodeRush personal settings 575 | .cr/personal 576 | 577 | # Python Tools for Visual Studio (PTVS) 578 | __pycache__/ 579 | *.pyc 580 | 581 | # Cake - Uncomment if you are using it 582 | # tools/** 583 | # !tools/packages.config 584 | 585 | # Tabs Studio 586 | *.tss 587 | 588 | # Telerik's JustMock configuration file 589 | *.jmconfig 590 | 591 | # BizTalk build output 592 | *.btp.cs 593 | *.btm.cs 594 | *.odx.cs 595 | *.xsd.cs 596 | 597 | # OpenCover UI analysis results 598 | OpenCover/ 599 | 600 | # Azure Stream Analytics local run output 601 | ASALocalRun/ 602 | 603 | # MSBuild Binary and Structured Log 604 | *.binlog 605 | 606 | # NVidia Nsight GPU debugger configuration file 607 | *.nvuser 608 | 609 | # MFractors (Xamarin productivity tool) working folder 610 | .mfractor/ 611 | 612 | # Local History for Visual Studio 613 | .localhistory/ 614 | 615 | # BeatPulse healthcheck temp database 616 | healthchecksdb 617 | 618 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 619 | MigrationBackup/ 620 | 621 | # End of https://www.gitignore.io/api/node,linux,macos,windows,visualstudio,jetbrains+all 622 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Read this first 3 | 4 | *** 5 | 6 | I have been away from the world of Glue so it is difficult for me to maintain this plugin since I do not have an AWS account to test and improve the plugin. So if someone wants to keep making updates to the repository, talk to me internally to add it as a maintainer so I can publish new versions. 7 | 8 | Regards from Chile! 9 | *** 10 | 11 | # Serverless Glue 12 | 13 | Serverless-glue is an open source MIT licensed project, which has been able to grow thanks to the community. This project is the result of an idea that did not let it rest in oblivion and many hours of work after hours. 14 | 15 | If you want to help me you can do it in the following ways: 16 | 17 | - With a donation through Paypal [here](https://paypal.me/toryas). 18 | - Sharing your feedback [here](https://github.com/toryas/serverless-glue/issues). 19 | 20 | I hope you liked this project and it is useful for you. 21 | 22 | Any problems? Join to the [slack channel](https://join.slack.com/t/serverless-glue/shared_invite/zt-14f50ztyb-Jdp8wRtVMlW7OriGBkG4oA). 23 | 24 | --- 25 | The principal changes are available [here](#changelog) 26 | --- 27 | 28 | This is a plugin for Serverless framework that provide the possibility to deploy AWS Glue Jobs and Triggers 29 | 30 | ## Install 31 | 32 | 1. run `npm install --save-dev serverless-glue` 33 | 2. add **serverless-glue** in serverless.yml plugin section 34 | ```yml 35 | plugins: 36 | - serverless-glue 37 | ``` 38 | ## How it works 39 | 40 | The plugin creates CloufFormation resources of your configuration before making the serverless deploy then add it to the serverless template. 41 | 42 | So any glue-job deployed with this plugin is part of your stack too. 43 | 44 | ## How to configure your GlueJob(s) 45 | 46 | Configure your glue jobs in the root of servelress.yml like this: 47 | 48 | ```yml 49 | Glue: 50 | bucketDeploy: someBucket # Required 51 | createBucket: true # Optional, default = false 52 | createBucketConfig: # Optional 53 | ACL: private # Optional, private | public-read | public-read-write | authenticated-read 54 | LocationConstraint: af-south-1 55 | GrantFullControl: 'STRING_VALUE' # Optional 56 | GrantRead: 'STRING_VALUE' # Optional 57 | GrantReadACP: 'STRING_VALUE' # Optional 58 | GrantWrite: 'STRING_VALUE' # Optional 59 | GrantWriteACP: 'STRING_VALUE' # Optional 60 | ObjectLockEnabledForBucket: true # Optional 61 | ObjectOwnership: BucketOwnerPreferred # Optional 62 | s3Prefix: some/s3/key/location/ # optional, default = 'glueJobs/' 63 | tempDirBucket: someBucket # optional, default = '{serverless.serviceName}-{provider.stage}-gluejobstemp' 64 | tempDirS3Prefix: some/s3/key/location/ # optional, default = ''. The job name will be appended to the prefix name 65 | jobs: 66 | - name: super-glue-job # Required 67 | id: # Optional, string 68 | scriptPath: src/script.py # Required script will be named with the name after '/' and uploaded to s3Prefix location 69 | Description: # Optional, string 70 | tempDir: true # Optional true | false 71 | type: spark # spark / spark_streaming / pythonshell # Required 72 | glueVersion: python3-2.0 # Required "python3.9-1.0" | "python3.9-2.0" | "python3.9-3.0" | "python3-1.0" | "python3-2.0" | "python3-3.0" | "python2-1.0" | "python2-0.9" | "scala2-1.0" | "scala2-0.9" | "scala2-2.0" | "scala3-3.0" 73 | role: arn:aws:iam::000000000:role/someRole # Required 74 | MaxCapacity: 1 #Optional 75 | MaxConcurrentRuns: 3 # Optional 76 | WorkerType: Standard # Optional, G.1X | G.2X 77 | NumberOfWorkers: 1 # Optional 78 | SecurityConfiguration: # Optional, name of security configuration 79 | Connections: # Optional 80 | - some-conection-string 81 | - other-conection-string 82 | Timeout: # Optional, number 83 | MaxRetries: # Optional, number 84 | DefaultArguments: # Optional 85 | class: string # Optional 86 | scriptLocation: string # Optional 87 | extraPyFiles: string # Optional 88 | extraJars: string # Optional 89 | userJarsFirst: string # Optional 90 | usePostgresDriver: string # Optional 91 | extraFiles: string # Optional 92 | disableProxy: string # Optional 93 | jobBookmarkOption: string # Optional 94 | enableAutoScaling: string # Optional 95 | enableS3ParquetOptimizedCommitter: string # Optional 96 | enableRenameAlgorithmV2: string # Optional 97 | enableGlueDatacatalog: string # Optional 98 | enableMetrics: string # Optional 99 | enableContinuousCloudwatchLog: string # Optional 100 | enableContinuousLogFilter: string # Optional 101 | continuousLogLogGroup: string # Optional 102 | continuousLogLogStreamPrefix: string # Optional 103 | continuousLogConversionPattern: string # Optional 104 | enableSparkUi: string # Optional 105 | sparkEventLogsPath: string # Optional 106 | additionalPythonModules: string # Optional 107 | customArguments: # Optional; these are user-specified custom default arguments that are passed into cloudformation with a leading -- (required for glue) 108 | custom_arg_1: custom_value 109 | custom_arg_2: other_custom_value 110 | SupportFiles: # Optional 111 | - local_path: path/to/file/or/folder/ # Required if SupportFiles is given, you can pass a folder path or a file path 112 | s3_bucket: bucket-name-where-to-upload-files # Required if SupportFiles is given 113 | s3_prefix: some/s3/key/location/ # Required if SupportFiles is given 114 | execute_upload: True # Boolean, True to execute upload, False to not upload. Required if SupportFiles is given 115 | Tags: 116 | job_tag_example_1: example1 117 | job_tag_example_2: example2 118 | triggers: 119 | - name: some-trigger-name # Required 120 | Description: # Optional, string 121 | StartOnCreation: True # Optional, True or False 122 | schedule: 30 12 * * ? * # Optional, CRON expression. The trigger will be created with On-Demand type if the schedule is not provided. 123 | Tags: 124 | trigger_tag_example_1: example1 125 | actions: # Required. One or more jobs to trigger 126 | - name: super-glue-job # Required 127 | args: # Optional 128 | custom_arg_1: custom_value 129 | custom_arg_2: other_custom_value 130 | timeout: 30 # Optional, if set, it overwrites specific jobs timeout when job starts via trigger 131 | SecurityConfiguration: # Optional, name of security configuration 132 | 133 | ``` 134 | 135 | You can define a lot of jobs... 136 | 137 | ```yml 138 | Glue: 139 | bucketDeploy: someBucket 140 | jobs: 141 | - name: jobA 142 | scriptPath: scriptA 143 | ... 144 | - name: jobB 145 | scriptPath: scriptB 146 | ... 147 | 148 | ``` 149 | 150 | And a lot of triggers... 151 | 152 | ```yml 153 | Glue: 154 | triggers: 155 | - name: 156 | ... 157 | - name: 158 | ... 159 | 160 | ``` 161 | 162 | ### Glue configuration parameters 163 | 164 | |Parameter|Type|Description|Required| 165 | |-|-|-|-| 166 | |bucketDeploy|String|S3 Bucket name|true| 167 | |createBucket|Boolean|If true, a bucket named as `bucketDeploy` will be created before. Helpful if you have not created the bucket first|false| 168 | createBucketConfig|createBucketConfig| Bucket configuration for creation on S3 |false| 169 | |s3Prefix|String|S3 prefix name|false| 170 | |tempDirBucket|String|S3 Bucket name for Glue temporary directory. If dont pass argument the bucket'name will generates with pattern {serverless.serviceName}-{provider.stage}-gluejobstemp|false| 171 | |tempDirS3Prefix|String|S3 prefix name for Glue temporary directory|false| 172 | |jobs|Array|Array of glue jobs to deploy|true| 173 | 174 | ### CreateBucket confoguration parameters 175 | 176 | |Parameter|Type|Description|Required| 177 | |-|-|-|-| 178 | |ACL|String|The canned ACL to apply to the bucket. Possible values include:|False| 179 | |LocationConstraint|String| Specifies the Region where the bucket will be created. If you don't specify a Region, the bucket is created in the US East (N. Virginia) Region (us-east-1). Possible values are: |false| 180 | |GrantFullControl|String|Allows grantee the read, write, read ACP, and write ACP permissions on the bucket.|false| 181 | |GrantRead|(String|Allows grantee to list the objects in the bucket.|false| 182 | |GrantReadACP|String|Allows grantee to read the bucket ACL.|false| 183 | |GrantWrite|String|Allows grantee to create new objects in the bucket. For the bucket and object owners of existing objects, also allows deletions and overwrites of those objects.|false| 184 | |GrantWriteACP|String|Allows grantee to write the ACL for the applicable bucket.|false| 185 | |ObjectLockEnabledForBucket|Boolean|Specifies whether you want S3 Object Lock to be enabled for the new bucket.|false| 186 | |ObjectOwnership|String|The container element for object ownership for a bucket's ownership controls.Possible values include:|false| 187 | 188 | ### Jobs configurations parameters 189 | 190 | |Parameter|Type|Description|Required| 191 | |-|-|-|-| 192 | |name|String|name of job|true| 193 | |id|String|logical ID in CloudFormation for the job|false| 194 | |Description|String|Description of the job|False| 195 | |scriptPath|String|script path in the project|true| 196 | |tempDir|Boolean|flag indicate if job required a temp folder, if true plugin create a bucket for tmp|false| 197 | |type|String|Indicate if the type of your job. Values can use are : `spark`, `spark_streaming` or `pythonshell`|true| 198 | |glueVersion|String|Indicate language and glue version to use ( `[language][version]-[glue version]`) the value can you use are: |true| 199 | |role|String| arn role to execute job|true| 200 | |MaxCapacity|Double| The number of AWS Glue data processing units (DPUs) that can be allocated when this job runs|false| 201 | |MaxConcurrentRuns|Double|max concurrent runs of the job|false| 202 | |MaxRetries|Int|Maximum number of retires in case of failure|False| 203 | |Timeout|Int|Job timeout in number of minutes|False| 204 | |WorkerType|String|The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, or G.2X.|false| 205 | |NumberOfWorkers|Integer|number of workers|false| 206 | |SecurityConfiguration|String|The name of the security configuration that the job should use|false| 207 | |Connections|List|a list of connections used by the job|false| 208 | |DefaultArguments|object|Special Parameters Used by AWS Glue for mor information see this read the [AWS documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html)|false| 209 | |SupportFiles|List|List of supporting files for the glue job that need upload to S3|false| 210 | |Tags|JSON|The tags to use with this job. You may use tags to limit access to the job. For more information about tags in AWS Glue, see AWS Tags in AWS Glue in the developer guide.|false| 211 | 212 | ### Triggers configuration parameters 213 | 214 | |Parameter|Type|Description|Required| 215 | |-|-|-|-| 216 | |name|String|name of the trigger|true| 217 | |schedule|String|CRON expression|false| 218 | |actions|Array|An array of jobs to trigger|true| 219 | |Description|String|Description of the Trigger|False| 220 | |StartOnCreation|Boolean|Whether the trigger starts when created. Not supperted for ON_DEMAND triggers|False| 221 | 222 | Only On-Demand and Scheduled triggers are supported. 223 | 224 | ### Trigger job configuration parameters 225 | 226 | |Parameter|Type|Description|Required| 227 | |-|-|-|-| 228 | |name|String|The name of the Glue job to trigger|true| 229 | |timeout|Integer|Job execution timeout. It overwrites|false| 230 | |args|Map|job arguments|false| 231 | |Tags|JSON|The tags to use with this triggers. For more information about tags in AWS Glue, see AWS Tags in AWS Glue in the developer guide.|false| 232 | 233 | 234 | ## And now?... 235 | 236 | Only run `serverless deploy` 237 | --- 238 | 239 | # Changelog 240 | 241 | 242 | ## [2.14.0] - 2025-02-13 243 | 244 | ### Add 245 | - included job queuing feature [#59](https://github.com/toryas/serverless-glue/pull/59) 246 | 247 | ## [2.13.0] - 2024-12-09 248 | 249 | ### Add 250 | - allow glue v4.0 [#58](https://github.com/toryas/serverless-glue/pull/58) 251 | 252 | ## [2.12.1] - 2023-05-31 253 | 254 | ### Fix 255 | - Fixed tempDir being added when set to false 256 | 257 | ## [2.12.0] - 2023-02-09 258 | 259 | ### Add 260 | - Add support for spark streams 261 | - Add support for `--additional-python-modules` 262 | 263 | ### Fix 264 | 265 | - fix when parametter `s3Prefix` is omitted generate a undefine prefix 266 | 267 | ## [2.11.1] - 2022-09-13 268 | 269 | ### Add 270 | - Add support for custom logical IDs for jobs 271 | 272 | ### Fix 273 | - Fix Pascal Case generation for sections of names that are only numeric 274 | 275 | ## [2.10.0] - 2022-09-12 276 | 277 | ### Add 278 | - Add support for python 3.9 shell jobs 279 | 280 | ## [2.9.0] - 2022-06-03 281 | 282 | ### Add 283 | 284 | - Add support to Glue 3.0 (Spark 3.1.1/Python 3.7) 285 | - Now aws-s3 client is generated with region defined on "provider" part of serverless.yml 286 | 287 | ### Fix 288 | 289 | - the hard coded path generator is replaced by the "path" package, to solve problems when running the package on Windows 290 | - the last "/" characters on `tempDirS3Prefix` are automatically removed to avoid wrong paths in S3 291 | 292 | ## [2.8.0] - 2022-03-31 293 | 294 | ### Add 295 | 296 | - Add check if bucket exist before create it 297 | ## [2.7.0] - 2022-02-25 298 | 299 | ### Add 300 | - Add configuration MaxCapacity for job 301 | ## [2.6.0] - 2022-02-25 302 | 303 | ### Add 304 | - Add support for SecurityConfiguration property 305 | 306 | ## [2.5.0] - 2022-02-14 307 | 308 | ### Add 309 | - Add the `createBucketConfig` feature to set the bucket creation configuration. 310 | 311 | ### Changed 312 | - Removed message when support files not found, now logging message when support files exist. 313 | ### Fix 314 | - Improve the `createBucket` example of documentation. 315 | 316 | ## [2.4.1] - 2022-02-01 317 | ### Fix 318 | - Fix schema typo that blocks serverless 3. 319 | ## [2.4.0] - 2022-01-17 320 | ### Fix 321 | - Fix NumberOwfWorkers typo. 322 | 323 | ### Add 324 | - Added `Timeout`, `MaxRetries` and `Description` parameters to Glue Job arguments. Added `Description` and `StartOnCreation` parameters to Glue Job Trigger arguments. 325 | - Added `SupportFiles` to Glue Job arguments handling the upload to S3 of relevant-to-the-Glue-Job(s) files 326 | 327 | 328 | ## [2.3.0] - 2021-12-23 329 | 330 | ### Add 331 | - Implement Custom Arguments for Jobs 332 | ## [2.2.0] - 2021-12-22 333 | 334 | ### Add 335 | - Implement Tags for jobs and triggers 336 | ## [2.1.1] - 2021-12-21 337 | 338 | ### Fixed 339 | - Remove empty connections object from CF template when don`t specify any conection 340 | ## [2.1.0] - 2021-12-15 341 | 342 | ### Add 343 | - Implement DefaultArguments for jobs 344 | ## [2.0.2] - 2021-12-13 345 | 346 | ### Fixed 347 | - Replace incorrect async loop in serverless 348 | 349 | ## [2.0.1] - 2021-12-09 350 | 351 | ### Changed 352 | - Move typescript dependencie to dev 353 | ## [2.0.0] - 2021-11-29 354 | ### Changed 355 | - Refactoring code from JS to TS, and restructured folders. 356 | - Plugin`s configuration get out from *custom* level in serverless.yml now are in root of file. 357 | - Remove redundant level *job* in jobs config. 358 | - **script** attribute are rename to ***scriptPath** 359 | - Remove redundant level *Connections* in **Connections** config. 360 | - Remove redundant level trigger from triggers config. 361 | - Rename **job** attribute to **action** in trigger config. 362 | ### Fixed 363 | - Improve documentation for **Connections** config. 364 | -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | const { task, src, series, dest } = require('gulp'); 2 | const file = require('gulp-file'); 3 | const pkg = require('./package.json'); 4 | const del = require('del') 5 | 6 | 7 | task('clean', async () => { 8 | return del.sync(['lib']) 9 | }) 10 | 11 | task('copyToDist', async () => { 12 | // await src('./src/module/**/*').pipe(dest('./dist/')) 13 | await src('./README.md').pipe(dest('./lib/')) 14 | }) 15 | 16 | task('makePackageJson', async () => { 17 | let distPkg = { 18 | name:pkg.name, 19 | version:pkg.version, 20 | } 21 | distPkg.dependencies = pkg.dependencies 22 | distPkg.description = pkg.description; 23 | distPkg.main = pkg.main; 24 | distPkg.keywords = pkg.keywords; 25 | distPkg.license = pkg.license; 26 | distPkg.bugs = pkg.bugs; 27 | distPkg.homepage = pkg.homepage; 28 | distPkg.repository = pkg.repository; 29 | distPkg.author = pkg.author; 30 | 31 | await file('package.json', JSON.stringify(distPkg, null, 2), { src: true }) 32 | .pipe(dest('lib')); 33 | 34 | 35 | }) 36 | 37 | task('build', series( 38 | // 'clean', 39 | 'copyToDist', 40 | 'makePackageJson' 41 | )); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverless-glue", 3 | "version": "2.14.0", 4 | "description": "Serverless plugin to deploy Glue Jobs", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "build": "tsc", 9 | "bp": "npm run gulp && npm pack lib/ ", 10 | "gulp": "gulp clean && npm run build && gulp build", 11 | "publishme": "npm run gulp && npm publish lib/" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/toryas/serverless-glue.git" 16 | }, 17 | "keywords": [ 18 | "serverless", 19 | "aws", 20 | "glue", 21 | "gluejob", 22 | "trigger", 23 | "triggers" 24 | ], 25 | "author": { 26 | "name": "Felipe Nuñez", 27 | "email": "toutox@gmail.com", 28 | "url": "https://github.com/toryas" 29 | }, 30 | "license": "MIT", 31 | "bugs": { 32 | "url": "https://github.com/toryas/serverless-glue/issues" 33 | }, 34 | "homepage": "https://github.com/toryas/serverless-glue#readme", 35 | "devDependencies": { 36 | "@types/node": "^17.0.8", 37 | "del": "^5.1.0", 38 | "gulp": "^4.0.2", 39 | "gulp-file": "^0.4.0", 40 | "typescript": "^4.5.4" 41 | }, 42 | "dependencies": { 43 | "aws-sdk": "^2.701.0" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/constants/global.constant.ts: -------------------------------------------------------------------------------- 1 | export const Global = { 2 | GLUE_TEMP_BUCKET_REF:'GlueJobTempBucket' 3 | } -------------------------------------------------------------------------------- /src/domain/default-arguments.ts: -------------------------------------------------------------------------------- 1 | import { DefaultArgumentsInterface } from "../interfaces/default-arguments.interface"; 2 | 3 | export class DefaultArguments implements DefaultArgumentsInterface { 4 | additionalPythonModules?: string | undefined; 5 | jobLanguage?: string | undefined; 6 | class?: string | undefined; 7 | scriptLocation?: string | undefined; 8 | extraPyFiles?: string | undefined; 9 | extraJars?: string | undefined; 10 | userJarsFirst?: string | undefined; 11 | usePostgresDriver?: string | undefined; 12 | extraFiles?: string | undefined; 13 | disableProxy?: string | undefined; 14 | jobBookmarkOption?: string | undefined; 15 | enableAutoScaling?: string | undefined; 16 | enableS3ParquetOptimizedCommitter?: string | undefined; 17 | enableRenameAlgorithmV2?: string | undefined; 18 | enableGlueDatacatalog?: string | undefined; 19 | enableMetrics?: string | undefined; 20 | enableContinuousCloudwatchLog?: string | undefined; 21 | enableContinuousLogFilter?: string | undefined; 22 | continuousLogLogGroup?: string | undefined; 23 | continuousLogLogStreamPrefix?: string | undefined; 24 | continuousLogConversionPattern?: string | undefined; 25 | enableSparkUi?: string | undefined; 26 | sparkEventLogsPath?: string | undefined; 27 | tempDir?: any; 28 | customArguments?: Map | undefined; 29 | librarySet: string | undefined; 30 | } 31 | -------------------------------------------------------------------------------- /src/domain/glue-job.ts: -------------------------------------------------------------------------------- 1 | import { DefaultArgumentsInterface } from "../interfaces/default-arguments.interface"; 2 | import { GlueJobInterface } from "../interfaces/glue-job.interface"; 3 | import { SupportFilesInterface } from "../interfaces/support-files.interface"; 4 | 5 | export const GlueVersions = [ 6 | "python3.9-1.0", 7 | "python3.9-2.0", 8 | "python3.9-3.0", 9 | "python3-4.0", 10 | "python3-1.0", 11 | "python3-2.0", 12 | "python3-3.0", 13 | "python2-1.0", 14 | "python2-0.9", 15 | "scala2-1.0", 16 | "scala2-0.9", 17 | "scala2-2.0", 18 | "scala3-3.0" 19 | ] as const; 20 | 21 | export type GlueVersionType = typeof GlueVersions[number]; 22 | 23 | export class GlueJob implements GlueJobInterface { 24 | name: string; 25 | id?: string; 26 | scriptPath: string; 27 | tempDir?: boolean; 28 | type: "spark" | "spark_streaming" | "pythonshell"; 29 | glueVersion: GlueVersionType; 30 | Description: string; 31 | role: string; 32 | MaxCapacity?: number; 33 | MaxConcurrentRuns?: number; 34 | WorkerType?: "G.1X" | "G.2X" | "Standard" | undefined; 35 | NumberOfWorkers?: number | undefined; 36 | Connections?: string[] | undefined; 37 | scriptS3Location?: string; 38 | commandName?: "glueetl" | "gluestreaming" | "pythonshell"; 39 | pythonVersion?: string; 40 | glueVersionJob?: string; 41 | DefaultArguments: DefaultArgumentsInterface; 42 | Tags?: Map; 43 | Timeout: number; 44 | MaxRetries: number; 45 | SupportFiles: SupportFilesInterface[]; 46 | SecurityConfiguration?: string; 47 | JobRunQueuingEnabled?: boolean; 48 | 49 | 50 | constructor(job: GlueJobInterface) { 51 | this.DefaultArguments = job.DefaultArguments ?? {}; 52 | this.name = job.name; 53 | this.id = job.id; 54 | this.scriptPath = job.scriptPath; 55 | this.role = job.role; 56 | this.glueVersion = job.glueVersion; 57 | this.Description = job.Description; 58 | this.type = job.type; 59 | this.MaxCapacity = job.MaxCapacity; 60 | this.MaxConcurrentRuns = job.MaxConcurrentRuns; 61 | this.MaxRetries = job.MaxRetries; 62 | this.WorkerType = job.WorkerType; 63 | this.NumberOfWorkers = job.NumberOfWorkers; 64 | this.Connections = job.Connections; 65 | this.defineCommandName(job.type); 66 | this.setGlueVersion(this.glueVersion); 67 | this.Tags = job.Tags; 68 | this.Timeout = job.Timeout; 69 | this.MaxRetries = job.MaxRetries; 70 | this.SupportFiles = job.SupportFiles; 71 | this.SecurityConfiguration = job.SecurityConfiguration; 72 | this.JobRunQueuingEnabled = job.JobRunQueuingEnabled; 73 | } 74 | 75 | setScriptS3Location(s3url: string) { 76 | this.scriptS3Location = s3url; 77 | } 78 | 79 | defineCommandName(type: "spark" | "spark_streaming" | "pythonshell") { 80 | switch (type) { 81 | case "spark": 82 | this.commandName = "glueetl"; 83 | break; 84 | case "spark_streaming": 85 | this.commandName = "gluestreaming"; 86 | break; 87 | case "pythonshell": 88 | this.commandName = "pythonshell"; 89 | break; 90 | } 91 | } 92 | 93 | setGlueVersion(glueVersion: GlueVersionType) { 94 | let parts = glueVersion.split("-"); 95 | let pythonVersion = parts[0].replace(/[A-Za-z]*/, ''); 96 | let language = parts[0].match(/[A-Za-z]*/)?.toString(); 97 | this.pythonVersion = pythonVersion; 98 | this.glueVersionJob = parts[1]; 99 | this.DefaultArguments.jobLanguage = language; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/domain/glue-trigger.ts: -------------------------------------------------------------------------------- 1 | import { GlueTriggerActionInterface} from "../interfaces/glue-trigger-action.interface"; 2 | import { GlueTriggerInterface } from "../interfaces/glue-trigger.interface"; 3 | 4 | export class GlueTrigger implements GlueTriggerInterface{ 5 | 6 | name: string; 7 | schedule?: string; 8 | type?:string; 9 | actions: GlueTriggerActionInterface[]; 10 | Description: string; 11 | Tags?: Map; 12 | StartOnCreation: boolean; 13 | 14 | constructor(trigger:GlueTriggerInterface){ 15 | this.name = trigger.name; 16 | this.actions = trigger.actions; 17 | this.Description = trigger.Description 18 | this.setSchedule(trigger.schedule); 19 | this.Tags = trigger.Tags; 20 | this.StartOnCreation = trigger.StartOnCreation; 21 | } 22 | 23 | setSchedule(cronSchedule:string|undefined) { 24 | if (cronSchedule) { 25 | this.type = "SCHEDULED" 26 | this.schedule = `cron(${cronSchedule})`; 27 | } else { 28 | this.type = "ON_DEMAND" 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/domain/support-files.ts: -------------------------------------------------------------------------------- 1 | import { SupportFilesInterface } from "../interfaces/support-files.interface"; 2 | 3 | export class SupportFile implements SupportFilesInterface{ 4 | 5 | local_path: string; 6 | s3_bucket: string; 7 | s3_prefix: string; 8 | execute_upload: boolean; 9 | 10 | constructor(SupportFiles:SupportFilesInterface){ 11 | this.local_path = SupportFiles.local_path; 12 | this.s3_bucket = SupportFiles.s3_bucket; 13 | this.s3_prefix = SupportFiles.s3_prefix; 14 | this.execute_upload = SupportFiles.execute_upload; 15 | } 16 | 17 | } -------------------------------------------------------------------------------- /src/helpers/aws.helper.ts: -------------------------------------------------------------------------------- 1 | import * as AWS from "aws-sdk"; 2 | 3 | export class AwsHelper { 4 | provider: any; 5 | s3: AWS.S3; 6 | 7 | constructor(serverless: any) { 8 | this.provider = serverless.getProvider("aws"); 9 | this.s3 = AwsHelper.makeS3service(this.provider); 10 | } 11 | 12 | 13 | /** 14 | * Return a new S3 service 15 | * @param {*} credentials AWS Credentials 16 | */ 17 | static makeS3service(provider: any) { 18 | const credentials = provider.getCredentials().credentials; 19 | const region = provider.options.region; 20 | return new AWS.S3({ credentials: credentials, region: region }); 21 | } 22 | 23 | /** 24 | * create a S3 Bucket in AWS 25 | * @param options 26 | */ 27 | async createBucket(options: AWS.S3.CreateBucketRequest) { 28 | await this.s3.createBucket(options).promise(); 29 | } 30 | 31 | /** 32 | * Upload file to bucket 33 | * @param options 34 | */ 35 | async uploadFileToS3(options: AWS.S3.PutObjectRequest) { 36 | if (!process.env.SLSGLUE_SKIP_UPLOADS) { 37 | await this.s3.upload(options).promise(); 38 | } 39 | } 40 | 41 | existBucket(options: AWS.S3.CreateBucketRequest) { 42 | return new Promise((resolve, _rejects) => { 43 | this.s3.headBucket({ Bucket: options.Bucket }, (err, data) => { 44 | if (err) { 45 | resolve(false); 46 | } else { 47 | resolve(true); 48 | } 49 | }); 50 | }); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/helpers/glue.helper.ts: -------------------------------------------------------------------------------- 1 | import { GlueJob } from "../domain/glue-job"; 2 | import { GlueTrigger } from "../domain/glue-trigger"; 3 | import { GluePluginConfigInterface } from "../interfaces/glue-plugin-config.interce"; 4 | import { Global } from "../constants/global.constant"; 5 | import { SupportFile } from "../domain/support-files"; 6 | 7 | export class GlueHelper { 8 | constructor(private config: GluePluginConfigInterface) {} 9 | 10 | /** 11 | * Return an array with al GlueJobs config in serverless yml file 12 | */ 13 | getGlueJobs() { 14 | let jobs: GlueJob[] = []; 15 | if(!this.config.jobs){ 16 | return jobs 17 | } 18 | for (let job of this.config.jobs) { 19 | let glueJob = new GlueJob(job); 20 | if (job.tempDir) { 21 | this.setTempBucketForJob(glueJob); 22 | } 23 | jobs.push(glueJob); 24 | } 25 | return jobs; 26 | } 27 | 28 | setTempBucketForJob(glueJob: GlueJob) { 29 | const jobTempDirBucket = this.config.tempDirBucket ?? { 30 | Ref: Global.GLUE_TEMP_BUCKET_REF, 31 | }; 32 | let jobTempDirS3Prefix = ""; 33 | if (this.config.tempDirS3Prefix) { 34 | this.config.tempDirS3Prefix = this.config.tempDirS3Prefix.split("/").filter(a=>a!="").join("/"); 35 | jobTempDirS3Prefix += `/${this.config.tempDirS3Prefix}`; 36 | } 37 | jobTempDirS3Prefix += `/${glueJob.name}`; 38 | 39 | glueJob.DefaultArguments.tempDir = { 40 | "Fn::Join": ["", ["s3://", jobTempDirBucket, jobTempDirS3Prefix]], 41 | }; 42 | } 43 | 44 | /** 45 | * Get GlueJobTriggers configured in serverless.yml 46 | * @param {Object} config plugin config 47 | */ 48 | getGlueTriggers() { 49 | let triggers:GlueTrigger[] = []; 50 | if(!this.config.triggers){ 51 | return triggers 52 | } 53 | for (let trigger of this.config.triggers) { 54 | let glueTrigger = new GlueTrigger(trigger); 55 | triggers.push(glueTrigger); 56 | } 57 | return triggers; 58 | } 59 | 60 | getSupportFiles(job: GlueJob) { 61 | let supportFiles:SupportFile[] = []; 62 | if(!job.SupportFiles){ 63 | return supportFiles 64 | } 65 | for (let supportFile of job.SupportFiles) { 66 | let glueSupportFile = new SupportFile(supportFile) 67 | supportFiles.push(glueSupportFile) 68 | } 69 | return supportFiles 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/helpers/serverless.helper.ts: -------------------------------------------------------------------------------- 1 | import { GluePluginConfigInterface } from "../interfaces/glue-plugin-config.interce"; 2 | 3 | export class ServerlessHelper { 4 | resources: any; 5 | output: any; 6 | constructor(private serverless: any) { 7 | this.resources = 8 | this.serverless.service.provider.compiledCloudFormationTemplate.Resources; 9 | this.output = 10 | this.serverless.service.provider.compiledCloudFormationTemplate.Outputs; 11 | } 12 | 13 | getPluginConfig(): GluePluginConfigInterface { 14 | return this.serverless.configSchemaHandler.serverless.configurationInput 15 | .Glue; 16 | } 17 | 18 | appendToTemplate( 19 | node: "resources" | "outputs", 20 | elementName: string, 21 | cfElement: any 22 | ) { 23 | switch (node) { 24 | case "resources": 25 | this.resources[elementName] = cfElement; 26 | break; 27 | case "outputs": 28 | this.output[elementName] = cfElement; 29 | break; 30 | } 31 | } 32 | 33 | log(message: string) { 34 | this.serverless.cli.log(`[Serverless-Glue]: ${message}`); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { ServerlessService } from "./services/serverless.service"; 2 | import { GlueSchema } from "./schemas/glue.schema"; 3 | 4 | class GluePlugin { 5 | serverless: any; 6 | options: any; 7 | hooks: any; 8 | 9 | constructor(serverless: any, options: any) { 10 | this.serverless = serverless; 11 | this.options = options; 12 | 13 | this.configGlueSchema(); 14 | 15 | this.hooks = { 16 | "aws:package:finalize:mergeCustomProviderResources": 17 | this.deploy.bind(this), 18 | }; 19 | } 20 | 21 | async deploy() { 22 | const service = new ServerlessService(this.serverless); 23 | await service.main(); 24 | } 25 | 26 | configGlueSchema() { 27 | if (this.serverless.configSchemaHandler) { 28 | this.serverless.configSchemaHandler.defineTopLevelProperty( 29 | "Glue", 30 | GlueSchema 31 | ); 32 | } 33 | } 34 | } 35 | 36 | module.exports = GluePlugin; 37 | -------------------------------------------------------------------------------- /src/interfaces/bucket-options.interface.ts: -------------------------------------------------------------------------------- 1 | export interface BucketOptions { 2 | Bucket: string; 3 | Key?: string; 4 | Body?: any; 5 | } 6 | -------------------------------------------------------------------------------- /src/interfaces/create-bucket-config.interface.ts: -------------------------------------------------------------------------------- 1 | export interface CreateBucketConfigInterface { 2 | ACL?: string; 3 | CreateBucketConfiguration?: LocationConstraintInterface; 4 | GrantFullControl?: string; 5 | GrantRead?: string; 6 | GrantReadACP?: string; 7 | GrantWrite?: string; 8 | GrantWriteACP?: string; 9 | ObjectLockEnabledForBucket?: boolean; 10 | ObjectOwnership?: string; 11 | LocationConstraint?: string; 12 | } 13 | 14 | interface LocationConstraintInterface { 15 | LocationConstraint: string; 16 | } 17 | -------------------------------------------------------------------------------- /src/interfaces/default-arguments.interface.ts: -------------------------------------------------------------------------------- 1 | export interface DefaultArgumentsInterface { 2 | additionalPythonModules?: string; 3 | jobLanguage?: string; 4 | tempDir?: any; 5 | class?: string; 6 | scriptLocation?: string; 7 | extraPyFiles?: string; 8 | extraJars?: string; 9 | userJarsFirst?: string; 10 | usePostgresDriver?: string; 11 | extraFiles?: string; 12 | disableProxy?: string; 13 | jobBookmarkOption?: string; 14 | enableAutoScaling?: string; 15 | enableS3ParquetOptimizedCommitter?: string; 16 | enableRenameAlgorithmV2?: string; 17 | enableGlueDatacatalog?: string; 18 | enableMetrics?: string; 19 | enableContinuousCloudwatchLog?: string; 20 | enableContinuousLogFilter?: string; 21 | continuousLogLogGroup?: string; 22 | continuousLogLogStreamPrefix?: string; 23 | continuousLogConversionPattern?: string; 24 | enableSparkUi?: string; 25 | sparkEventLogsPath?: string; 26 | customArguments?: Map; 27 | librarySet: string | undefined; 28 | } 29 | -------------------------------------------------------------------------------- /src/interfaces/glue-job.interface.ts: -------------------------------------------------------------------------------- 1 | import { DefaultArgumentsInterface } from "./default-arguments.interface"; 2 | import { SupportFilesInterface } from "./support-files.interface"; 3 | import { GlueVersionType } from "../domain/glue-job"; 4 | 5 | export interface GlueJobInterface { 6 | name: string; 7 | id?: string; 8 | scriptPath: string; 9 | tempDir?: boolean; 10 | type: "spark" | "spark_streaming" | "pythonshell"; 11 | glueVersion: GlueVersionType 12 | Description: string; 13 | role: string; 14 | MaxCapacity?: number; 15 | MaxConcurrentRuns?: number; 16 | WorkerType?: "G.1X" | "G.2X" | "Standard"; 17 | NumberOfWorkers?: number; 18 | Connections?: string[]; 19 | scriptS3Location?: string; 20 | commandName?: "glueetl" | "gluestreaming" | "pythonshell"; 21 | DefaultArguments: DefaultArgumentsInterface; 22 | Tags?: Map; 23 | Timeout: number; 24 | MaxRetries: number; 25 | SupportFiles: SupportFilesInterface[]; 26 | SecurityConfiguration?: string; 27 | JobRunQueuingEnabled?: boolean; 28 | } 29 | -------------------------------------------------------------------------------- /src/interfaces/glue-plugin-config.interce.ts: -------------------------------------------------------------------------------- 1 | import { CreateBucketConfigInterface } from "./create-bucket-config.interface"; 2 | import { GlueJobInterface } from "./glue-job.interface"; 3 | import { GlueTriggerInterface } from "./glue-trigger.interface"; 4 | 5 | export interface GluePluginConfigInterface { 6 | bucketDeploy: string; 7 | s3Prefix?: string; 8 | tempDirBucket?: string; 9 | tempDirS3Prefix?: string; 10 | createBucket?:boolean; 11 | createBucketConfig?: CreateBucketConfigInterface; 12 | jobs?: GlueJobInterface[]; 13 | triggers?:GlueTriggerInterface[]; 14 | } 15 | 16 | -------------------------------------------------------------------------------- /src/interfaces/glue-trigger-action.interface.ts: -------------------------------------------------------------------------------- 1 | export interface GlueTriggerActionInterface { 2 | name:string; 3 | args?: {[k:string]:string}; 4 | timeout?: number; 5 | SecurityConfiguration?: string; 6 | } -------------------------------------------------------------------------------- /src/interfaces/glue-trigger.interface.ts: -------------------------------------------------------------------------------- 1 | import { GlueTriggerActionInterface } from "./glue-trigger-action.interface" 2 | 3 | export interface GlueTriggerInterface { 4 | name: string; 5 | schedule?: string; 6 | actions: GlueTriggerActionInterface[]; 7 | Description: string; 8 | Tags?: Map; 9 | StartOnCreation: boolean; 10 | } 11 | 12 | -------------------------------------------------------------------------------- /src/interfaces/support-files.interface.ts: -------------------------------------------------------------------------------- 1 | export interface SupportFilesInterface { 2 | local_path: string; 3 | s3_bucket: string; 4 | s3_prefix: string; 5 | execute_upload: boolean; 6 | } -------------------------------------------------------------------------------- /src/schemas/glue.schema.ts: -------------------------------------------------------------------------------- 1 | export const GlueSchema = { 2 | type: "object", 3 | properties: { 4 | bucketDeploy: { type: "string" }, 5 | createBucket: { type: "boolean" }, 6 | s3Prefix: { type: "string" }, 7 | tempDirBucket: { type: "string" }, 8 | tempDirS3Prefix: { type: "string" }, 9 | }, 10 | required: ["bucketDeploy"], 11 | }; 12 | -------------------------------------------------------------------------------- /src/services/serverless.service.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync } from "fs"; 2 | import { Global } from "../constants/global.constant"; 3 | import { GlueJob } from "../domain/glue-job"; 4 | import { GlueTrigger } from "../domain/glue-trigger"; 5 | import { SupportFile } from "../domain/support-files"; 6 | import { AwsHelper } from "../helpers/aws.helper"; 7 | import { GlueHelper } from "../helpers/glue.helper"; 8 | import { ServerlessHelper } from "../helpers/serverless.helper"; 9 | import { GluePluginConfigInterface } from "../interfaces/glue-plugin-config.interce"; 10 | import { CloudFormationUtils } from "../utils/cloud-formation.utils"; 11 | import { StringUtils } from "../utils/string.utils"; 12 | import fs from "fs"; 13 | import path from "path" 14 | 15 | export class ServerlessService { 16 | awsHelper: AwsHelper; 17 | glueHelper: GlueHelper; 18 | config?: GluePluginConfigInterface; 19 | helperless: ServerlessHelper; 20 | constructor(private serverless: any) { 21 | this.helperless = new ServerlessHelper(this.serverless); 22 | this.config = this.helperless.getPluginConfig(); 23 | this.awsHelper = new AwsHelper(this.serverless); 24 | this.glueHelper = new GlueHelper(this.config); 25 | } 26 | 27 | async main() { 28 | if (!this.config) { 29 | this.helperless.log("Glue Config Not Found."); 30 | return; 31 | } 32 | if (this.config) { 33 | this.helperless.log("Glue config detected."); 34 | await this.processGlueJobs(); 35 | this.processTriggers(); 36 | } else { 37 | this.helperless.log("Glue config not detected."); 38 | } 39 | } 40 | async processGlueJobs() { 41 | if (!this.config?.jobs) { 42 | this.helperless.log("Jobs not found."); 43 | return; 44 | } 45 | this.helperless.log("Processing Jobs."); 46 | let jobs = this.glueHelper.getGlueJobs(); 47 | if (this.config?.createBucket) { 48 | let params = { 49 | Bucket: this.config.bucketDeploy, 50 | }; 51 | 52 | if (this.config?.createBucketConfig) { 53 | if (this.config?.createBucketConfig.LocationConstraint) { 54 | this.config.createBucketConfig.CreateBucketConfiguration = { 55 | LocationConstraint: 56 | this.config?.createBucketConfig.LocationConstraint, 57 | }; 58 | delete this.config?.createBucketConfig.LocationConstraint; 59 | } 60 | params = { 61 | ...params, 62 | ...this.config.createBucketConfig, 63 | }; 64 | } 65 | if (!(await this.awsHelper.existBucket(params))) { 66 | this.helperless.log("Bucket don't exist, I try to create it."); 67 | await this.awsHelper.createBucket(params); 68 | this.helperless.log("Bucket created."); 69 | } 70 | } 71 | 72 | for (const job of jobs) { 73 | await this.uploadJobScripts(job); 74 | await this.uploadSupportFiles(job); 75 | const jobCFId = job.id ?? StringUtils.toPascalCase(job.name); 76 | const jobCFTemplate = CloudFormationUtils.glueJobToCF(job); 77 | this.helperless.appendToTemplate( 78 | "resources", 79 | jobCFId, 80 | jobCFTemplate 81 | ); 82 | } 83 | 84 | if ( 85 | jobs.filter((e) => e.tempDir).length > 0 && 86 | !this.config?.tempDirBucket 87 | ) { 88 | const bucketTemplate = CloudFormationUtils.generateBucketTemplate( 89 | `GlueTempBucket-${StringUtils.randomString(8)}` 90 | ); 91 | this.helperless.appendToTemplate( 92 | "resources", 93 | Global.GLUE_TEMP_BUCKET_REF, 94 | bucketTemplate 95 | ); 96 | this.helperless.appendToTemplate("outputs", "GlueJobTempBucketName", { 97 | Value: Global.GLUE_TEMP_BUCKET_REF, 98 | }); 99 | } 100 | } 101 | 102 | processTriggers() { 103 | if (!this.config?.triggers) { 104 | this.helperless.log("Triggers not found."); 105 | return; 106 | } 107 | this.helperless.log("Processing Triggers."); 108 | let triggers = this.glueHelper.getGlueTriggers(); 109 | triggers.forEach((trigger: GlueTrigger) => { 110 | const triggerCFTemplate = CloudFormationUtils.glueTriggerToCF(trigger); 111 | this.helperless.appendToTemplate( 112 | "resources", 113 | StringUtils.toPascalCase(trigger.name), 114 | triggerCFTemplate 115 | ); 116 | }); 117 | } 118 | 119 | async uploadJobScripts(job: GlueJob) { 120 | if (!this.config) throw new Error("Glue Config not found."); 121 | const fileName = path.parse(job.scriptPath).base; 122 | const params = { 123 | Bucket: this.config.bucketDeploy, 124 | Body: readFileSync(path.join(job.scriptPath)), 125 | Key: `${this.config?.s3Prefix ?? "glueJobs/"}${fileName}`, 126 | }; 127 | await this.awsHelper.uploadFileToS3(params); 128 | job.setScriptS3Location(`s3://${params.Bucket}/${params.Key}`); 129 | } 130 | 131 | async uploadSupportFiles(job: GlueJob) { 132 | if (!job.SupportFiles) { 133 | return; 134 | } 135 | this.helperless.log("Support Files found."); 136 | this.helperless.log("Processing Support Files."); 137 | let supportFiles = this.glueHelper.getSupportFiles(job); 138 | 139 | supportFiles.forEach(async (supportFile: SupportFile) => { 140 | if ( 141 | supportFile.local_path == null || 142 | supportFile.s3_bucket == null || 143 | supportFile.s3_prefix == null || 144 | supportFile.execute_upload == null 145 | ) { 146 | throw new Error("Please provide all parameters for SupportFiles."); 147 | } 148 | 149 | if (!supportFile.execute_upload) { 150 | this.helperless.log(`Skipping upload for: ${supportFile.local_path}`); 151 | } 152 | 153 | if (supportFile.execute_upload) { 154 | if (fs.lstatSync(supportFile.local_path).isFile()) { 155 | this.helperless.log(`Uploading file: ${supportFile.local_path}`); 156 | let filename = require("path").basename(supportFile.local_path); 157 | const params = { 158 | Bucket: supportFile.s3_bucket, 159 | Body: readFileSync(supportFile.local_path), 160 | Key: `${supportFile.s3_prefix}${filename}`, 161 | }; 162 | this.awsHelper.uploadFileToS3(params); 163 | this.helperless.log( 164 | `Uploaded '${filename}' in s3://${params.Bucket}/${params.Key}` 165 | ); 166 | } 167 | 168 | if (fs.lstatSync(supportFile.local_path).isDirectory()) { 169 | this.helperless.log( 170 | `Uploading all files in: ${supportFile.local_path}` 171 | ); 172 | fs.readdirSync(supportFile.local_path).forEach((filename) => { 173 | const params = { 174 | Bucket: supportFile.s3_bucket, 175 | Body: readFileSync(`${supportFile.local_path}/${filename}`), 176 | Key: `${supportFile.s3_prefix}${filename}`, 177 | }; 178 | this.awsHelper.uploadFileToS3(params); 179 | this.helperless.log( 180 | `Uploaded '${filename}' in s3://${params.Bucket}/${params.Key}` 181 | ); 182 | }); 183 | } 184 | } 185 | }); 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/utils/cloud-formation.utils.ts: -------------------------------------------------------------------------------- 1 | import { GlueJob } from "../domain/glue-job"; 2 | import { GlueTrigger } from "../domain/glue-trigger"; 3 | import { GlueTriggerActionInterface } from "../interfaces/glue-trigger-action.interface"; 4 | 5 | export class CloudFormationUtils { 6 | static glueJobToCF(glueJob: GlueJob) { 7 | let cfn: { [k: string]: any } = { 8 | Type: "AWS::Glue::Job", 9 | Properties: { 10 | Command: { 11 | Name: glueJob.commandName, 12 | PythonVersion: glueJob.pythonVersion, 13 | ScriptLocation: glueJob.scriptS3Location, 14 | }, 15 | GlueVersion: glueJob.glueVersionJob, 16 | Name: glueJob.name, 17 | Description: glueJob.Description, 18 | Role: glueJob.role, 19 | MaxCapacity: glueJob.MaxCapacity, 20 | ExecutionProperty: { 21 | MaxConcurrentRuns: glueJob.MaxConcurrentRuns ?? 1, 22 | }, 23 | DefaultArguments: { 24 | "--additional-python-modules": glueJob.DefaultArguments?.additionalPythonModules, 25 | "--job-language": glueJob.DefaultArguments?.jobLanguage, 26 | "--class": glueJob.DefaultArguments?.class, 27 | "--scriptLocation": glueJob.DefaultArguments?.scriptLocation, 28 | "--extra-py-files": glueJob.DefaultArguments?.extraPyFiles, 29 | "--extra-jars": glueJob.DefaultArguments?.extraJars, 30 | "--user-jars-first": glueJob.DefaultArguments?.userJarsFirst, 31 | "--use-postgres-driver": glueJob.DefaultArguments?.usePostgresDriver, 32 | "--extra-files": glueJob.DefaultArguments?.extraFiles, 33 | "--disable-proxy": glueJob.DefaultArguments?.disableProxy, 34 | "--job-bookmark-option": glueJob.DefaultArguments?.jobBookmarkOption, 35 | "--enable-auto-scaling": glueJob.DefaultArguments?.enableAutoScaling, 36 | "--enable-s3-parquet-optimized-committer": 37 | glueJob.DefaultArguments?.enableS3ParquetOptimizedCommitter, 38 | "--enable-rename-algorithm-v2": 39 | glueJob.DefaultArguments?.enableRenameAlgorithmV2, 40 | "--enable-glue-datacatalog": 41 | glueJob.DefaultArguments?.enableGlueDatacatalog, 42 | "--enable-metrics": glueJob.DefaultArguments?.enableMetrics, 43 | "--enable-continuous-cloudwatch-log": 44 | glueJob.DefaultArguments?.enableContinuousCloudwatchLog, 45 | "--enable-continuous-log-filter": 46 | glueJob.DefaultArguments?.enableContinuousLogFilter, 47 | "--continuous-log-logGroup": 48 | glueJob.DefaultArguments?.continuousLogLogGroup, 49 | "--continuous-log-logStreamPrefix": 50 | glueJob.DefaultArguments?.continuousLogLogStreamPrefix, 51 | "--continuous-log-conversionPattern": 52 | glueJob.DefaultArguments?.continuousLogConversionPattern, 53 | "--enable-spark-ui": glueJob.DefaultArguments?.enableSparkUi, 54 | "--spark-event-logs-path": 55 | glueJob.DefaultArguments?.sparkEventLogsPath, 56 | "library-set": glueJob.DefaultArguments?.librarySet, 57 | }, 58 | Tags: glueJob.Tags, 59 | Timeout: glueJob.Timeout, 60 | MaxRetries: glueJob.MaxRetries, 61 | SecurityConfiguration: glueJob.SecurityConfiguration 62 | }, 63 | }; 64 | if (glueJob.DefaultArguments?.tempDir) { 65 | cfn.Properties.DefaultArguments = { 66 | ...cfn.Properties.DefaultArguments, 67 | "--TempDir": glueJob.DefaultArguments.tempDir, 68 | } 69 | } 70 | if (glueJob.DefaultArguments.customArguments) { 71 | const customArguments = CloudFormationUtils.parseCustomArguments(glueJob.DefaultArguments.customArguments); 72 | cfn.Properties.DefaultArguments = { 73 | ...cfn.Properties.DefaultArguments, 74 | ...customArguments 75 | } 76 | } 77 | if (glueJob.Connections) { 78 | cfn.Properties.Connections = { 79 | Connections: glueJob.Connections, 80 | }; 81 | } 82 | if (["glueetl", "gluestreaming"].indexOf(glueJob.commandName || '') !== -1) { 83 | if (glueJob.WorkerType) { 84 | cfn.Properties.WorkerType = glueJob.WorkerType; 85 | } 86 | if (glueJob.NumberOfWorkers) { 87 | cfn.Properties.NumberOfWorkers = glueJob.NumberOfWorkers; 88 | } 89 | } 90 | if (glueJob.JobRunQueuingEnabled !== undefined) { 91 | cfn.Properties.JobRunQueuingEnabled = glueJob.JobRunQueuingEnabled; 92 | } 93 | 94 | return cfn; 95 | } 96 | 97 | static parseCustomArguments(customArguments: { [k: string]: any }) { 98 | const customArgumentsJson: { [k: string]: any } = {}; 99 | const keyArguments = Object.keys(customArguments); 100 | for (const argumentName of keyArguments) { 101 | const _argumentName = argumentName.startsWith("--") 102 | ? argumentName 103 | : `--${argumentName}`; 104 | customArgumentsJson[_argumentName] = customArguments[argumentName]; 105 | } 106 | return customArgumentsJson; 107 | } 108 | 109 | static generateBucketTemplate(bucketName: string) { 110 | return { 111 | Type: "AWS::S3::Bucket", 112 | Properties: { 113 | BucketName: bucketName, 114 | }, 115 | }; 116 | } 117 | 118 | static glueTriggerToCF(trigger: GlueTrigger) { 119 | const actions = trigger.actions.map( 120 | (action: GlueTriggerActionInterface) => { 121 | return { 122 | JobName: action.name, 123 | Arguments: action.args, 124 | Timeout: action.timeout, 125 | SecurityConfiguration: action.SecurityConfiguration, 126 | }; 127 | } 128 | ); 129 | return { 130 | Type: "AWS::Glue::Trigger", 131 | Properties: { 132 | Type: trigger.type, 133 | Actions: actions, 134 | Name: trigger.name, 135 | Description: trigger.Description, 136 | Tags: trigger.Tags, 137 | StartOnCreation: trigger.StartOnCreation, 138 | ...(trigger.schedule && { Schedule: trigger.schedule }), 139 | }, 140 | }; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/utils/string.utils.ts: -------------------------------------------------------------------------------- 1 | export class StringUtils { 2 | static toPascalCase(str: string) { 3 | return str.toLowerCase().replace(/([-_ ][a-z0-9])|(^[a-zA-Z])/g, (group) => { 4 | return group 5 | .toUpperCase() 6 | .replace("-", "") 7 | .replace("_", "") 8 | .replace(" ", ""); 9 | }); 10 | } 11 | 12 | static randomString(length: number) { 13 | let result = ""; 14 | const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; 15 | for (let i = 0; i < length; i++) { 16 | result += characters.charAt( 17 | Math.floor(Math.random() * characters.length) 18 | ); 19 | } 20 | return result; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 4 | /* Projects */ 5 | // "incremental": true, /* Enable incremental compilation */ 6 | // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ 7 | // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ 8 | // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ 9 | // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ 10 | // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ 11 | /* Language and Environment */ 12 | "target": "es5", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ 13 | // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ 14 | // "jsx": "preserve", /* Specify what JSX code is generated. */ 15 | // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ 16 | // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ 17 | // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ 18 | // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ 19 | // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ 20 | // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ 21 | // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ 22 | // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ 23 | /* Modules */ 24 | "module": "commonjs", /* Specify what module code is generated. */ 25 | "rootDir": "./src", /* Specify the root folder within your source files. */ 26 | // "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ 27 | // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ 28 | // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ 29 | // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ 30 | // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ 31 | // "types": [], /* Specify type package names to be included without being referenced in a source file. */ 32 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 33 | // "resolveJsonModule": true, /* Enable importing .json files */ 34 | // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ 35 | /* JavaScript Support */ 36 | // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */ 37 | // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ 38 | // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ 39 | /* Emit */ 40 | // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ 41 | // "declarationMap": true, /* Create sourcemaps for d.ts files. */ 42 | // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ 43 | // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ 44 | // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ 45 | "outDir": "./lib", /* Specify an output folder for all emitted files. */ 46 | "removeComments": true, /* Disable emitting comments. */ 47 | // "noEmit": true, /* Disable emitting files from a compilation. */ 48 | // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ 49 | // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ 50 | // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ 51 | // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ 52 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 53 | // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ 54 | // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ 55 | // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ 56 | // "newLine": "crlf", /* Set the newline character for emitting files. */ 57 | // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ 58 | // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ 59 | // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ 60 | // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ 61 | // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ 62 | /* Interop Constraints */ 63 | // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ 64 | // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ 65 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */ 66 | // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ 67 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ 68 | /* Type Checking */ 69 | "strict": true, /* Enable all strict type-checking options. */ 70 | // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ 71 | // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ 72 | // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ 73 | // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ 74 | // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ 75 | // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ 76 | // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ 77 | // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ 78 | // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ 79 | // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ 80 | // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ 81 | // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ 82 | // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ 83 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ 84 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ 85 | // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ 86 | // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ 87 | // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ 88 | /* Completeness */ 89 | // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ 90 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 91 | } 92 | } --------------------------------------------------------------------------------