├── .eslintrc.json ├── .gitignore ├── LICENSE.txt ├── Makefile ├── Makefile_Pandoc ├── README-SAR.md ├── README.md ├── example ├── .gitignore ├── Makefile ├── src │ ├── child-process-promise.js │ ├── index.js │ └── s3-util.js └── template.yaml └── template.yaml /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "crockford", 3 | "env": { 4 | "node": true, 5 | "es6": true 6 | }, 7 | "parserOptions": { 8 | "ecmaVersion": 6 9 | }, 10 | "rules": { 11 | "semi": ["error", "always"], 12 | "strict": ["error", "function"], 13 | "no-unused-vars": "error", 14 | "indent": ["error", "tab" ], 15 | "no-const-assign": "error", 16 | "one-var": "error", 17 | "prefer-const": "error", 18 | "no-var": "error", 19 | "prefer-arrow-callback": "error", 20 | "no-plusplus": ["error", { "allowForLoopAfterthoughts": true }], 21 | "quotes": ["error", "single", {"avoidEscape": true, "allowTemplateLiterals": true}] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | cache 3 | result 4 | build 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | GPL v2, see https://github.com/jgm/pandoc/blob/master/COPYRIGHT 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT_ROOT = $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 2 | 3 | DOCKER_IMAGE ?= lambci/lambda-base-2:build 4 | TARGET ?=/opt/ 5 | 6 | MOUNTS = -v $(PROJECT_ROOT):/var/task \ 7 | -v $(PROJECT_ROOT)result:$(TARGET) \ 8 | -v $(PROJECT_ROOT)cache:/usr/local 9 | 10 | DOCKER = docker run -it --rm -w=/var/task/build 11 | build result cache: 12 | mkdir $@ 13 | 14 | clean: 15 | rm -rf build result cache 16 | 17 | test: 18 | $(DOCKER) $(MOUNTS) --entrypoint /opt/bin/pandoc -t $(DOCKER_IMAGE) -v 19 | 20 | bash: 21 | $(DOCKER) $(MOUNTS) --entrypoint /bin/bash -t $(DOCKER_IMAGE) 22 | 23 | all: build result cache 24 | $(DOCKER) $(MOUNTS) --entrypoint /usr/bin/make -t $(DOCKER_IMAGE) TARGET_DIR=$(TARGET) -f ../Makefile_Pandoc $@ 25 | 26 | 27 | STACK_NAME ?= pandoc-layer 28 | 29 | #result/bin/pandoc: all 30 | 31 | build/output.yaml: template.yaml result/bin/pandoc 32 | aws cloudformation package --template $< --s3-bucket $(DEPLOYMENT_BUCKET) --output-template-file $@ 33 | 34 | deploy: build/output.yaml 35 | aws cloudformation deploy --template $< --stack-name $(STACK_NAME) 36 | aws cloudformation describe-stacks --stack-name $(STACK_NAME) --query Stacks[].Outputs --output table 37 | 38 | deploy-example: 39 | cd example && \ 40 | make deploy DEPLOYMENT_BUCKET=$(DEPLOYMENT_BUCKET) PANDOC_STACK_NAME=$(STACK_NAME) 41 | -------------------------------------------------------------------------------- /Makefile_Pandoc: -------------------------------------------------------------------------------- 1 | GHC_VERSION ?= 8.4.4 2 | CABAL_VERSION ?= 2.2.0.0 3 | EPEL_VERSION ?= latest-7 4 | 5 | TARGET_DIR ?= /opt/ 6 | PROJECT_ROOT = $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 7 | CACHE_DIR=$(PROJECT_ROOT)build/cache 8 | 9 | .ONESHELL: 10 | 11 | ### step 1: install GHC 12 | # GHC is self-referential, so we need some version of GHC to install the latest version; 13 | # so first install the older one that comes with EPEL on RPM, then update to the latest 14 | 15 | GHC_SOURCE=ghc-$(GHC_VERSION)-x86_64-centos70-linux.tar.xz 16 | 17 | /etc/yum.repos.d/epel.repo: 18 | curl -LO https://dl.fedoraproject.org/pub/epel/epel-release-$(EPEL_VERSION).noarch.rpm 19 | rpm -i epel-release-$(EPEL_VERSION).noarch.rpm 20 | 21 | /usr/bin/ghc: /etc/yum.repos.d/epel.repo 22 | yum install ghc -y 23 | 24 | $(GHC_SOURCE): 25 | curl -LO https://downloads.haskell.org/~ghc/$(GHC_VERSION)/$(GHC_SOURCE) 26 | 27 | 28 | /usr/local/bin/ghc: $(GHC_SOURCE) /usr/bin/ghc 29 | tar xf $(GHC_SOURCE) 30 | rm $(GHC_SOURCE) 31 | cd ghc* 32 | ./configure --prefix=/usr/local 33 | make install 34 | 35 | 36 | ### step 2: set up a cabal sandbox 37 | 38 | CABAL_SOURCE=cabal-install-$(CABAL_VERSION)-x86_64-unknown-linux.tar.gz 39 | 40 | $(CABAL_SOURCE): 41 | curl -LO https://www.haskell.org/cabal/release/cabal-install-$(CABAL_VERSION)/$(CABAL_SOURCE) 42 | 43 | /usr/local/bin/cabal: $(CABAL_SOURCE) /usr/local/bin/ghc 44 | tar xf $(CABAL_SOURCE) 45 | mv cabal /usr/local/bin 46 | 47 | /root/.cabal/packages/hackage.haskell.org: /usr/local/bin/cabal 48 | cabal update 49 | 50 | cabal.sandbox.config: /root/.cabal/packages/hackage.haskell.org 51 | cabal sandbox init --sandbox . 52 | 53 | ### step 3: compile pandoc 54 | 55 | 56 | bin/pandoc: cabal.sandbox.config 57 | cabal install --disable-documentation pandoc -fembed_data_files 58 | 59 | $(TARGET_DIR)bin/pandoc: bin/pandoc 60 | mkdir -p $(TARGET_DIR)bin 61 | cp bin/pandoc $(TARGET_DIR)bin 62 | 63 | all: $(TARGET_DIR)bin/pandoc 64 | -------------------------------------------------------------------------------- /README-SAR.md: -------------------------------------------------------------------------------- 1 | # Pandoc Lambda Layer for Amazon Linux 2 AMIs 2 | 3 | Static build of Pandoc for Amazon Linux 2, packaged as a Lambda layer. 4 | Bundles Pandoc 2.7.2. 5 | 6 | This application provides a single output, `LayerVersion`, which points to a 7 | Lambda Layer ARN you can use with Lambda runtimes based on Amazon Linux 2 (such 8 | as the `nodejs10.x` runtime). 9 | 10 | For an example of how to use the layer, check out 11 | 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pandoc for AWS Lambda 2 | 3 | Scripts to compile Pandoc for AWS Lambda instances powered by Amazon Linux 2.x, such as the `nodejs10.x` runtime, and the updated 2018.03 Amazon Linux 1 runtimes. 4 | 5 | ## Usage 6 | 7 | Absolutely the easiest way of using this is to pull it directly from the AWS Serverless Application repository into a CloudFormation/SAM application, or deploy directly from the Serverless Application Repository into your account, and then link as a layer. 8 | 9 | The `pandoc` binary will be in `/opt/bin/pandoc` after linking the layer to a Lambda function. 10 | 11 | For more information, check out the [pandoc-lambda-layer](https://console.aws.amazon.com/serverlessrepo/home?region=us-east-1#/published-applications/arn:aws:serverlessrepo:us-east-1:145266761615:applications~pandoc-lambda-layer) application in the Serverless App Repository. 12 | 13 | For manual deployments and custom builds, read below... 14 | 15 | ## Prerequisites 16 | 17 | * Docker desktop 18 | * Unix Make environment 19 | * AWS command line utilities (just for deployment) 20 | 21 | ## Compiling the code 22 | 23 | * start Docker services 24 | * `make all` 25 | 26 | There are two `make` scripts in this project. 27 | 28 | * [`Makefile`](Makefile) is intended to run on the build system, and just starts a Docker container matching the AWS Linux 2 environment for Lambda runtimes to compile Pandoc using the second script. 29 | * [`Makefile_Pandoc`](Makefile_Pandoc) is the script that will run inside the container, and actually compile binaries. 30 | 31 | The output will be in the `result` dir. 32 | 33 | ### Configuring the build 34 | 35 | By default, this compiles a version expecting to run as a Lambda layer from `/opt`. You can change the expected location by providing a `TARGET` variable when invoking `make`. 36 | 37 | The default Docker image used is `lambci/lambda-base-2:build`. To use a different base, provide a `DOCKER_IMAGE` variable when invoking `make`. 38 | 39 | Modify the versions of libraries or Pandoc directly in [`Makefile_Pandoc`](Makefile_Pandoc). 40 | 41 | ### Experimenting 42 | 43 | * `make bash` to open an interactive shell with all the build directories mounted 44 | 45 | ### Compiled info 46 | 47 | ``` 48 | pandoc 2.7.2 49 | Compiled with pandoc-types 1.17.5.4, texmath 0.11.2.2, skylighting 0.7.7 50 | ``` 51 | 52 | ## Deploying to AWS as a layer 53 | 54 | Run the following command to deploy the compiled result as a layer in your AWS account. 55 | 56 | ``` 57 | make deploy DEPLOYMENT_BUCKET= 58 | ``` 59 | 60 | ### configuring the deployment 61 | 62 | By default, this uses `pandoc-layer` as the stack name. Provide a `STACK_NAME` variable when 63 | calling `make deploy` to use an alternative name. 64 | 65 | ### example usage 66 | 67 | An example project is in the [example](example) directory. It sets up two buckets, and listens to file uploads on the first bucket to convert and generate HTML files from markdown. You can deploy it from the root Makefile using: 68 | 69 | ``` 70 | make deploy-example DEPLOYMENT_BUCKET= 71 | ``` 72 | 73 | For more information, check out: 74 | 75 | * https://pandoc.org/installing.html 76 | 77 | ## Author 78 | 79 | Gojko Adzic 80 | 81 | ## License 82 | 83 | * These scripts: [MIT](https://opensource.org/licenses/MIT) 84 | * Pandoc: https://github.com/jgm/pandoc/blob/master/COPYRIGHT 85 | * Contained libraries all have separate licenses, check the respective web sites for more information 86 | -------------------------------------------------------------------------------- /example/.gitignore: -------------------------------------------------------------------------------- 1 | output.yaml 2 | -------------------------------------------------------------------------------- /example/Makefile: -------------------------------------------------------------------------------- 1 | STACK_NAME ?= pandoc-layer-example 2 | PANDOC_STACK_NAME ?= pandoc-layer 3 | 4 | PANDOC_LAYER ?=$(shell aws cloudformation describe-stacks --stack-name $(PANDOC_STACK_NAME) --query Stacks[].Outputs[].OutputValue --output text) 5 | SOURCES=$(shell find src/) 6 | 7 | clean: 8 | rm -rf build 9 | 10 | output.yaml: template.yaml $(SOURCES) 11 | mkdir -p build 12 | aws cloudformation package --template-file $< --output-template-file $@ --s3-bucket $(DEPLOYMENT_BUCKET) 13 | 14 | deploy: output.yaml 15 | aws cloudformation deploy --template-file $< --stack-name $(STACK_NAME) --capabilities CAPABILITY_IAM --parameter-overrides PandocLayer=$(PANDOC_LAYER) 16 | aws cloudformation describe-stacks --stack-name $(STACK_NAME) --query Stacks[].Outputs --output table 17 | 18 | -------------------------------------------------------------------------------- /example/src/child-process-promise.js: -------------------------------------------------------------------------------- 1 | /*global module, require, console, Promise */ 2 | 'use strict'; 3 | const childProcess = require('child_process'), 4 | spawnPromise = function (command, argsarray, envOptions) { 5 | return new Promise((resolve, reject) => { 6 | console.log('executing', command, argsarray.join(' ')); 7 | const childProc = childProcess.spawn(command, argsarray, envOptions || {env: process.env, cwd: process.cwd()}), 8 | resultBuffers = []; 9 | childProc.stdout.on('data', buffer => { 10 | console.log(buffer.toString()); 11 | resultBuffers.push(buffer); 12 | }); 13 | childProc.stderr.on('data', buffer => console.error(buffer.toString())); 14 | childProc.on('exit', (code, signal) => { 15 | console.log(`${command} completed with ${code}:${signal}`); 16 | if (code || signal) { 17 | reject(`${command} failed with ${code || signal}`); 18 | } else { 19 | resolve(Buffer.concat(resultBuffers).toString().trim()); 20 | } 21 | }); 22 | }); 23 | }; 24 | module.exports = { 25 | spawn: spawnPromise 26 | }; 27 | -------------------------------------------------------------------------------- /example/src/index.js: -------------------------------------------------------------------------------- 1 | 2 | const s3Util = require('./s3-util'), 3 | childProcessPromise = require('./child-process-promise'), 4 | path = require('path'), 5 | os = require('os'), 6 | EXTENSION = process.env.EXTENSION, 7 | THUMB_WIDTH = process.env.THUMB_WIDTH, 8 | OUTPUT_BUCKET = process.env.OUTPUT_BUCKET, 9 | MIME_TYPE = process.env.MIME_TYPE; 10 | 11 | exports.handler = function (eventObject, context) { 12 | const eventRecord = eventObject.Records && eventObject.Records[0], 13 | inputBucket = eventRecord.s3.bucket.name, 14 | key = eventRecord.s3.object.key, 15 | id = context.awsRequestId, 16 | resultKey = key.replace(/\.[^.]+$/, EXTENSION), 17 | workdir = os.tmpdir(), 18 | inputFile = path.join(workdir, id + path.extname(key)), 19 | outputFile = path.join(workdir, 'converted-' + id + EXTENSION); 20 | 21 | 22 | console.log('converting', inputBucket, key, 'using', inputFile); 23 | return s3Util.downloadFileFromS3(inputBucket, key, inputFile) 24 | .then(() => childProcessPromise.spawn( 25 | '/opt/bin/pandoc', 26 | [inputFile, '-o', outputFile], 27 | {env: process.env, cwd: workdir} 28 | )) 29 | .then(() => s3Util.uploadFileToS3(OUTPUT_BUCKET, resultKey, outputFile, MIME_TYPE)); 30 | }; 31 | -------------------------------------------------------------------------------- /example/src/s3-util.js: -------------------------------------------------------------------------------- 1 | /*global module, require, Promise, console */ 2 | 3 | const aws = require('aws-sdk'), 4 | fs = require('fs'), 5 | s3 = new aws.S3(), 6 | downloadFileFromS3 = function (bucket, fileKey, filePath) { 7 | 'use strict'; 8 | console.log('downloading', bucket, fileKey, filePath); 9 | return new Promise(function (resolve, reject) { 10 | const file = fs.createWriteStream(filePath), 11 | stream = s3.getObject({ 12 | Bucket: bucket, 13 | Key: fileKey 14 | }).createReadStream(); 15 | stream.on('error', reject); 16 | file.on('error', reject); 17 | file.on('finish', function () { 18 | console.log('downloaded', bucket, fileKey); 19 | resolve(filePath); 20 | }); 21 | stream.pipe(file); 22 | }); 23 | }, uploadFileToS3 = function (bucket, fileKey, filePath, contentType) { 24 | 'use strict'; 25 | console.log('uploading', bucket, fileKey, filePath); 26 | return s3.upload({ 27 | Bucket: bucket, 28 | Key: fileKey, 29 | Body: fs.createReadStream(filePath), 30 | ACL: 'private', 31 | ContentType: contentType 32 | }).promise(); 33 | }; 34 | 35 | module.exports = { 36 | downloadFileFromS3: downloadFileFromS3, 37 | uploadFileToS3: uploadFileToS3 38 | }; -------------------------------------------------------------------------------- /example/template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Transform: AWS::Serverless-2016-10-31 3 | Description: > 4 | Example project demonstrating the usage of the ImageMagic Layer for AWS Linux 2 runtimes. 5 | 6 | Parameters: 7 | PandocLayer: 8 | Type: String 9 | ConversionFileType: 10 | Type: String 11 | Default: html 12 | ConversionMimeType: 13 | Type: String 14 | Default: text/html 15 | Resources: 16 | UploadBucket: 17 | Type: AWS::S3::Bucket 18 | 19 | ResultsBucket: 20 | Type: AWS::S3::Bucket 21 | 22 | ConvertFileFunction: 23 | Type: AWS::Serverless::Function 24 | Properties: 25 | Handler: index.handler 26 | Timeout: 180 27 | MemorySize: 1024 28 | Runtime: nodejs10.x 29 | CodeUri: src 30 | Layers: 31 | - !Ref PandocLayer 32 | Policies: 33 | - S3CrudPolicy: 34 | BucketName: !Sub "${AWS::StackName}-*" 35 | Environment: 36 | Variables: 37 | OUTPUT_BUCKET: !Ref ResultsBucket 38 | EXTENSION: !Sub '.${ConversionFileType}' 39 | MIME_TYPE: !Ref ConversionMimeType 40 | Events: 41 | FileUpload: 42 | Type: S3 43 | Properties: 44 | Bucket: !Ref UploadBucket 45 | Events: s3:ObjectCreated:* 46 | 47 | Outputs: 48 | UploadBucket: 49 | Description: "Upload S3 bucket" 50 | Value: !Ref UploadBucket 51 | ResultsBucket: 52 | Description: "Results S3 bucket" 53 | Value: !Ref ResultsBucket 54 | 55 | -------------------------------------------------------------------------------- /template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Transform: AWS::Serverless-2016-10-31 3 | Description: > 4 | Static build of Pandoc for Amazon Linux 2 5 | 6 | Check out https://github.com/serverlesspub/pandoc-aws-lambda-binary 7 | for more information. 8 | Resources: 9 | PandocLayer: 10 | Type: AWS::Serverless::LayerVersion 11 | Properties: 12 | LayerName: pandoc 13 | Description: Static build of Pandoc for Amazon Linux 2 14 | ContentUri: result/ 15 | CompatibleRuntimes: 16 | - nodejs10.x 17 | LicenseInfo: https://github.com/jgm/pandoc/blob/master/COPYRIGHT 18 | RetentionPolicy: Retain 19 | 20 | Outputs: 21 | LayerVersion: 22 | Description: Layer ARN Reference 23 | Value: !Ref PandocLayer 24 | 25 | Metadata: 26 | AWS::ServerlessRepo::Application: 27 | Name: pandoc-lambda-layer 28 | Description: > 29 | Static build of Pandoc for Amazon Linux 2, 30 | packaged as a Lambda layer. Bundles Pandoc 2.7.2. 31 | Author: Gojko Adzic 32 | SpdxLicenseId: GPL-2.0 33 | LicenseUrl: LICENSE.txt 34 | ReadmeUrl: README-SAR.md 35 | Labels: ['layer', 'image', 'lambda', 'pandoc'] 36 | HomePageUrl: https://github.com/serverlesspub/pandoc-aws-lambda-binary 37 | SemanticVersion: 1.0.0 38 | SourceCodeUrl: https://github.com/serverlesspub/pandoc-aws-lambda-binary 39 | --------------------------------------------------------------------------------