├── .Rbuildignore ├── .github ├── .gitignore └── workflows │ ├── R-CMD-check.yaml │ ├── lint.yaml │ ├── pkgdown.yaml │ └── test-coverage.yaml ├── .gitignore ├── .lintr ├── DESCRIPTION ├── NAMESPACE ├── NEWS.md ├── R ├── boto3.R ├── botor.R ├── checkmate.R ├── iam.R ├── kinesis.R ├── kms.R ├── python_modules.R ├── s3.R ├── sm.R ├── ssm.R ├── utils.R └── zzz.R ├── README.md ├── _pkgdown.yml ├── inst └── tests │ ├── testthat.R │ └── testthat │ ├── test-checkmate.R │ └── test-load.R └── man ├── base64_dec.Rd ├── base64_enc.Rd ├── boto3.Rd ├── boto3_version.Rd ├── botor.Rd ├── botor_client.Rd ├── botor_session.Rd ├── botor_session_pid.Rd ├── botor_session_uuid.Rd ├── check_s3_uri.Rd ├── clients.Rd ├── coerce_bytes_literals_to_string.Rd ├── figures └── logo.png ├── iam.Rd ├── iam_get_user.Rd ├── iam_whoami.Rd ├── kinesis.Rd ├── kinesis_describe_stream.Rd ├── kinesis_get_records.Rd ├── kinesis_get_shard_iterator.Rd ├── kinesis_put_record.Rd ├── kms.Rd ├── kms_decrypt.Rd ├── kms_decrypt_file.Rd ├── kms_encrypt.Rd ├── kms_encrypt_file.Rd ├── kms_generate_data_key.Rd ├── mime_guess.Rd ├── python_version.Rd ├── require_python_builtins.Rd ├── require_python_module.Rd ├── s3.Rd ├── s3_copy.Rd ├── s3_delete.Rd ├── s3_download_file.Rd ├── s3_exists.Rd ├── s3_list_buckets.Rd ├── s3_ls.Rd ├── s3_object.Rd ├── s3_put_object_tagging.Rd ├── s3_read.Rd ├── s3_split_uri.Rd ├── s3_upload_file.Rd ├── s3_write.Rd ├── sm.Rd ├── sm_get_secret.Rd ├── ssm.Rd ├── ssm_get_parameter.Rd ├── sts_whoami.Rd ├── trypy.Rd └── uuid.Rd /.Rbuildignore: -------------------------------------------------------------------------------- 1 | .travis.yml 2 | _pkgdown.yml 3 | .editorconfig 4 | .gitignore 5 | .lintr 6 | .*.sh$ 7 | .backup 8 | ^\.github$ 9 | -------------------------------------------------------------------------------- /.github/.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | -------------------------------------------------------------------------------- /.github/workflows/R-CMD-check.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | 8 | name: R-CMD-check.yaml 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | R-CMD-check: 14 | runs-on: ubuntu-latest 15 | env: 16 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 17 | R_KEEP_PKG_SOURCE: yes 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - uses: r-lib/actions/setup-r@v2 22 | with: 23 | use-public-rspm: true 24 | 25 | - uses: r-lib/actions/setup-r-dependencies@v2 26 | with: 27 | extra-packages: any::rcmdcheck 28 | needs: check 29 | 30 | - uses: r-lib/actions/check-r-package@v2 31 | with: 32 | upload-snapshots: true 33 | build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")' 34 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | 8 | name: lint.yaml 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | lint: 14 | runs-on: ubuntu-latest 15 | env: 16 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 17 | steps: 18 | - uses: actions/checkout@v4 19 | 20 | - uses: r-lib/actions/setup-r@v2 21 | with: 22 | use-public-rspm: true 23 | 24 | - uses: r-lib/actions/setup-r-dependencies@v2 25 | with: 26 | extra-packages: any::lintr, local::. 27 | needs: lint 28 | 29 | - name: Lint 30 | run: lintr::lint_package() 31 | shell: Rscript {0} 32 | env: 33 | LINTR_ERROR_ON_LINT: true 34 | -------------------------------------------------------------------------------- /.github/workflows/pkgdown.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | release: 8 | types: [published] 9 | workflow_dispatch: 10 | 11 | name: pkgdown.yaml 12 | 13 | permissions: read-all 14 | 15 | jobs: 16 | pkgdown: 17 | runs-on: ubuntu-latest 18 | # Only restrict concurrency for non-PR jobs 19 | concurrency: 20 | group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} 21 | env: 22 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 23 | permissions: 24 | contents: write 25 | steps: 26 | - uses: actions/checkout@v4 27 | 28 | - uses: r-lib/actions/setup-pandoc@v2 29 | 30 | - uses: r-lib/actions/setup-r@v2 31 | with: 32 | use-public-rspm: true 33 | 34 | - uses: r-lib/actions/setup-r-dependencies@v2 35 | with: 36 | extra-packages: any::pkgdown, local::. 37 | needs: website 38 | 39 | - name: Build site 40 | run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) 41 | shell: Rscript {0} 42 | 43 | - name: Deploy to GitHub pages 🚀 44 | if: github.event_name != 'pull_request' 45 | uses: JamesIves/github-pages-deploy-action@v4.5.0 46 | with: 47 | clean: false 48 | branch: gh-pages 49 | folder: docs 50 | -------------------------------------------------------------------------------- /.github/workflows/test-coverage.yaml: -------------------------------------------------------------------------------- 1 | # Workflow derived from https://github.com/r-lib/actions/tree/v2/examples 2 | # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help 3 | on: 4 | push: 5 | branches: [main, master] 6 | pull_request: 7 | 8 | name: test-coverage.yaml 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | test-coverage: 14 | runs-on: ubuntu-latest 15 | env: 16 | GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - uses: actions/setup-python@v5 22 | - run: pip install boto3 23 | 24 | - uses: r-lib/actions/setup-r@v2 25 | with: 26 | use-public-rspm: true 27 | 28 | - uses: r-lib/actions/setup-r-dependencies@v2 29 | with: 30 | extra-packages: any::covr, any::xml2 31 | needs: coverage 32 | 33 | - name: Test coverage 34 | run: | 35 | cov <- covr::package_coverage( 36 | quiet = FALSE, 37 | clean = FALSE, 38 | install_path = file.path(normalizePath(Sys.getenv("RUNNER_TEMP"), winslash = "/"), "package") 39 | ) 40 | covr::to_cobertura(cov) 41 | shell: Rscript {0} 42 | 43 | - uses: codecov/codecov-action@v4 44 | with: 45 | # Fail if error if not on PR, or if on PR and token is given 46 | fail_ci_if_error: ${{ github.event_name != 'pull_request' || secrets.CODECOV_TOKEN }} 47 | file: ./cobertura.xml 48 | plugin: noop 49 | disable_search: true 50 | token: ${{ secrets.CODECOV_TOKEN }} 51 | 52 | - name: Show testthat output 53 | if: always() 54 | run: | 55 | ## -------------------------------------------------------------------- 56 | find '${{ runner.temp }}/package' -name 'testthat.Rout*' -exec cat '{}' \; || true 57 | shell: bash 58 | 59 | - name: Upload test results 60 | if: failure() 61 | uses: actions/upload-artifact@v4 62 | with: 63 | name: coverage-test-failures 64 | path: ${{ runner.temp }}/package 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.sh 2 | -------------------------------------------------------------------------------- /.lintr: -------------------------------------------------------------------------------- 1 | linters: linters_with_defaults( 2 | line_length_linter(160L), 3 | indentation_linter(indent=4L, hanging_indent_style="never"), 4 | quotes_linter("'"), 5 | brace_linter() 6 | ) 7 | exclude: '# nolint' 8 | exclude_start: '# nolint start' 9 | exclude_end: '# nolint end' 10 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Type: Package 2 | Package: botor 3 | Authors@R: c( 4 | person("Gergely", "Daróczi", , "daroczig@rapporter.net", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-3149-8537")), 5 | person("System1", role = c("fnd")) 6 | ) 7 | Title: 'AWS Python SDK' ('boto3') for R 8 | Description: Fork-safe, raw access to the 'Amazon Web Services' ('AWS') 'SDK' via the 'boto3' 'Python' module, and convenient helper functions to query the 'Simple Storage Service' ('S3') and 'Key Management Service' ('KMS'), partial support for 'IAM', the 'Systems Manager Parameter Store' and 'Secrets Manager'. 9 | SystemRequirements: Python and boto3 (https://aws.amazon.com/sdk-for-python) 10 | Version: 0.4.1 11 | Date: 2025-01-23 12 | URL: https://daroczig.github.io/botor/ 13 | BugReports: https://github.com/daroczig/botor/issues 14 | RoxygenNote: 7.3.2 15 | License: AGPL-3 16 | Encoding: UTF-8 17 | Imports: 18 | utils, 19 | reticulate, 20 | checkmate, 21 | logger, 22 | jsonlite 23 | Suggests: 24 | testthat, 25 | covr, 26 | digest 27 | -------------------------------------------------------------------------------- /NAMESPACE: -------------------------------------------------------------------------------- 1 | # Generated by roxygen2: do not edit by hand 2 | 3 | export(assert_s3_uri) 4 | export(boto3) 5 | export(boto3_version) 6 | export(botor) 7 | export(botor_client) 8 | export(check_s3_uri) 9 | export(expect_s3_uri) 10 | export(iam) 11 | export(iam_get_user) 12 | export(iam_whoami) 13 | export(kinesis) 14 | export(kinesis_describe_stream) 15 | export(kinesis_get_records) 16 | export(kinesis_get_shard_iterator) 17 | export(kinesis_put_record) 18 | export(kms) 19 | export(kms_decrypt) 20 | export(kms_decrypt_file) 21 | export(kms_encrypt) 22 | export(kms_encrypt_file) 23 | export(kms_generate_data_key) 24 | export(mime_guess) 25 | export(s3) 26 | export(s3_copy) 27 | export(s3_delete) 28 | export(s3_download_file) 29 | export(s3_exists) 30 | export(s3_list_buckets) 31 | export(s3_ls) 32 | export(s3_object) 33 | export(s3_put_object_tagging) 34 | export(s3_read) 35 | export(s3_upload_file) 36 | export(s3_write) 37 | export(sm) 38 | export(sm_get_secret) 39 | export(ssm) 40 | export(ssm_get_parameter) 41 | export(sts_whoami) 42 | export(test_s3_uri) 43 | importFrom(checkmate,assert_class) 44 | importFrom(checkmate,assert_directory_exists) 45 | importFrom(checkmate,assert_file_exists) 46 | importFrom(checkmate,assert_flag) 47 | importFrom(checkmate,assert_integer) 48 | importFrom(checkmate,assert_string) 49 | importFrom(checkmate,assert_vector) 50 | importFrom(checkmate,check_string) 51 | importFrom(checkmate,makeAssertion) 52 | importFrom(checkmate,makeAssertionFunction) 53 | importFrom(checkmate,makeExpectation) 54 | importFrom(checkmate,makeExpectationFunction) 55 | importFrom(checkmate,makeTestFunction) 56 | importFrom(checkmate,vname) 57 | importFrom(jsonlite,fromJSON) 58 | importFrom(logger,log_debug) 59 | importFrom(logger,log_error) 60 | importFrom(logger,log_info) 61 | importFrom(logger,log_trace) 62 | importFrom(logger,log_warn) 63 | importFrom(reticulate,import) 64 | importFrom(reticulate,iter_next) 65 | importFrom(reticulate,iterate) 66 | importFrom(reticulate,py_clear_last_error) 67 | importFrom(reticulate,py_last_error) 68 | importFrom(reticulate,py_str) 69 | -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- 1 | # botor 0.4.1.9000 (development) 2 | 3 | * Support passing ExtraArgs to S3 `upload_file` (#14) 4 | 5 | # botor 0.4.1 (2025-01-23) 6 | 7 | Maintenance release: 8 | 9 | * Fix S3 listing bug (#18, #20) via #19 (thanks @jburos) 10 | * Support unsigned S3 requests 11 | * Linting and code style updates 12 | * Move CI/CD from Travis to GHA 13 | 14 | # botor 0.4.0 (2023-03-12) 15 | 16 | * Fix encoding issue of bytearrays 17 | * Python 2 and Python 3 compatibility fixes 18 | * Pass limit param to get records from Kinesis 19 | * Coerce NULL to NA 20 | * Support for AWS Systems Manager 21 | 22 | # botor 0.3.0 (2020-02-16) 23 | 24 | * Make caching client/resource optional (#7) 25 | * Fix `boto3_version` calling the right object 26 | * S3 taggingfunctions thanks to @katrinabrock (#8) 27 | * Fix S3 URI schema parser (#9 and #10) 28 | * Fix documenting and passing ... in the Kinesis helpers 29 | 30 | # botor 0.2 (2019-09-23) 31 | 32 | Initial CRAN release. 33 | -------------------------------------------------------------------------------- /R/boto3.R: -------------------------------------------------------------------------------- 1 | #' Raw access to the boto3 module imported at package load time 2 | #' @note You may rather want to use \code{\link{botor}} instead, that provides a fork-safe \code{boto3} session. 3 | #' @export 4 | boto3 <- NULL 5 | 6 | 7 | #' boto3 version 8 | #' @return string 9 | #' @export 10 | boto3_version <- function() { 11 | boto3$`__version__` 12 | } 13 | -------------------------------------------------------------------------------- /R/botor.R: -------------------------------------------------------------------------------- 1 | #' Internal boto3 session 2 | #' @keywords internal 3 | botor_session <- NULL 4 | 5 | 6 | #' The default, fork-safe Boto3 session 7 | #' @param aws_access_key_id AWS access key ID 8 | #' @param aws_secret_access_key AWS secret access key 9 | #' @param aws_session_token AWS temporary session token 10 | #' @param region_name Default region when creating new connections 11 | #' @param botocore_session Use this Botocore session instead of creating a new default one 12 | #' @param profile_name The name of a profile to use. If not given, then the default profile is used 13 | #' @return boto3 \code{Session} 14 | #' @export 15 | botor <- function(aws_access_key_id, aws_secret_access_key, aws_session_token, region_name, botocore_session, profile_name) { 16 | 17 | mc <- match.call() 18 | args <- as.list(mc[-1]) 19 | 20 | if (length(args) != 0 || is.null(botor_session) || botor_session_pid() != Sys.getpid()) { 21 | 22 | if (!is.null(botor_session) && length(args) == 0) { 23 | args <- attr(botor_session, 'args') 24 | } 25 | 26 | session <- do.call(boto3$session$Session, args) 27 | 28 | utils::assignInMyNamespace( 29 | 'botor_session', 30 | structure( 31 | session, 32 | pid = Sys.getpid(), 33 | uuid = uuid(), 34 | args = args)) 35 | } 36 | 37 | botor_session 38 | 39 | } 40 | 41 | 42 | #' Look up the PID used to initialize the Boto3 session 43 | #' @return int 44 | #' @keywords internal 45 | botor_session_pid <- function() { 46 | attr(botor_session, 'pid') 47 | } 48 | 49 | 50 | #' Look up the UUID of the initialized Boto3 session 51 | #' @return int 52 | #' @keywords internal 53 | botor_session_uuid <- function() { 54 | attr(botor_session, 'uuid') 55 | } 56 | 57 | 58 | #' boto3 clients cache 59 | #' @keywords internal 60 | clients <- new.env() 61 | 62 | 63 | #' Creates an initial or reinitialize an already existing AWS client 64 | #' or resource cached in the package's namespace 65 | #' @param service string, eg S3 or IAM 66 | #' @param type AWS service client or resource to be created, eg 67 | #' \code{s3} 68 | #' @param cache booelan flag for caching the client or resource in the 69 | #' package namespace. For (internal) package functions, it's best 70 | #' to set to \code{TRUE} to avoid reinitializing the 71 | #' client/resource, but for custom use and when you need to use 72 | #' multiple clients for the same service in parallel (eg working 73 | #' with different regions etc), you might want to set this to 74 | #' \code{FALSE} 75 | #' @param ... further parameters passed to the \code{client} or 76 | #' \code{resource}, eg \code{endpoint_url} 77 | #' @references 78 | #' \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html} 79 | #' @return cached AWS client 80 | #' @importFrom checkmate assert_string assert_flag 81 | #' @export 82 | botor_client <- function(service, type = c('client', 'resource'), cache = TRUE, ...) { # nolint 83 | 84 | assert_string(service) 85 | assert_flag(cache) 86 | type <- match.arg(type) 87 | 88 | client <- tryCatch( 89 | get(service, envir = clients, inherits = FALSE), 90 | error = function(e) NULL) 91 | 92 | client_params <- list(...) 93 | if (cache == TRUE && !is.null(client) && 94 | attr(client, 'uuid') == botor_session_uuid() && 95 | ## compare custom client/resource parameters 96 | all.equal(client_params, attr(client, 'botor_client_params')) && 97 | ## compare serialized configs 98 | all.equal(client_params$config$`__dict__`, attr(client, 'botor_client_params')$config$`__dict__`)) { 99 | return(client) 100 | } 101 | 102 | if (type == 'client') { 103 | client <- botor()$client(service, ...) 104 | } else { 105 | client <- botor()$resource(service, ...) 106 | } 107 | attr(client, 'botor_client_params') <- client_params 108 | 109 | if (cache == FALSE) { 110 | return(client) 111 | } 112 | 113 | assign( 114 | x = service, 115 | value = structure( 116 | client, 117 | uuid = botor_session_uuid()), 118 | envir = clients) 119 | return(client) 120 | 121 | } 122 | -------------------------------------------------------------------------------- /R/checkmate.R: -------------------------------------------------------------------------------- 1 | #' Check if an argument looks like an S3 bucket 2 | #' @param x string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key 3 | #' @export 4 | #' @importFrom checkmate makeAssertionFunction makeTestFunction makeExpectationFunction makeAssertion makeExpectation check_string vname 5 | #' @examples 6 | #' check_s3_uri('s3://foo/bar') 7 | #' check_s3_uri('https://foo/bar') 8 | #' \dontrun{ 9 | #' assert_s3_uri('https://foo/bar') 10 | #' } 11 | #' @aliases check_s3_uri assert_s3_uri test_s3_uri expect_s3_uri 12 | check_s3_uri <- function(x) { 13 | regex <- '^s3://[a-z0-9][a-z0-9\\.-]+[a-z0-9](/(.*)?)?$' 14 | check <- check_string(x, pattern = regex) 15 | if (isTRUE(check)) { 16 | return(TRUE) 17 | } 18 | paste('Does not seem to be an S3 URI as per regular expression:', shQuote(regex)) 19 | } 20 | #' @export 21 | assert_s3_uri <- makeAssertionFunction(check_s3_uri) 22 | #' @export 23 | test_s3_uri <- makeTestFunction(check_s3_uri) 24 | #' @export 25 | expect_s3_uri <- makeExpectationFunction(check_s3_uri) 26 | -------------------------------------------------------------------------------- /R/iam.R: -------------------------------------------------------------------------------- 1 | #' The default, fork-safe IAM client on the top of \code{botor} 2 | #' @return \code{botocore.client.IAM} 3 | #' @export 4 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/iam.html} 5 | iam <- function() { 6 | botor_client('iam', type = 'client') 7 | } 8 | 9 | 10 | #' Retrieves information about the specified IAM user, including the user's creation date, path, unique ID, and ARN 11 | #' @param ... optional extra arguments passed 12 | #' @return list 13 | #' @export 14 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/iam.html#IAM.Client.get_user} 15 | iam_get_user <- function(...) { 16 | iam()$get_user(...) 17 | } 18 | 19 | #' Get the current AWS username 20 | #' @return string 21 | #' @export 22 | #' @seealso \code{\link{sts_whoami}} 23 | iam_whoami <- function() { 24 | iam_get_user()$User$UserName 25 | } 26 | 27 | 28 | #' Returns details about the IAM user or role whose credentials are used to call the operation 29 | #' @return \code{list} with \code{UserId}, \code{Account} and \code{Arn} 30 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.get_caller_identity} 31 | #' @export 32 | #' @seealso \code{\link{iam_whoami}} 33 | sts_whoami <- function() { 34 | botor_client('sts', type = 'client')$get_caller_identity() 35 | } 36 | -------------------------------------------------------------------------------- /R/kinesis.R: -------------------------------------------------------------------------------- 1 | #' The default, fork-safe Kinesis client on the top of \code{botor} 2 | #' @return \code{botocore.client.Kinesis} 3 | #' @export 4 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html} 5 | kinesis <- function() { 6 | botor_client('kinesis', type = 'client') 7 | } 8 | 9 | 10 | #' Describes the specified Kinesis data stream 11 | #' @param stream the name of the stream to describe 12 | #' @export 13 | #' @return list 14 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.describe_stream} 15 | kinesis_describe_stream <- function(stream) { 16 | kinesis()$describe_stream(StreamName = stream) 17 | } 18 | 19 | 20 | #' Writes a single data record into an Amazon Kinesis data stream 21 | #' @inheritParams kinesis_describe_stream 22 | #' @param data the data blob (<1 MB) to put into the record, which is 23 | #' base64-encoded when the blob is serialized 24 | #' @param partition_key Unicode string with a maximum length limit of 25 | #' 256 characters determining which shard in the stream the data 26 | #' record is assigned to 27 | #' @param ... optional further parameters, such as 28 | #' \code{ExplicitHashKey} or \code{SequenceNumberForOrdering} 29 | #' @export 30 | #' @return list of \code{ShardId}, \code{SequenceNumber} and 31 | #' \code{EncryptionType} 32 | #' @references 33 | #' \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.put_record} 34 | kinesis_put_record <- function(stream, data, partition_key, ...) { 35 | kinesis()$put_record(StreamName = stream, Data = data, PartitionKey = partition_key) 36 | } 37 | 38 | #' Gets an Amazon Kinesis shard iterator 39 | #' @inheritParams kinesis_put_record 40 | #' @param shard the shard ID of the Kinesis Data Streams shard to get 41 | #' the iterator for 42 | #' @param shard_iterator_type determines how the shard iterator is 43 | #' used to start reading data records from the shard 44 | #' @param ... optional further parameters, such as 45 | #' \code{StartingSequenceNumber} or \code{Timestamp} 46 | #' @export 47 | #' @return list of \code{ShardIterator} 48 | #' @references 49 | #' \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.get_shard_iterator} 50 | #' @seealso \code{\link{kinesis_get_records}} 51 | kinesis_get_shard_iterator <- function(stream, shard, shard_iterator_type = c( 52 | 'TRIM_HORIZON', 'LATEST', 53 | 'AT_SEQUENCE_NUMBER', 'AFTER_SEQUENCE_NUMBER', 'AT_TIMESTAMP'), ...) { 54 | shard_iterator_type <- match.arg(shard_iterator_type) 55 | kinesis()$get_shard_iterator(StreamName = stream, ShardId = shard, ShardIteratorType = shard_iterator_type, ...) 56 | } 57 | 58 | #' Gets data records from a Kinesis data stream's shard 59 | #' @param shard_iterator the position in the shard from which you want 60 | #' to start sequentially reading data records, usually provided by 61 | #' \code{\link{kinesis_get_shard_iterator}} 62 | #' @param limit maximum number of records to return 63 | #' @export 64 | #' @return list of \code{Records}, \code{NextShardIterator} and 65 | #' \code{MillisBehindLatest} 66 | #' @references 67 | #' \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.get_records} 68 | #' @examples \dontrun{ 69 | #' botor(profile_name = 'botor-tester') 70 | #' iterator <- kinesis_get_shard_iterator(stream = 'botor-tester', shard = '0') 71 | #' kinesis_get_records(iterator$ShardIterator) 72 | #' } 73 | kinesis_get_records <- function(shard_iterator, limit = 25L) { 74 | kinesis()$get_records(ShardIterator = shard_iterator, Limit = limit) 75 | } 76 | -------------------------------------------------------------------------------- /R/kms.R: -------------------------------------------------------------------------------- 1 | #' The default, fork-safe KMS client on the top of \code{botor} 2 | #' @return \code{botocore.client.KMS} 3 | #' @export 4 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kms.html} 5 | kms <- function() { 6 | botor_client('kms', type = 'client') 7 | } 8 | 9 | 10 | #' Encrypt plain text via KMS 11 | #' @param key the KMS customer master key identifier as a fully 12 | #' specified Amazon Resource Name (eg 13 | #' \code{arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012}) 14 | #' or an alias with the \code{alias/} prefix (eg 15 | #' \code{alias/foobar}) 16 | #' @param text max 4096 bytes long string, eg an RSA key, a database 17 | #' password, or other sensitive customer information 18 | #' @param simplify returns Base64-encoded text instead of raw list 19 | #' @return string or \code{list} 20 | #' @export 21 | #' @seealso \code{\link{kms_decrypt}} 22 | kms_encrypt <- function(key, text, simplify = TRUE) { 23 | assert_string(key) 24 | assert_string(text) 25 | assert_flag(simplify) 26 | res <- kms()$encrypt(KeyId = key, Plaintext = charToRaw(text)) 27 | if (simplify == TRUE) { 28 | res <- res$CiphertextBlob 29 | res <- base64_enc(res) 30 | res <- coerce_bytes_literals_to_string(res) 31 | } 32 | res 33 | } 34 | 35 | 36 | #' Decrypt cipher into plain text via KMS 37 | #' @param cipher Base64-encoded ciphertext 38 | #' @param simplify returns decrypted plain-text instead of raw list 39 | #' @return decrypted text as string or \code{list} 40 | #' @export 41 | #' @seealso \code{\link{kms_encrypt}} 42 | kms_decrypt <- function(cipher, simplify = TRUE) { 43 | assert_string(cipher) 44 | assert_flag(simplify) 45 | res <- kms()$decrypt(CiphertextBlob = base64_dec(cipher)) 46 | if (simplify == TRUE) { 47 | res <- res$Plaintext 48 | res <- coerce_bytes_literals_to_string(res) 49 | } 50 | res 51 | } 52 | 53 | 54 | #' Generate a data encryption key for envelope encryption via KMS 55 | #' @param bytes the required length of the data encryption key in 56 | #' bytes (so provide eg \code{64L} for a 512-bit key) 57 | #' @return \code{list} of the Base64-encoded encrypted version of the 58 | #' data encryption key (to be stored on disk), the \code{raw} 59 | #' object of the encryption key and the KMS customer master key 60 | #' used to generate this object 61 | #' @inheritParams kms_encrypt 62 | #' @export 63 | #' @importFrom checkmate assert_integer 64 | kms_generate_data_key <- function(key, bytes = 64L) { 65 | 66 | assert_string(key) 67 | assert_integer(bytes) 68 | 69 | data_key <- kms()$generate_data_key(KeyId = key, NumberOfBytes = bytes) 70 | 71 | list( 72 | cipher = coerce_bytes_literals_to_string(base64_enc(data_key$CiphertextBlob)), 73 | key = data_key$KeyId, 74 | text = require_python_builtins()$bytearray(data_key$Plaintext)) 75 | 76 | } 77 | 78 | 79 | #' Encrypt file via KMS 80 | #' @param file file path 81 | #' @return two files created with \code{enc} (encrypted data) and 82 | #' \code{key} (encrypted key) extensions 83 | #' @inheritParams kms_encrypt 84 | #' @export 85 | #' @seealso \code{\link{kms_encrypt}} \code{\link{kms_decrypt_file}} 86 | #' @importFrom checkmate assert_file_exists 87 | kms_encrypt_file <- function(key, file) { 88 | 89 | assert_string(key) 90 | assert_file_exists(file) 91 | if (!requireNamespace('digest', quietly = TRUE)) { 92 | stop('The digest package is required to encrypt files') 93 | } 94 | 95 | ## load the file to be encrypted 96 | msg <- readBin(file, 'raw', n = file.size(file)) 97 | ## the text length must be a multiple of 16 bytes 98 | ## so let's Base64-encode just in case 99 | msg <- charToRaw(base64_enc(msg)) 100 | msg <- c(msg, as.raw(rep(as.raw(0), 16 - length(msg) %% 16))) 101 | 102 | ## generate encryption key 103 | key <- kms_generate_data_key(key, bytes = 32L) 104 | 105 | ## encrypt file using the encryption key 106 | aes <- digest::AES(key$text, mode = 'ECB') 107 | writeBin(aes$encrypt(msg), paste0(file, '.enc')) 108 | 109 | ## store encrypted key 110 | cat(key$cipher, file = paste0(file, '.key')) 111 | 112 | ## return file paths 113 | list( 114 | file = file, 115 | encrypted = paste0(file, '.enc'), 116 | key = paste0(file, '.key') 117 | ) 118 | 119 | } 120 | 121 | 122 | #' Decrypt file via KMS 123 | #' @param file base file path (without the \code{enc} or \code{key} suffix) 124 | #' @param return where to place the encrypted file (defaults to \code{file}) 125 | #' @return decrypted file path 126 | #' @export 127 | #' @seealso \code{\link{kms_encrypt}} \code{\link{kms_encrypt_file}} 128 | kms_decrypt_file <- function(file, return = file) { 129 | 130 | if (!file.exists(paste0(file, '.enc'))) { 131 | stop(paste('Encrypted file does not exist:', paste0(file, '.enc'))) 132 | } 133 | if (!file.exists(paste0(file, '.key'))) { 134 | stop(paste('Encryption key does not exist:', paste0(file, '.key'))) 135 | } 136 | if (file.exists(return)) { 137 | stop(paste('Encrypted file already exists:', return)) 138 | } 139 | if (!requireNamespace('digest', quietly = TRUE)) { 140 | stop('The digest package is required to encrypt files') 141 | } 142 | 143 | ## load the encryption key 144 | key <- charToRaw(kms_decrypt(paste(readLines(paste0(file, '.key'), warn = FALSE), collapse = ''))) 145 | 146 | ## load the encrypted file 147 | msg <- readBin(paste0(file, '.enc'), 'raw', n = file.size(paste0(file, '.enc'))) 148 | 149 | ## decrypt the file using the encryption key 150 | aes <- digest::AES(key, mode = 'ECB') 151 | msg <- aes$decrypt(msg, raw = TRUE) 152 | 153 | msg <- base64_dec(rawToChar(msg)) 154 | 155 | ## Base64-decode and return 156 | writeBin(msg, return) 157 | 158 | ## return file paths 159 | return 160 | 161 | } 162 | -------------------------------------------------------------------------------- /R/python_modules.R: -------------------------------------------------------------------------------- 1 | python_modules <- new.env() 2 | python_builtins <- NULL 3 | 4 | #' Imports and caches a Python module 5 | #' @param module a Python module name 6 | #' @return imported Python module 7 | #' @keywords internal 8 | require_python_module <- function(module) { 9 | 10 | tryCatch( 11 | get(module, envir = python_modules, inherits = FALSE), 12 | error = function(e) { 13 | loaded_module <- import(module = module) 14 | assign(x = module, value = loaded_module, envir = python_modules) 15 | loaded_module 16 | }) 17 | 18 | } 19 | 20 | #' Imports and caches a Python module 21 | #' @return imported Python module 22 | #' @keywords internal 23 | require_python_builtins <- function() { 24 | utils::assignInMyNamespace( 25 | 'python_builtins', 26 | reticulate::import_builtins()) 27 | python_builtins 28 | } 29 | -------------------------------------------------------------------------------- /R/s3.R: -------------------------------------------------------------------------------- 1 | #' The default, fork-safe Amazon Simple Storage Service (S3) client on the top of \code{botor} 2 | #' @return \code{s3.ServiceResource} 3 | #' @export 4 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#service-resource} 5 | #' @importFrom logger log_trace log_debug log_info log_warn log_error 6 | #' @param disable_signing boolean if requests should be signed. Set to \code{FALSE} when interacting with public S3 buckets requiring unauthenticated access. 7 | s3 <- function(disable_signing = getOption('botor-s3-disable-signing')) { 8 | botocore <- require_python_module('botocore') 9 | botor_client('s3', type = 'resource', config = botocore$config$Config( 10 | signature_version = if (isTRUE(disable_signing)) botocore$UNSIGNED else 's3v4' 11 | )) 12 | } 13 | 14 | 15 | #' Split the bucket name and object key from the S3 URI 16 | #' @inheritParams s3_object 17 | #' @return list 18 | #' @keywords internal 19 | s3_split_uri <- function(uri) { 20 | assert_s3_uri(uri) 21 | ## kill URI schema 22 | path <- sub('^s3://', '', uri) 23 | ## bucket name is anything before the first slash 24 | bucket <- sub('/.*$', '', path) 25 | ## object key is the remaining bit 26 | key <- sub(paste0('^', bucket, '/?'), '', path) 27 | list(bucket_name = bucket, key = key) 28 | } 29 | 30 | 31 | #' Create an S3 Object reference from an URI 32 | #' @param uri string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key 33 | #' @return \code{s3$Object} 34 | #' @export 35 | s3_object <- function(uri) { 36 | uri_parts <- s3_split_uri(uri) 37 | s3()$Object( 38 | bucket_name = uri_parts$bucket_name, 39 | key = uri_parts$key) 40 | } 41 | 42 | 43 | #' List all S3 buckets 44 | #' @param simplify return bucket names as a character vector 45 | #' @return \code{list} of \code{boto3.resources.factory.s3.Bucket} or a character vector 46 | #' @export 47 | #' @importFrom reticulate iter_next 48 | s3_list_buckets <- function(simplify = TRUE) { 49 | log_trace('Listing all S3 buckets ...') 50 | buckets <- iter_next(s3()$buckets$pages()) 51 | log_debug('Found %s S3 buckets', length(buckets)) 52 | if (simplify == TRUE) { 53 | buckets <- sapply(buckets, `[[`, 'name') 54 | } 55 | buckets 56 | } 57 | 58 | 59 | #' Download a file from S3 60 | #' @inheritParams s3_object 61 | #' @param file string, location of local file 62 | #' @param force boolean, overwrite local file if exists 63 | #' @export 64 | #' @importFrom checkmate assert_string assert_directory_exists assert_flag 65 | #' @return invisibly \code{file} 66 | #' @examples \dontrun{ 67 | #' s3_download_file('s3://botor/example-data/mtcars.csv', tempfile()) 68 | #' } 69 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.download_file} 70 | s3_download_file <- function(uri, file, force = TRUE) { 71 | assert_string(file) 72 | assert_directory_exists(dirname(file)) 73 | log_trace('Downloading %s to %s ...', uri, shQuote(file)) 74 | if (force == FALSE && file.exists(file)) { 75 | stop(paste(file, 'already exists')) 76 | } 77 | assert_s3_uri(uri) 78 | assert_flag(force) 79 | s3object <- s3_object(uri) 80 | trypy(s3object$download_file(file)) 81 | log_debug('Downloaded %s bytes from %s and saved at %s', file.info(file)$size, uri, shQuote(file)) 82 | invisible(file) 83 | } 84 | 85 | 86 | #' Download and read a file from S3, then clean up 87 | #' @inheritParams s3_download_file 88 | #' @param fun R function to read the file, eg \code{fromJSON}, \code{stream_in}, \code{fread} or \code{readRDS} 89 | #' @param ... optional params passed to \code{fun} 90 | #' @param extract optionally extract/decompress the file after downloading from S3 but before passing to \code{fun} 91 | #' @return R object 92 | #' @export 93 | #' @examples \dontrun{ 94 | #' s3_read('s3://botor/example-data/mtcars.csv', read.csv) 95 | #' s3_read('s3://botor/example-data/mtcars.csv', data.table::fread) 96 | #' s3_read('s3://botor/example-data/mtcars.csv2', read.csv2) 97 | #' s3_read('s3://botor/example-data/mtcars.RDS', readRDS) 98 | #' s3_read('s3://botor/example-data/mtcars.json', jsonlite::fromJSON) 99 | #' s3_read('s3://botor/example-data/mtcars.jsonl', jsonlite::stream_in) 100 | #' 101 | #' ## read compressed data 102 | #' s3_read('s3://botor/example-data/mtcars.csv.gz', read.csv, extract = 'gzip') 103 | #' s3_read('s3://botor/example-data/mtcars.csv.gz', data.table::fread, extract = 'gzip') 104 | #' s3_read('s3://botor/example-data/mtcars.csv.bz2', read.csv, extract = 'bzip2') 105 | #' s3_read('s3://botor/example-data/mtcars.csv.xz', read.csv, extract = 'xz') 106 | #' } 107 | s3_read <- function(uri, fun, ..., extract = c('none', 'gzip', 'bzip2', 'xz')) { 108 | 109 | t <- tempfile() 110 | on.exit({ 111 | log_trace('Deleted %s', t) 112 | unlink(t) 113 | }) 114 | 115 | s3_download_file(uri, t) 116 | 117 | ## decompress/extract downloaded file 118 | extract <- match.arg(extract) 119 | if (extract != 'none') { 120 | 121 | filesize <- file.info(t)$size 122 | 123 | ## gzfile can handle bzip2 and xz as well 124 | filecon <- gzfile(t, open = 'rb') 125 | 126 | ## paginate read compressed file by 1MB chunks 127 | ## as we have no idea about the size of the uncompressed data 128 | chunksize <- 1024L * 1024L 129 | chunks <- list(readBin(filecon, 'raw', n = chunksize)) 130 | while (length(chunks[[length(chunks)]]) == chunksize) { 131 | chunks[[length(chunks) + 1]] <- readBin(filecon, 'raw', n = chunksize) 132 | } 133 | filecontent <- unlist(chunks, use.names = FALSE) 134 | close(filecon) 135 | 136 | ## overwrite compressed temp file with uncompressed data 137 | writeBin(filecontent, t) 138 | log_trace('Decompressed %s via %s from %s to %s bytes', t, extract, filesize, file.info(t)$size) 139 | 140 | } 141 | 142 | if (deparse(substitute(fun)) %in% c('jsonlite::stream_in', 'stream_in')) { 143 | t <- file(t) 144 | } 145 | 146 | fun(t, ...) 147 | 148 | } 149 | 150 | 151 | #' Upload a file to S3 152 | #' @inheritParams s3_object 153 | #' @param file string, location of local file 154 | #' @param content_type content type of a file that is auto-guess if omitted 155 | #' @param ... optional parameters passed to the \code{upload_file} method's \code{ExtraArgs} 156 | #' @export 157 | #' @importFrom checkmate assert_file_exists 158 | #' @importFrom reticulate import 159 | #' @return invisibly \code{uri} 160 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.upload_file} 161 | #' @seealso \code{\link{s3_download_file}} 162 | #' @examples \dontrun{ 163 | #' t <- tempfile() 164 | #' write.csv(mtcars, t, row.names = FALSE) 165 | #' s3_upload_file(t, 's3://botor/example-data/mtcars.csv') 166 | #' unlink(t) 167 | #' ## note that s3_write would have been a much nicer solution for the above 168 | #' } 169 | s3_upload_file <- function(file, uri, content_type = mime_guess(file), ...) { 170 | 171 | assert_string(file) 172 | assert_file_exists(file) 173 | assert_s3_uri(uri) 174 | assert_string(content_type, na.ok = TRUE) 175 | 176 | ## set content type 177 | if (!is.na(content_type)) { 178 | extra_args <- list(ContentType = content_type, ...) 179 | } else { 180 | extra_args <- list(...) 181 | } 182 | 183 | log_trace('Uploading %s to %s ...', shQuote(file), uri) 184 | s3object <- s3_object(uri) 185 | trypy(s3object$upload_file(file, ExtraArgs = extra_args)) 186 | log_debug( 187 | 'Uploaded %s bytes from %s to %s with %s content type', 188 | file.info(file)$size, shQuote(file), uri, shQuote(content_type)) 189 | invisible(uri) 190 | 191 | } 192 | 193 | 194 | #' Write an R object into S3 195 | #' @param x R object 196 | #' @param fun R function with \code{file} argument to serialize 197 | #' \code{x} to disk before uploading, eg \code{write.csv}, 198 | #' \code{write_json}, \code{stream_out} or \code{saveRDS} 199 | #' @param compress optionally compress the file before uploading to 200 | #' S3. If compression is used, it's better to include the related 201 | #' file extension in \code{uri} as well (that is not done 202 | #' automatically). 203 | #' @param ... optional further arguments passed to \code{fun} 204 | #' @inheritParams s3_object 205 | #' @export 206 | #' @note The temp file used for this operation is automatically 207 | #' removed. 208 | #' @examples \dontrun{ 209 | #' s3_write(mtcars, write.csv, 's3://botor/example-data/mtcars.csv', row.names = FALSE) 210 | #' s3_write(mtcars, write.csv2, 's3://botor/example-data/mtcars.csv2', row.names = FALSE) 211 | #' s3_write(mtcars, jsonlite::write_json, 's3://botor/example-data/mtcars.json', row.names = FALSE) 212 | #' s3_write(mtcars, jsonlite::stream_out, 's3://botor/example-data/mtcars.jsonl', row.names = FALSE) 213 | #' s3_write(mtcars, saveRDS, 's3://botor/example-data/mtcars.RDS') 214 | #' 215 | #' ## compress file after writing to disk but before uploading to S3 216 | #' s3_write(mtcars, write.csv, 's3://botor/example-data/mtcars.csv.gz', 217 | #' compress = 'gzip', row.names = FALSE) 218 | #' s3_write(mtcars, write.csv, 's3://botor/example-data/mtcars.csv.bz2', 219 | #' compress = 'bzip2', row.names = FALSE) 220 | #' s3_write(mtcars, write.csv, 's3://botor/example-data/mtcars.csv.xz', 221 | #' compress = 'xz', row.names = FALSE) 222 | #' } 223 | s3_write <- function(x, fun, uri, compress = c('none', 'gzip', 'bzip2', 'xz'), ...) { 224 | 225 | t <- tempfile() 226 | on.exit({ 227 | log_trace('Deleted %s', t) 228 | unlink(t) 229 | }) 230 | 231 | if (deparse(substitute(fun)) %in% c('jsonlite::write_json', 'write_json')) { 232 | fun(x, path = t, ...) 233 | } else { 234 | if (deparse(substitute(fun)) %in% c('jsonlite::stream_out', 'stream_out')) { 235 | fun(x, con = file(t), ...) 236 | } else { 237 | fun(x, file = t, ...) 238 | } 239 | } 240 | log_trace('Wrote %s bytes to %s', file.info(t)$size, t) 241 | 242 | compress <- match.arg(compress) 243 | if (compress != 'none') { 244 | filesize <- file.info(t)$size 245 | filecontent <- readBin(t, 'raw', n = filesize) 246 | compressor <- switch( 247 | compress, 248 | 'gzip' = gzfile, 249 | 'bzip2' = bzfile, 250 | 'xz' = xzfile) 251 | filecon <- compressor(t, open = 'wb') 252 | ## overwrite 253 | writeBin(filecontent, filecon) 254 | close(filecon) 255 | log_trace('Compressed %s via %s from %s to %s bytes', t, compress, filesize, file.info(t)$size) 256 | } 257 | 258 | s3_upload_file(t, uri) 259 | 260 | } 261 | 262 | 263 | #' List objects at an S3 path 264 | #' @param uri string, should start with \code{s3://}, then bucket name 265 | #' and optional object key prefix 266 | #' @return \code{data.frame} with \code{bucket_name}, object 267 | #' \code{key}, \code{uri} (that can be directly passed to eg 268 | #' \code{\link{s3_read}}), \code{size} in bytes, \code{owner} and 269 | #' \code{last_modified} timestamp 270 | #' @export 271 | #' @importFrom reticulate iterate 272 | s3_ls <- function(uri) { 273 | 274 | log_trace('Recursive listing of files in %s', uri) 275 | uri_parts <- s3_split_uri(uri) 276 | 277 | objects <- s3()$Bucket(uri_parts$bucket_name)$objects 278 | objects <- objects$filter(Prefix = uri_parts$key) 279 | objects <- trypy(iterate(objects$pages(), simplify = FALSE)) 280 | objects <- unlist(objects, recursive = FALSE) 281 | 282 | objects <- do.call(rbind, lapply(objects, function(object) { 283 | object <- object$meta$`__dict__` 284 | data.frame( 285 | bucket_name = uri_parts$bucket_name, 286 | key = object$data$Key, 287 | uri = file.path('s3:/', uri_parts$bucket_name, object$data$Key), 288 | size = object$data$Size, 289 | ## owner might be missing, so coerce NULL to NA 290 | owner = c(object$data$Owner$DisplayName, NA_character_)[1], 291 | last_modified = object$data$LastModified, 292 | stringsAsFactors = FALSE) 293 | })) 294 | 295 | log_debug('Found %s item(s) in %s', nrow(objects), uri) 296 | objects 297 | 298 | } 299 | 300 | 301 | #' Checks if an object exists in S3 302 | #' @inheritParams s3_object 303 | #' @export 304 | #' @return boolean 305 | #' @examples \dontrun{ 306 | #' s3_exists('s3://botor/example-data/mtcars.csv') 307 | #' s3_exists('s3://botor/example-data/UNDEFINED.CSVLX') 308 | #' } 309 | s3_exists <- function(uri) { 310 | assert_s3_uri(uri) 311 | uri_parts <- s3_split_uri(uri) 312 | log_trace('Checking if object at %s exist ...', uri) 313 | head <- tryCatch( 314 | trypy(s3()$meta$client$head_object(Bucket = uri_parts$bucket_name, Key = uri_parts$key)), 315 | error = function(e) e) 316 | found <- !inherits(head, 'error') 317 | log_debug('%s %s', uri, ifelse(found, 'found', 'not found')) 318 | invisible(found) 319 | } 320 | 321 | 322 | #' Copy an object from one S3 location to another 323 | #' @param uri_source string, location of the source file 324 | #' @param uri_target string, location of the target file 325 | #' @export 326 | #' @return invisibly \code{uri_target} 327 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Object.copy} 328 | s3_copy <- function(uri_source, uri_target) { 329 | assert_s3_uri(uri_source) 330 | assert_s3_uri(uri_target) 331 | log_trace('Copying %s to %s ...', uri_source, uri_target) 332 | source <- s3_split_uri(uri_source) 333 | target <- s3_object(uri_target) 334 | trypy(target$copy(list(Bucket = source$bucket_name, Key = source$key))) 335 | log_debug('Copied %s to %s', uri_source, uri_target) 336 | invisible(uri_target) 337 | } 338 | 339 | 340 | #' Delete an object stored in S3 341 | #' @inheritParams s3_object 342 | #' @export 343 | s3_delete <- function(uri) { 344 | assert_s3_uri(uri) 345 | log_trace('Deleting %s ...', uri) 346 | s3_object(uri)$delete() 347 | log_debug('Deleted %s', uri) 348 | } 349 | 350 | 351 | #' Sets tags on s3 object overwriting all existing tags. Note: tags 352 | #' and metadata tags are not the same 353 | #' @param uri string, URI of an S3 object, should start with 354 | #' \code{s3://}, then bucket name and object key 355 | #' @param tags named character vector, e.g. \code{c(my_first_name = 356 | #' 'my_first_value', my_second_name = 'my_second_value')} where 357 | #' names are the tag names and values are the tag values. 358 | #' @export 359 | #' @references 360 | #' \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object_tagging} 361 | s3_put_object_tagging <- function(uri, tags) { 362 | assert_s3_uri(uri) 363 | tag_set <- mapply(list, Key = names(tags), Value = tags, SIMPLIFY = FALSE, USE.NAMES = FALSE) 364 | uri_parts <- s3_split_uri(uri) 365 | s3()$meta$client$put_object_tagging( 366 | Bucket = uri_parts$bucket_name, 367 | Key = uri_parts$key, 368 | Tagging = list('TagSet' = tag_set) 369 | ) 370 | } 371 | -------------------------------------------------------------------------------- /R/sm.R: -------------------------------------------------------------------------------- 1 | #' The default, fork-safe AWS Systems Manager (SecretManager) client on the top of \code{botor} 2 | #' @return \code{botocore.client.secretsmanager} 3 | #' @export 4 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html} 5 | sm <- function() { 6 | botor_client('secretsmanager', type = 'client') 7 | } 8 | 9 | 10 | #' Read AWS System Manager's Secrets Manager via Secret Manager 11 | #' @param path name/path of the key to be read 12 | #' @param key single key or a vector of keys. 13 | #' @param parse_json logical. Default TRUE 14 | #' @importFrom checkmate assert_flag assert_vector 15 | #' @importFrom jsonlite fromJSON 16 | #' @importFrom logger log_warn 17 | #' @return (optionally decrypted) value 18 | #' @export 19 | sm_get_secret <- function(path, key = NULL, parse_json = TRUE) { 20 | assert_flag(parse_json) 21 | assert_vector(key, null.ok = TRUE) 22 | 23 | if (!is.null(key) && parse_json == FALSE) { 24 | stop('Need to set parse_json=TRUE when extracting keys') 25 | } 26 | 27 | log_trace('Looking up SecretId %s in AWS Secrets Manager', path) 28 | 29 | resp <- trypy(sm()$get_secret_value(SecretId = path))$SecretString 30 | 31 | if (parse_json) { 32 | resp <- fromJSON(resp) 33 | } 34 | for (k in key) { 35 | resp <- resp[[k]] 36 | } 37 | 38 | resp 39 | } 40 | -------------------------------------------------------------------------------- /R/ssm.R: -------------------------------------------------------------------------------- 1 | #' The default, fork-safe AWS Systems Manager (SSM) client on the top of \code{botor} 2 | #' @return \code{botocore.client.SSM} 3 | #' @export 4 | #' @references \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html} 5 | ssm <- function() { 6 | botor_client('ssm', type = 'client') 7 | } 8 | 9 | 10 | #' Read AWS System Manager's Parameter Store 11 | #' @param path name/path of the key to be read 12 | #' @param decrypt decrypt the value or return the raw ciphertext 13 | #' @return (optionally decrypted) value 14 | #' @export 15 | ssm_get_parameter <- function(path, decrypt = TRUE) { 16 | log_trace("Looking up %s in AWS System Manager's Parameter Store", path) 17 | trypy(ssm()$get_parameter( 18 | Name = path, 19 | WithDecryption = decrypt)$Parameter$Value) 20 | } 21 | -------------------------------------------------------------------------------- /R/utils.R: -------------------------------------------------------------------------------- 1 | #' Extract error message from a Python exception 2 | #' @param expression R expression 3 | #' @return error 4 | #' @keywords internal 5 | #' @examples \dontrun{ 6 | #' trypy(botor()$resource('foobar')) 7 | #' trypy(sum(1:2)) 8 | #' trypy(sum(1:1foo)) 9 | #' } 10 | #' @importFrom reticulate py_clear_last_error py_last_error 11 | trypy <- function(expression) { 12 | py_clear_last_error() 13 | tryCatch( 14 | eval.parent(expression), 15 | error = function(e) { 16 | e$call <- sys.calls()[[1]] 17 | if (sys.nframe() > 5) { 18 | e$call <- sys.calls()[[sys.nframe() - 5]] 19 | } 20 | pe <- py_last_error() 21 | if (!is.null(pe)) { 22 | e$message <- paste0('Python `', pe$type, '`: ', pe$value) 23 | } 24 | stop(e) 25 | }) 26 | } 27 | 28 | 29 | #' Base64-encode raw bytes using Python's base64 module 30 | #' @param text \code{raw}, R string or Python string 31 | #' @return string 32 | #' @keywords internal 33 | #' @seealso \code{\link{base64_dec}} 34 | #' @importFrom checkmate assert_class 35 | base64_enc <- function(text) { 36 | as.character(require_python_module('base64')$b64encode(text)) 37 | } 38 | 39 | 40 | #' Base64-decode a string into raw bytes using Python's base64 module 41 | #' @param text string 42 | #' @return \code{raw} bytes 43 | #' @keywords internal 44 | #' @examples \dontrun{ 45 | #' botor:::base64_dec(botor:::base64_enc(charToRaw('foobar'))) 46 | #' } 47 | #' @seealso \code{\link{base64_enc}} 48 | #' @importFrom reticulate import 49 | base64_dec <- function(text) { 50 | assert_string(text) 51 | require_python_builtins()$bytearray(require_python_module('base64')$b64decode(text)) 52 | } 53 | 54 | 55 | #' Generate UUID using Python's uuid module 56 | #' @return string 57 | #' @keywords internal 58 | #' @importFrom reticulate py_str 59 | uuid <- function() { 60 | py_str(require_python_module('uuid')$uuid1()) 61 | } 62 | 63 | 64 | #' Guess the type of a file based on the filename using \code{mimetypes} Python module 65 | #' @param file path 66 | #' @return string 67 | #' @export 68 | #' @importFrom reticulate import 69 | mime_guess <- function(file) { 70 | 71 | content_type <- require_python_module('mimetypes')$guess_type(file)[[1]] 72 | 73 | ## return NA instead of NULL 74 | if (is.null(content_type)) { 75 | content_type <- NA 76 | } 77 | 78 | content_type 79 | 80 | } 81 | 82 | 83 | #' Looks up major version of Python (eg 2 or 3) 84 | #' @return number 85 | #' @keywords internal 86 | python_version <- function() { 87 | import('sys')$version_info$major 88 | } 89 | 90 | 91 | #' Transforms a python2 string literal or python3 bytes literal into an R string 92 | #' 93 | #' This is useful to call eg for the KMS call, where python2 returns a 94 | #' string, but python3 returns bytes literals -- calling "decode" is 95 | #' tricky, but bytearray conversion, then passing the raw vector to R 96 | #' and converting that a string works. 97 | #' @param x string 98 | #' @return string 99 | #' @keywords internal 100 | coerce_bytes_literals_to_string <- function(x) { # nolint 101 | if (inherits(x, 'python.builtin.bytes')) { 102 | rawToChar(require_python_builtins()$bytearray(x)) 103 | } else { 104 | rawToChar(require_python_builtins()$bytearray(x, encoding = 'utf8')) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /R/zzz.R: -------------------------------------------------------------------------------- 1 | .onLoad <- function(libname, pkgname) { 2 | 3 | utils::assignInMyNamespace( 4 | 'boto3', 5 | reticulate::import( 6 | module = 'boto3', 7 | delay_load = list( 8 | on_error = function(e) stop(e$message) 9 | ))) 10 | 11 | ## although glue would be more convenient, 12 | ## but let's use the always available sprintf formatter function for logging 13 | logger::log_formatter(logger::formatter_sprintf, namespace = pkgname) 14 | logger::log_threshold(logger::DEBUG, namespace = pkgname) 15 | 16 | } 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # botor: Reticulate wrapper on 'boto3' botor website 2 | 3 | 4 | [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) ![CRAN](https://www.r-pkg.org/badges/version/botor) [![R-CMD-check](https://github.com/daroczig/botor/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/daroczig/botor/actions/workflows/R-CMD-check.yaml) [![Code Coverage](https://codecov.io/gh/daroczig/botor/branch/master/graph/badge.svg)](https://app.codecov.io/gh/daroczig/botor) 5 | 6 | 7 | This R package provides raw access to the 'Amazon Web Services' ('AWS') 'SDK' via the 'boto3' Python module and some convenient helper functions (currently for S3 and KMS) and workarounds, eg taking care of spawning new resources in forked R processes. 8 | 9 | ## Installation 10 | 11 | This package requires Python to be installed along with the `boto3` Python module, which can be installed from R via: 12 | 13 | ```r 14 | reticulate::py_install('boto3') 15 | ``` 16 | 17 | If that might result in technical problems that you cannot solve, then it's probably easier to install a standalone Python along with the system dependencies etc via [`rminiconda`](https://github.com/hafen/rminiconda). 18 | 19 | Once the Python dependencies are resolved, you can either install from CRAN or the most recent (development version) of `botor` can be installed from GitHub: 20 | 21 | ```r 22 | remotes::install_github('daroczig/botor') 23 | ``` 24 | 25 | ## Loading the package 26 | 27 | Loading the `botor` package might take a while as it will also `import` the `boto3` Python module in the background: 28 | 29 | ```r 30 | system.time(library(botor)) 31 | #> user system elapsed 32 | #> 1.131 0.250 1.191 33 | ``` 34 | 35 | ## Getting started 36 | 37 | Quick examples: 38 | 39 | 1. Check the currently used AWS user's name: 40 | 41 | ```r 42 | iam_whoami() 43 | #> [1] "gergely-dev" 44 | ``` 45 | 46 | 2. Read a `csv` file stored in S3 using a helper function: 47 | 48 | ```r 49 | s3_read('s3://botor/example-data/mtcars.csv', read.csv) 50 | #> mpg cyl disp hp drat wt qsec vs am gear carb 51 | #> 1 21.0 6 160.0 110 3.90 2.620 16.46 0 1 4 4 52 | #> 2 21.0 6 160.0 110 3.90 2.875 17.02 0 1 4 4 53 | #> ... 54 | ``` 55 | 56 | 3. Encrypt a string via KMS using a helper function: 57 | 58 | ```r 59 | kms_encrypt('alias/key', 'secret') 60 | #> [1] "QWERTY..." 61 | ``` 62 | 63 | 4. Get more info on the currently used AWS user calling the IAM client directly: 64 | 65 | ```r 66 | iam()$get_user() 67 | ``` 68 | 69 | 5. Create a new client to a service without helper functions: 70 | 71 | ```r 72 | ec2 <- botor_client('ec2') 73 | ec2$describe_vpcs() 74 | ``` 75 | 76 | ## AWS Auth 77 | 78 | The `botor` package by default will use the credentials and related options set in [environmental variables](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#configuration) or in the `~/.aws/config` and `~/.aws/credentials` files. If you need to specify a custom profile or AWS region etc, there are various options with different complexity and flexibility: 79 | 80 | * set the related environment variable(s) before loading `botor` 81 | * call the `botor()` function with the relevant argument to set the config of the default session for the `botor` helper functions, eg 82 | 83 | ```r 84 | botor(region_name = 'eu-west-42') 85 | ``` 86 | 87 | * if you need to manage multiple sessions, then use the raw `boto3` object from the `botor` package and `boto3.session.Session` to init these custom sessions and the required clients/resources on the top of those, eg 88 | 89 | ```r 90 | my_custom_session1 <- boto3$Session(region_name = 'us-west-1') 91 | my_custom_s3_client1 <- my_custom_session1$client('s3') 92 | my_custom_session2 <- boto3$Session(region_name = 'us-west-2') 93 | my_custom_s3_client2 <- my_custom_session2$client('s3') 94 | ``` 95 | 96 | ## Using the raw `boto3` module 97 | 98 | The `botor` package provides the `boto3` object with full access to the `boto3` Python SDK. Quick example on listing all S3 buckets: 99 | 100 | ```r 101 | library(botor) 102 | s3 <- boto3$resource('s3') 103 | library(reticulate) 104 | iter_next(s3$buckets$pages()) 105 | ``` 106 | 107 | Note that this approach requires a stable understanding of the `boto3` Python module, plus a decent familiarity with `reticulate` as well (see eg `iter_next`) -- so you might want to rather consider using the helper functions described below. 108 | 109 | ## Using the default `botor` session 110 | 111 | Calling `botor()` will provide you a default `boto3` session that is cached internally. You can always override the default session by calling `botor()` again with new arguments. See eg setting the default `boto3` session to use `us-west-2`: 112 | 113 | ```r 114 | botor(region_name = 'us-west-2') 115 | botor()$resource('s3') 116 | ``` 117 | 118 | A great advantage of using `botor()` instead of custom sessions is that it's fork-safe. See eg: 119 | 120 | ```r 121 | attr(botor(), 'pid') 122 | #> [1] 31225 123 | attr(botor(), 'pid') 124 | #> [1] 31225 125 | 126 | lapply(1:2, function(i) attr(botor(), 'pid')) 127 | #> [[1]] 128 | #> [1] 31225 129 | #> 130 | #> [[2]] 131 | #> [1] 31225 132 | 133 | mclapply(1:2, function(i) attr(botor(), 'pid'), mc.cores = 2) 134 | #> [[1]] 135 | #> [1] 13209 136 | #> 137 | #> [[2]] 138 | #> [1] 13210 139 | ``` 140 | 141 | ## Convenient helper functions 142 | 143 | Besides the `botor` pre-initialized default Boto3 session, the package also provides some further R helper functions for the most common AWS actions, like interacting with S3 or KMS. Note, that the list of these functions is pretty limited for now, but you can always fall back to the raw Boto3 functions if needed. PRs on new helper functions are appreciated :) 144 | 145 | Examples: 146 | 147 | 1. Listing all S3 buckets takes some time as it will first initialize the S3 Boto3 client in the background: 148 | 149 | ```r 150 | system.time(s3_list_buckets())[['elapsed']] 151 | #> [1] 1.426 152 | ``` 153 | 154 | 2. But the second query is much faster as reusing the same `s3` Boto3 resource: 155 | 156 | ```r 157 | system.time(s3_list_buckets())[['elapsed']] 158 | #> [1] 0.323 159 | ``` 160 | 161 | 3. Unfortunately, sharing the same Boto3 resource between (forked) processes is not ideal, so `botor` will take care of that by spawning new resources in the forked threads: 162 | 163 | ```r 164 | library(parallel) 165 | simplify2array(mclapply(1:4, function(i) system.time(s3_list_buckets())[['elapsed']], mc.cores = 2)) 166 | #> [1] 1.359 1.356 0.406 0.397 167 | ``` 168 | 169 | 4. Want to speed it up more? 170 | 171 | ```r 172 | library(memoise) 173 | s3_list_buckets <- memoise(s3_list_buckets) 174 | simplify2array(mclapply(1:4, function(i) system.time(s3_list_buckets())[['elapsed']], mc.cores = 2)) 175 | #> [1] 1.330 1.332 0.000 0.000 176 | ``` 177 | 178 | The currently supported resources and features via helper functions: https://daroczig.github.io/botor/reference/index.html 179 | 180 | ## Error handling 181 | 182 | The convenient helper functions try to suppress the boring Python traceback and provide you only the most relevant information on the error. If you want to see the full tracelog and more details after an error, call `reticulate::py_last_error()`. When working with the raw `boto3` wrapper, you may find `botor:::trypy` useful as well. 183 | 184 | ```r 185 | s3_download_file('s3://bottttor/example-data/mtcars.csv', tempfile()) 186 | #> Error in s3_download_file("s3://bottttor/example-data/mtcars.csv", tempfile()) : 187 | #> Python `ClientError`: An error occurred (404) when calling the HeadObject operation: Not Found 188 | 189 | s3_read('s3://botor/example-data/mtcars2.csv', read.csv) 190 | #> Error in s3_download(object, t) : 191 | #> Python `ClientError`: An error occurred (403) when calling the HeadObject operation: Forbidden 192 | 193 | botor(region_name = 'us-west-2') 194 | s3_read('s3://botor/example-data/mtcars.csv', read.csv) 195 | #> mpg cyl disp hp drat wt qsec vs am gear carb 196 | #> 1 21.0 6 160.0 110 3.90 2.620 16.46 0 1 4 4 197 | #> 2 21.0 6 160.0 110 3.90 2.875 17.02 0 1 4 4 198 | #> ... 199 | ``` 200 | 201 | ## Logging 202 | 203 | `botor` uses the [`logger`](https://daroczig.github.io/logger/) package to write log messages to the console by default with the following log level standards: 204 | 205 | * `TRACE` start of an AWS query (eg just about to start listing all S3 buckets in an AWS account) 206 | * `DEBUG` summary on the result of an AWS query (eg number of S3 buckets found in an AWS account) 207 | * `INFO` currently not used 208 | * `WARN` currently not used 209 | * `ERROR` something bad happened and logging extra context besides what's being returned in the error message 210 | * `FATAL` currently not used 211 | 212 | The default log level threshold set to `DEBUG`. If you want to update that, use the package name for the `namespace` argument of `log_threshold` from the `logger` package, eg to enable all log messages: 213 | 214 | ```r 215 | library(logger) 216 | log_threshold(TRACE, namespace = 'botor') 217 | 218 | s3_download_file('s3://botor/example-data/mtcars.csv', tempfile()) 219 | #> TRACE [2019-01-11 14:48:07] Downloading s3://botor/example-data/mtcars.csv to '/tmp/RtmpCPNrOk/file6fac556567d4' ... 220 | #> DEBUG [2019-01-11 14:48:09] Downloaded 1303 bytes from s3://botor/example-data/mtcars.csv and saved at '/tmp/RtmpCPNrOk/file6fac556567d4' 221 | ``` 222 | 223 | Or update to not fire the less important messages than warnings: 224 | 225 | ```r 226 | library(logger) 227 | log_threshold(WARN, namespace = 'botor') 228 | ``` 229 | 230 | You can use the same approach to set custom (or more than one) log appenders, eg writing the log messages to files, a database etc -- check the `logger` docs for more details. 231 | 232 | ## Why the name? 233 | 234 | `botor` means "goofy" in Hungarian. This is how I feel when looking back to all the dev hours spent on integrating the AWS Java SDK in R -- this includes `AWR.KMS`, where I ended up debugging and fixing many issues in forked processes, but `AWR.Kinesis` still rocks :) 235 | 236 | The name also reminds you that it's not exactly `boto3`, as eg you have to use `$` instead of `.` to access methods. 237 | -------------------------------------------------------------------------------- /_pkgdown.yml: -------------------------------------------------------------------------------- 1 | authors: 2 | System1: 3 | href: https://system1.com 4 | html: -------------------------------------------------------------------------------- /inst/tests/testthat.R: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | library(botor) 3 | 4 | test_check('botor') 5 | -------------------------------------------------------------------------------- /inst/tests/testthat/test-checkmate.R: -------------------------------------------------------------------------------- 1 | library(botor) 2 | library(testthat) 3 | 4 | context('custom checkmate extensions') 5 | test_that('S3 path', { 6 | expect_true(check_s3_uri('s3://foobar')) 7 | expect_true(check_s3_uri('s3://34foobar')) 8 | expect_error(assert_s3_uri('foobar')) 9 | expect_error(assert_s3_uri('https://foobar')) 10 | expect_error(assert_s3_uri('s3://..foobar')) 11 | }) 12 | -------------------------------------------------------------------------------- /inst/tests/testthat/test-load.R: -------------------------------------------------------------------------------- 1 | library(botor) 2 | library(testthat) 3 | 4 | context('s3') 5 | 6 | ## https://aws.amazon.com/marketplace/pp/prodview-zpajhdz2eccoo 7 | pubs3 <- file.path( 8 | 's3://pansurg-curation-raw-open-data/cwtest/SampleDataDuplBookmark/upload_date=1589299981', 9 | 'debabrata_7a7b6d77-d101-457c-a3c2-77c8f1f50a5e_1589299981_uploadfiles_1589299981.csv' 10 | ) 11 | ## need to disable signing for these public objects 12 | options_backup <- getOption('botor-s3-disable-signing') 13 | options('botor-s3-disable-signing' = TRUE) 14 | 15 | test_that('load data from s3', { 16 | expect_true(s3_exists(pubs3)) 17 | expect_error(s3_read(pubs3, read.csv), NA) 18 | }) 19 | 20 | options('botor-s3-disable-signing' = options_backup) 21 | -------------------------------------------------------------------------------- /man/base64_dec.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{base64_dec} 4 | \alias{base64_dec} 5 | \title{Base64-decode a string into raw bytes using Python's base64 module} 6 | \usage{ 7 | base64_dec(text) 8 | } 9 | \arguments{ 10 | \item{text}{string} 11 | } 12 | \value{ 13 | \code{raw} bytes 14 | } 15 | \description{ 16 | Base64-decode a string into raw bytes using Python's base64 module 17 | } 18 | \examples{ 19 | \dontrun{ 20 | botor:::base64_dec(botor:::base64_enc(charToRaw('foobar'))) 21 | } 22 | } 23 | \seealso{ 24 | \code{\link{base64_enc}} 25 | } 26 | \keyword{internal} 27 | -------------------------------------------------------------------------------- /man/base64_enc.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{base64_enc} 4 | \alias{base64_enc} 5 | \title{Base64-encode raw bytes using Python's base64 module} 6 | \usage{ 7 | base64_enc(text) 8 | } 9 | \arguments{ 10 | \item{text}{\code{raw}, R string or Python string} 11 | } 12 | \value{ 13 | string 14 | } 15 | \description{ 16 | Base64-encode raw bytes using Python's base64 module 17 | } 18 | \seealso{ 19 | \code{\link{base64_dec}} 20 | } 21 | \keyword{internal} 22 | -------------------------------------------------------------------------------- /man/boto3.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/boto3.R 3 | \docType{data} 4 | \name{boto3} 5 | \alias{boto3} 6 | \title{Raw access to the boto3 module imported at package load time} 7 | \format{ 8 | An object of class \code{python.builtin.module} (inherits from \code{python.builtin.object}) of length 1. 9 | } 10 | \usage{ 11 | boto3 12 | } 13 | \description{ 14 | Raw access to the boto3 module imported at package load time 15 | } 16 | \note{ 17 | You may rather want to use \code{\link{botor}} instead, that provides a fork-safe \code{boto3} session. 18 | } 19 | \keyword{datasets} 20 | -------------------------------------------------------------------------------- /man/boto3_version.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/boto3.R 3 | \name{boto3_version} 4 | \alias{boto3_version} 5 | \title{boto3 version} 6 | \usage{ 7 | boto3_version() 8 | } 9 | \value{ 10 | string 11 | } 12 | \description{ 13 | boto3 version 14 | } 15 | -------------------------------------------------------------------------------- /man/botor.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/botor.R 3 | \name{botor} 4 | \alias{botor} 5 | \title{The default, fork-safe Boto3 session} 6 | \usage{ 7 | botor( 8 | aws_access_key_id, 9 | aws_secret_access_key, 10 | aws_session_token, 11 | region_name, 12 | botocore_session, 13 | profile_name 14 | ) 15 | } 16 | \arguments{ 17 | \item{aws_access_key_id}{AWS access key ID} 18 | 19 | \item{aws_secret_access_key}{AWS secret access key} 20 | 21 | \item{aws_session_token}{AWS temporary session token} 22 | 23 | \item{region_name}{Default region when creating new connections} 24 | 25 | \item{botocore_session}{Use this Botocore session instead of creating a new default one} 26 | 27 | \item{profile_name}{The name of a profile to use. If not given, then the default profile is used} 28 | } 29 | \value{ 30 | boto3 \code{Session} 31 | } 32 | \description{ 33 | The default, fork-safe Boto3 session 34 | } 35 | -------------------------------------------------------------------------------- /man/botor_client.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/botor.R 3 | \name{botor_client} 4 | \alias{botor_client} 5 | \title{Creates an initial or reinitialize an already existing AWS client 6 | or resource cached in the package's namespace} 7 | \usage{ 8 | botor_client(service, type = c("client", "resource"), cache = TRUE, ...) 9 | } 10 | \arguments{ 11 | \item{service}{string, eg S3 or IAM} 12 | 13 | \item{type}{AWS service client or resource to be created, eg 14 | \code{s3}} 15 | 16 | \item{cache}{booelan flag for caching the client or resource in the 17 | package namespace. For (internal) package functions, it's best 18 | to set to \code{TRUE} to avoid reinitializing the 19 | client/resource, but for custom use and when you need to use 20 | multiple clients for the same service in parallel (eg working 21 | with different regions etc), you might want to set this to 22 | \code{FALSE}} 23 | 24 | \item{...}{further parameters passed to the \code{client} or 25 | \code{resource}, eg \code{endpoint_url}} 26 | } 27 | \value{ 28 | cached AWS client 29 | } 30 | \description{ 31 | Creates an initial or reinitialize an already existing AWS client 32 | or resource cached in the package's namespace 33 | } 34 | \references{ 35 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html} 36 | } 37 | -------------------------------------------------------------------------------- /man/botor_session.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/botor.R 3 | \docType{data} 4 | \name{botor_session} 5 | \alias{botor_session} 6 | \title{Internal boto3 session} 7 | \format{ 8 | An object of class \code{NULL} of length 0. 9 | } 10 | \usage{ 11 | botor_session 12 | } 13 | \description{ 14 | Internal boto3 session 15 | } 16 | \keyword{internal} 17 | -------------------------------------------------------------------------------- /man/botor_session_pid.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/botor.R 3 | \name{botor_session_pid} 4 | \alias{botor_session_pid} 5 | \title{Look up the PID used to initialize the Boto3 session} 6 | \usage{ 7 | botor_session_pid() 8 | } 9 | \value{ 10 | int 11 | } 12 | \description{ 13 | Look up the PID used to initialize the Boto3 session 14 | } 15 | \keyword{internal} 16 | -------------------------------------------------------------------------------- /man/botor_session_uuid.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/botor.R 3 | \name{botor_session_uuid} 4 | \alias{botor_session_uuid} 5 | \title{Look up the UUID of the initialized Boto3 session} 6 | \usage{ 7 | botor_session_uuid() 8 | } 9 | \value{ 10 | int 11 | } 12 | \description{ 13 | Look up the UUID of the initialized Boto3 session 14 | } 15 | \keyword{internal} 16 | -------------------------------------------------------------------------------- /man/check_s3_uri.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/checkmate.R 3 | \name{check_s3_uri} 4 | \alias{check_s3_uri} 5 | \alias{assert_s3_uri} 6 | \alias{test_s3_uri} 7 | \alias{expect_s3_uri} 8 | \title{Check if an argument looks like an S3 bucket} 9 | \usage{ 10 | check_s3_uri(x) 11 | } 12 | \arguments{ 13 | \item{x}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 14 | } 15 | \description{ 16 | Check if an argument looks like an S3 bucket 17 | } 18 | \examples{ 19 | check_s3_uri('s3://foo/bar') 20 | check_s3_uri('https://foo/bar') 21 | \dontrun{ 22 | assert_s3_uri('https://foo/bar') 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /man/clients.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/botor.R 3 | \docType{data} 4 | \name{clients} 5 | \alias{clients} 6 | \title{boto3 clients cache} 7 | \format{ 8 | An object of class \code{environment} of length 0. 9 | } 10 | \usage{ 11 | clients 12 | } 13 | \description{ 14 | boto3 clients cache 15 | } 16 | \keyword{internal} 17 | -------------------------------------------------------------------------------- /man/coerce_bytes_literals_to_string.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{coerce_bytes_literals_to_string} 4 | \alias{coerce_bytes_literals_to_string} 5 | \title{Transforms a python2 string literal or python3 bytes literal into an R string} 6 | \usage{ 7 | coerce_bytes_literals_to_string(x) 8 | } 9 | \arguments{ 10 | \item{x}{string} 11 | } 12 | \value{ 13 | string 14 | } 15 | \description{ 16 | This is useful to call eg for the KMS call, where python2 returns a 17 | string, but python3 returns bytes literals -- calling "decode" is 18 | tricky, but bytearray conversion, then passing the raw vector to R 19 | and converting that a string works. 20 | } 21 | \keyword{internal} 22 | -------------------------------------------------------------------------------- /man/figures/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/daroczig/botor/3c66839162f8b46bef562c9c44a09be0fa8e2d66/man/figures/logo.png -------------------------------------------------------------------------------- /man/iam.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/iam.R 3 | \name{iam} 4 | \alias{iam} 5 | \title{The default, fork-safe IAM client on the top of \code{botor}} 6 | \usage{ 7 | iam() 8 | } 9 | \value{ 10 | \code{botocore.client.IAM} 11 | } 12 | \description{ 13 | The default, fork-safe IAM client on the top of \code{botor} 14 | } 15 | \references{ 16 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/iam.html} 17 | } 18 | -------------------------------------------------------------------------------- /man/iam_get_user.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/iam.R 3 | \name{iam_get_user} 4 | \alias{iam_get_user} 5 | \title{Retrieves information about the specified IAM user, including the user's creation date, path, unique ID, and ARN} 6 | \usage{ 7 | iam_get_user(...) 8 | } 9 | \arguments{ 10 | \item{...}{optional extra arguments passed} 11 | } 12 | \value{ 13 | list 14 | } 15 | \description{ 16 | Retrieves information about the specified IAM user, including the user's creation date, path, unique ID, and ARN 17 | } 18 | \references{ 19 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/iam.html#IAM.Client.get_user} 20 | } 21 | -------------------------------------------------------------------------------- /man/iam_whoami.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/iam.R 3 | \name{iam_whoami} 4 | \alias{iam_whoami} 5 | \title{Get the current AWS username} 6 | \usage{ 7 | iam_whoami() 8 | } 9 | \value{ 10 | string 11 | } 12 | \description{ 13 | Get the current AWS username 14 | } 15 | \seealso{ 16 | \code{\link{sts_whoami}} 17 | } 18 | -------------------------------------------------------------------------------- /man/kinesis.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kinesis.R 3 | \name{kinesis} 4 | \alias{kinesis} 5 | \title{The default, fork-safe Kinesis client on the top of \code{botor}} 6 | \usage{ 7 | kinesis() 8 | } 9 | \value{ 10 | \code{botocore.client.Kinesis} 11 | } 12 | \description{ 13 | The default, fork-safe Kinesis client on the top of \code{botor} 14 | } 15 | \references{ 16 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html} 17 | } 18 | -------------------------------------------------------------------------------- /man/kinesis_describe_stream.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kinesis.R 3 | \name{kinesis_describe_stream} 4 | \alias{kinesis_describe_stream} 5 | \title{Describes the specified Kinesis data stream} 6 | \usage{ 7 | kinesis_describe_stream(stream) 8 | } 9 | \arguments{ 10 | \item{stream}{the name of the stream to describe} 11 | } 12 | \value{ 13 | list 14 | } 15 | \description{ 16 | Describes the specified Kinesis data stream 17 | } 18 | \references{ 19 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.describe_stream} 20 | } 21 | -------------------------------------------------------------------------------- /man/kinesis_get_records.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kinesis.R 3 | \name{kinesis_get_records} 4 | \alias{kinesis_get_records} 5 | \title{Gets data records from a Kinesis data stream's shard} 6 | \usage{ 7 | kinesis_get_records(shard_iterator, limit = 25L) 8 | } 9 | \arguments{ 10 | \item{shard_iterator}{the position in the shard from which you want 11 | to start sequentially reading data records, usually provided by 12 | \code{\link{kinesis_get_shard_iterator}}} 13 | 14 | \item{limit}{maximum number of records to return} 15 | } 16 | \value{ 17 | list of \code{Records}, \code{NextShardIterator} and 18 | \code{MillisBehindLatest} 19 | } 20 | \description{ 21 | Gets data records from a Kinesis data stream's shard 22 | } 23 | \examples{ 24 | \dontrun{ 25 | botor(profile_name = 'botor-tester') 26 | iterator <- kinesis_get_shard_iterator(stream = 'botor-tester', shard = '0') 27 | kinesis_get_records(iterator$ShardIterator) 28 | } 29 | } 30 | \references{ 31 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.get_records} 32 | } 33 | -------------------------------------------------------------------------------- /man/kinesis_get_shard_iterator.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kinesis.R 3 | \name{kinesis_get_shard_iterator} 4 | \alias{kinesis_get_shard_iterator} 5 | \title{Gets an Amazon Kinesis shard iterator} 6 | \usage{ 7 | kinesis_get_shard_iterator( 8 | stream, 9 | shard, 10 | shard_iterator_type = c("TRIM_HORIZON", "LATEST", "AT_SEQUENCE_NUMBER", 11 | "AFTER_SEQUENCE_NUMBER", "AT_TIMESTAMP"), 12 | ... 13 | ) 14 | } 15 | \arguments{ 16 | \item{stream}{the name of the stream to describe} 17 | 18 | \item{shard}{the shard ID of the Kinesis Data Streams shard to get 19 | the iterator for} 20 | 21 | \item{shard_iterator_type}{determines how the shard iterator is 22 | used to start reading data records from the shard} 23 | 24 | \item{...}{optional further parameters, such as 25 | \code{StartingSequenceNumber} or \code{Timestamp}} 26 | } 27 | \value{ 28 | list of \code{ShardIterator} 29 | } 30 | \description{ 31 | Gets an Amazon Kinesis shard iterator 32 | } 33 | \references{ 34 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.get_shard_iterator} 35 | } 36 | \seealso{ 37 | \code{\link{kinesis_get_records}} 38 | } 39 | -------------------------------------------------------------------------------- /man/kinesis_put_record.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kinesis.R 3 | \name{kinesis_put_record} 4 | \alias{kinesis_put_record} 5 | \title{Writes a single data record into an Amazon Kinesis data stream} 6 | \usage{ 7 | kinesis_put_record(stream, data, partition_key, ...) 8 | } 9 | \arguments{ 10 | \item{stream}{the name of the stream to describe} 11 | 12 | \item{data}{the data blob (<1 MB) to put into the record, which is 13 | base64-encoded when the blob is serialized} 14 | 15 | \item{partition_key}{Unicode string with a maximum length limit of 16 | 256 characters determining which shard in the stream the data 17 | record is assigned to} 18 | 19 | \item{...}{optional further parameters, such as 20 | \code{ExplicitHashKey} or \code{SequenceNumberForOrdering}} 21 | } 22 | \value{ 23 | list of \code{ShardId}, \code{SequenceNumber} and 24 | \code{EncryptionType} 25 | } 26 | \description{ 27 | Writes a single data record into an Amazon Kinesis data stream 28 | } 29 | \references{ 30 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html#Kinesis.Client.put_record} 31 | } 32 | -------------------------------------------------------------------------------- /man/kms.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kms.R 3 | \name{kms} 4 | \alias{kms} 5 | \title{The default, fork-safe KMS client on the top of \code{botor}} 6 | \usage{ 7 | kms() 8 | } 9 | \value{ 10 | \code{botocore.client.KMS} 11 | } 12 | \description{ 13 | The default, fork-safe KMS client on the top of \code{botor} 14 | } 15 | \references{ 16 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kms.html} 17 | } 18 | -------------------------------------------------------------------------------- /man/kms_decrypt.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kms.R 3 | \name{kms_decrypt} 4 | \alias{kms_decrypt} 5 | \title{Decrypt cipher into plain text via KMS} 6 | \usage{ 7 | kms_decrypt(cipher, simplify = TRUE) 8 | } 9 | \arguments{ 10 | \item{cipher}{Base64-encoded ciphertext} 11 | 12 | \item{simplify}{returns decrypted plain-text instead of raw list} 13 | } 14 | \value{ 15 | decrypted text as string or \code{list} 16 | } 17 | \description{ 18 | Decrypt cipher into plain text via KMS 19 | } 20 | \seealso{ 21 | \code{\link{kms_encrypt}} 22 | } 23 | -------------------------------------------------------------------------------- /man/kms_decrypt_file.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kms.R 3 | \name{kms_decrypt_file} 4 | \alias{kms_decrypt_file} 5 | \title{Decrypt file via KMS} 6 | \usage{ 7 | kms_decrypt_file(file, return = file) 8 | } 9 | \arguments{ 10 | \item{file}{base file path (without the \code{enc} or \code{key} suffix)} 11 | 12 | \item{return}{where to place the encrypted file (defaults to \code{file})} 13 | } 14 | \value{ 15 | decrypted file path 16 | } 17 | \description{ 18 | Decrypt file via KMS 19 | } 20 | \seealso{ 21 | \code{\link{kms_encrypt}} \code{\link{kms_encrypt_file}} 22 | } 23 | -------------------------------------------------------------------------------- /man/kms_encrypt.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kms.R 3 | \name{kms_encrypt} 4 | \alias{kms_encrypt} 5 | \title{Encrypt plain text via KMS} 6 | \usage{ 7 | kms_encrypt(key, text, simplify = TRUE) 8 | } 9 | \arguments{ 10 | \item{key}{the KMS customer master key identifier as a fully 11 | specified Amazon Resource Name (eg 12 | \code{arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012}) 13 | or an alias with the \code{alias/} prefix (eg 14 | \code{alias/foobar})} 15 | 16 | \item{text}{max 4096 bytes long string, eg an RSA key, a database 17 | password, or other sensitive customer information} 18 | 19 | \item{simplify}{returns Base64-encoded text instead of raw list} 20 | } 21 | \value{ 22 | string or \code{list} 23 | } 24 | \description{ 25 | Encrypt plain text via KMS 26 | } 27 | \seealso{ 28 | \code{\link{kms_decrypt}} 29 | } 30 | -------------------------------------------------------------------------------- /man/kms_encrypt_file.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kms.R 3 | \name{kms_encrypt_file} 4 | \alias{kms_encrypt_file} 5 | \title{Encrypt file via KMS} 6 | \usage{ 7 | kms_encrypt_file(key, file) 8 | } 9 | \arguments{ 10 | \item{key}{the KMS customer master key identifier as a fully 11 | specified Amazon Resource Name (eg 12 | \code{arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012}) 13 | or an alias with the \code{alias/} prefix (eg 14 | \code{alias/foobar})} 15 | 16 | \item{file}{file path} 17 | } 18 | \value{ 19 | two files created with \code{enc} (encrypted data) and 20 | \code{key} (encrypted key) extensions 21 | } 22 | \description{ 23 | Encrypt file via KMS 24 | } 25 | \seealso{ 26 | \code{\link{kms_encrypt}} \code{\link{kms_decrypt_file}} 27 | } 28 | -------------------------------------------------------------------------------- /man/kms_generate_data_key.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/kms.R 3 | \name{kms_generate_data_key} 4 | \alias{kms_generate_data_key} 5 | \title{Generate a data encryption key for envelope encryption via KMS} 6 | \usage{ 7 | kms_generate_data_key(key, bytes = 64L) 8 | } 9 | \arguments{ 10 | \item{key}{the KMS customer master key identifier as a fully 11 | specified Amazon Resource Name (eg 12 | \code{arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012}) 13 | or an alias with the \code{alias/} prefix (eg 14 | \code{alias/foobar})} 15 | 16 | \item{bytes}{the required length of the data encryption key in 17 | bytes (so provide eg \code{64L} for a 512-bit key)} 18 | } 19 | \value{ 20 | \code{list} of the Base64-encoded encrypted version of the 21 | data encryption key (to be stored on disk), the \code{raw} 22 | object of the encryption key and the KMS customer master key 23 | used to generate this object 24 | } 25 | \description{ 26 | Generate a data encryption key for envelope encryption via KMS 27 | } 28 | -------------------------------------------------------------------------------- /man/mime_guess.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{mime_guess} 4 | \alias{mime_guess} 5 | \title{Guess the type of a file based on the filename using \code{mimetypes} Python module} 6 | \usage{ 7 | mime_guess(file) 8 | } 9 | \arguments{ 10 | \item{file}{path} 11 | } 12 | \value{ 13 | string 14 | } 15 | \description{ 16 | Guess the type of a file based on the filename using \code{mimetypes} Python module 17 | } 18 | -------------------------------------------------------------------------------- /man/python_version.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{python_version} 4 | \alias{python_version} 5 | \title{Looks up major version of Python (eg 2 or 3)} 6 | \usage{ 7 | python_version() 8 | } 9 | \value{ 10 | number 11 | } 12 | \description{ 13 | Looks up major version of Python (eg 2 or 3) 14 | } 15 | \keyword{internal} 16 | -------------------------------------------------------------------------------- /man/require_python_builtins.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/python_modules.R 3 | \name{require_python_builtins} 4 | \alias{require_python_builtins} 5 | \title{Imports and caches a Python module} 6 | \usage{ 7 | require_python_builtins() 8 | } 9 | \value{ 10 | imported Python module 11 | } 12 | \description{ 13 | Imports and caches a Python module 14 | } 15 | \keyword{internal} 16 | -------------------------------------------------------------------------------- /man/require_python_module.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/python_modules.R 3 | \name{require_python_module} 4 | \alias{require_python_module} 5 | \title{Imports and caches a Python module} 6 | \usage{ 7 | require_python_module(module) 8 | } 9 | \arguments{ 10 | \item{module}{a Python module name} 11 | } 12 | \value{ 13 | imported Python module 14 | } 15 | \description{ 16 | Imports and caches a Python module 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /man/s3.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3} 4 | \alias{s3} 5 | \title{The default, fork-safe Amazon Simple Storage Service (S3) client on the top of \code{botor}} 6 | \usage{ 7 | s3(disable_signing = getOption("botor-s3-disable-signing")) 8 | } 9 | \arguments{ 10 | \item{disable_signing}{boolean if requests should be signed. Set to \code{FALSE} when interacting with public S3 buckets requiring unauthenticated access.} 11 | } 12 | \value{ 13 | \code{s3.ServiceResource} 14 | } 15 | \description{ 16 | The default, fork-safe Amazon Simple Storage Service (S3) client on the top of \code{botor} 17 | } 18 | \references{ 19 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#service-resource} 20 | } 21 | -------------------------------------------------------------------------------- /man/s3_copy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_copy} 4 | \alias{s3_copy} 5 | \title{Copy an object from one S3 location to another} 6 | \usage{ 7 | s3_copy(uri_source, uri_target) 8 | } 9 | \arguments{ 10 | \item{uri_source}{string, location of the source file} 11 | 12 | \item{uri_target}{string, location of the target file} 13 | } 14 | \value{ 15 | invisibly \code{uri_target} 16 | } 17 | \description{ 18 | Copy an object from one S3 location to another 19 | } 20 | \references{ 21 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Object.copy} 22 | } 23 | -------------------------------------------------------------------------------- /man/s3_delete.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_delete} 4 | \alias{s3_delete} 5 | \title{Delete an object stored in S3} 6 | \usage{ 7 | s3_delete(uri) 8 | } 9 | \arguments{ 10 | \item{uri}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 11 | } 12 | \description{ 13 | Delete an object stored in S3 14 | } 15 | -------------------------------------------------------------------------------- /man/s3_download_file.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_download_file} 4 | \alias{s3_download_file} 5 | \title{Download a file from S3} 6 | \usage{ 7 | s3_download_file(uri, file, force = TRUE) 8 | } 9 | \arguments{ 10 | \item{uri}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 11 | 12 | \item{file}{string, location of local file} 13 | 14 | \item{force}{boolean, overwrite local file if exists} 15 | } 16 | \value{ 17 | invisibly \code{file} 18 | } 19 | \description{ 20 | Download a file from S3 21 | } 22 | \examples{ 23 | \dontrun{ 24 | s3_download_file('s3://botor/example-data/mtcars.csv', tempfile()) 25 | } 26 | } 27 | \references{ 28 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.download_file} 29 | } 30 | -------------------------------------------------------------------------------- /man/s3_exists.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_exists} 4 | \alias{s3_exists} 5 | \title{Checks if an object exists in S3} 6 | \usage{ 7 | s3_exists(uri) 8 | } 9 | \arguments{ 10 | \item{uri}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 11 | } 12 | \value{ 13 | boolean 14 | } 15 | \description{ 16 | Checks if an object exists in S3 17 | } 18 | \examples{ 19 | \dontrun{ 20 | s3_exists('s3://botor/example-data/mtcars.csv') 21 | s3_exists('s3://botor/example-data/UNDEFINED.CSVLX') 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /man/s3_list_buckets.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_list_buckets} 4 | \alias{s3_list_buckets} 5 | \title{List all S3 buckets} 6 | \usage{ 7 | s3_list_buckets(simplify = TRUE) 8 | } 9 | \arguments{ 10 | \item{simplify}{return bucket names as a character vector} 11 | } 12 | \value{ 13 | \code{list} of \code{boto3.resources.factory.s3.Bucket} or a character vector 14 | } 15 | \description{ 16 | List all S3 buckets 17 | } 18 | -------------------------------------------------------------------------------- /man/s3_ls.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_ls} 4 | \alias{s3_ls} 5 | \title{List objects at an S3 path} 6 | \usage{ 7 | s3_ls(uri) 8 | } 9 | \arguments{ 10 | \item{uri}{string, should start with \code{s3://}, then bucket name 11 | and optional object key prefix} 12 | } 13 | \value{ 14 | \code{data.frame} with \code{bucket_name}, object 15 | \code{key}, \code{uri} (that can be directly passed to eg 16 | \code{\link{s3_read}}), \code{size} in bytes, \code{owner} and 17 | \code{last_modified} timestamp 18 | } 19 | \description{ 20 | List objects at an S3 path 21 | } 22 | -------------------------------------------------------------------------------- /man/s3_object.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_object} 4 | \alias{s3_object} 5 | \title{Create an S3 Object reference from an URI} 6 | \usage{ 7 | s3_object(uri) 8 | } 9 | \arguments{ 10 | \item{uri}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 11 | } 12 | \value{ 13 | \code{s3$Object} 14 | } 15 | \description{ 16 | Create an S3 Object reference from an URI 17 | } 18 | -------------------------------------------------------------------------------- /man/s3_put_object_tagging.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_put_object_tagging} 4 | \alias{s3_put_object_tagging} 5 | \title{Sets tags on s3 object overwriting all existing tags. Note: tags 6 | and metadata tags are not the same} 7 | \usage{ 8 | s3_put_object_tagging(uri, tags) 9 | } 10 | \arguments{ 11 | \item{uri}{string, URI of an S3 object, should start with 12 | \code{s3://}, then bucket name and object key} 13 | 14 | \item{tags}{named character vector, e.g. \code{c(my_first_name = 15 | 'my_first_value', my_second_name = 'my_second_value')} where 16 | names are the tag names and values are the tag values.} 17 | } 18 | \description{ 19 | Sets tags on s3 object overwriting all existing tags. Note: tags 20 | and metadata tags are not the same 21 | } 22 | \references{ 23 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object_tagging} 24 | } 25 | -------------------------------------------------------------------------------- /man/s3_read.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_read} 4 | \alias{s3_read} 5 | \title{Download and read a file from S3, then clean up} 6 | \usage{ 7 | s3_read(uri, fun, ..., extract = c("none", "gzip", "bzip2", "xz")) 8 | } 9 | \arguments{ 10 | \item{uri}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 11 | 12 | \item{fun}{R function to read the file, eg \code{fromJSON}, \code{stream_in}, \code{fread} or \code{readRDS}} 13 | 14 | \item{...}{optional params passed to \code{fun}} 15 | 16 | \item{extract}{optionally extract/decompress the file after downloading from S3 but before passing to \code{fun}} 17 | } 18 | \value{ 19 | R object 20 | } 21 | \description{ 22 | Download and read a file from S3, then clean up 23 | } 24 | \examples{ 25 | \dontrun{ 26 | s3_read('s3://botor/example-data/mtcars.csv', read.csv) 27 | s3_read('s3://botor/example-data/mtcars.csv', data.table::fread) 28 | s3_read('s3://botor/example-data/mtcars.csv2', read.csv2) 29 | s3_read('s3://botor/example-data/mtcars.RDS', readRDS) 30 | s3_read('s3://botor/example-data/mtcars.json', jsonlite::fromJSON) 31 | s3_read('s3://botor/example-data/mtcars.jsonl', jsonlite::stream_in) 32 | 33 | ## read compressed data 34 | s3_read('s3://botor/example-data/mtcars.csv.gz', read.csv, extract = 'gzip') 35 | s3_read('s3://botor/example-data/mtcars.csv.gz', data.table::fread, extract = 'gzip') 36 | s3_read('s3://botor/example-data/mtcars.csv.bz2', read.csv, extract = 'bzip2') 37 | s3_read('s3://botor/example-data/mtcars.csv.xz', read.csv, extract = 'xz') 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /man/s3_split_uri.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_split_uri} 4 | \alias{s3_split_uri} 5 | \title{Split the bucket name and object key from the S3 URI} 6 | \usage{ 7 | s3_split_uri(uri) 8 | } 9 | \arguments{ 10 | \item{uri}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 11 | } 12 | \value{ 13 | list 14 | } 15 | \description{ 16 | Split the bucket name and object key from the S3 URI 17 | } 18 | \keyword{internal} 19 | -------------------------------------------------------------------------------- /man/s3_upload_file.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_upload_file} 4 | \alias{s3_upload_file} 5 | \title{Upload a file to S3} 6 | \usage{ 7 | s3_upload_file(file, uri, content_type = mime_guess(file), ...) 8 | } 9 | \arguments{ 10 | \item{file}{string, location of local file} 11 | 12 | \item{uri}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 13 | 14 | \item{content_type}{content type of a file that is auto-guess if omitted} 15 | 16 | \item{...}{optional parameters passed to the \code{upload_file} method's \code{ExtraArgs}} 17 | } 18 | \value{ 19 | invisibly \code{uri} 20 | } 21 | \description{ 22 | Upload a file to S3 23 | } 24 | \examples{ 25 | \dontrun{ 26 | t <- tempfile() 27 | write.csv(mtcars, t, row.names = FALSE) 28 | s3_upload_file(t, 's3://botor/example-data/mtcars.csv') 29 | unlink(t) 30 | ## note that s3_write would have been a much nicer solution for the above 31 | } 32 | } 33 | \references{ 34 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.upload_file} 35 | } 36 | \seealso{ 37 | \code{\link{s3_download_file}} 38 | } 39 | -------------------------------------------------------------------------------- /man/s3_write.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/s3.R 3 | \name{s3_write} 4 | \alias{s3_write} 5 | \title{Write an R object into S3} 6 | \usage{ 7 | s3_write(x, fun, uri, compress = c("none", "gzip", "bzip2", "xz"), ...) 8 | } 9 | \arguments{ 10 | \item{x}{R object} 11 | 12 | \item{fun}{R function with \code{file} argument to serialize 13 | \code{x} to disk before uploading, eg \code{write.csv}, 14 | \code{write_json}, \code{stream_out} or \code{saveRDS}} 15 | 16 | \item{uri}{string, URI of an S3 object, should start with \code{s3://}, then bucket name and object key} 17 | 18 | \item{compress}{optionally compress the file before uploading to 19 | S3. If compression is used, it's better to include the related 20 | file extension in \code{uri} as well (that is not done 21 | automatically).} 22 | 23 | \item{...}{optional further arguments passed to \code{fun}} 24 | } 25 | \description{ 26 | Write an R object into S3 27 | } 28 | \note{ 29 | The temp file used for this operation is automatically 30 | removed. 31 | } 32 | \examples{ 33 | \dontrun{ 34 | s3_write(mtcars, write.csv, 's3://botor/example-data/mtcars.csv', row.names = FALSE) 35 | s3_write(mtcars, write.csv2, 's3://botor/example-data/mtcars.csv2', row.names = FALSE) 36 | s3_write(mtcars, jsonlite::write_json, 's3://botor/example-data/mtcars.json', row.names = FALSE) 37 | s3_write(mtcars, jsonlite::stream_out, 's3://botor/example-data/mtcars.jsonl', row.names = FALSE) 38 | s3_write(mtcars, saveRDS, 's3://botor/example-data/mtcars.RDS') 39 | 40 | ## compress file after writing to disk but before uploading to S3 41 | s3_write(mtcars, write.csv, 's3://botor/example-data/mtcars.csv.gz', 42 | compress = 'gzip', row.names = FALSE) 43 | s3_write(mtcars, write.csv, 's3://botor/example-data/mtcars.csv.bz2', 44 | compress = 'bzip2', row.names = FALSE) 45 | s3_write(mtcars, write.csv, 's3://botor/example-data/mtcars.csv.xz', 46 | compress = 'xz', row.names = FALSE) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /man/sm.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/sm.R 3 | \name{sm} 4 | \alias{sm} 5 | \title{The default, fork-safe AWS Systems Manager (SecretManager) client on the top of \code{botor}} 6 | \usage{ 7 | sm() 8 | } 9 | \value{ 10 | \code{botocore.client.secretsmanager} 11 | } 12 | \description{ 13 | The default, fork-safe AWS Systems Manager (SecretManager) client on the top of \code{botor} 14 | } 15 | \references{ 16 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/secretsmanager.html} 17 | } 18 | -------------------------------------------------------------------------------- /man/sm_get_secret.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/sm.R 3 | \name{sm_get_secret} 4 | \alias{sm_get_secret} 5 | \title{Read AWS System Manager's Secrets Manager via Secret Manager} 6 | \usage{ 7 | sm_get_secret(path, key = NULL, parse_json = TRUE) 8 | } 9 | \arguments{ 10 | \item{path}{name/path of the key to be read} 11 | 12 | \item{key}{single key or a vector of keys.} 13 | 14 | \item{parse_json}{logical. Default TRUE} 15 | } 16 | \value{ 17 | (optionally decrypted) value 18 | } 19 | \description{ 20 | Read AWS System Manager's Secrets Manager via Secret Manager 21 | } 22 | -------------------------------------------------------------------------------- /man/ssm.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/ssm.R 3 | \name{ssm} 4 | \alias{ssm} 5 | \title{The default, fork-safe AWS Systems Manager (SSM) client on the top of \code{botor}} 6 | \usage{ 7 | ssm() 8 | } 9 | \value{ 10 | \code{botocore.client.SSM} 11 | } 12 | \description{ 13 | The default, fork-safe AWS Systems Manager (SSM) client on the top of \code{botor} 14 | } 15 | \references{ 16 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html} 17 | } 18 | -------------------------------------------------------------------------------- /man/ssm_get_parameter.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/ssm.R 3 | \name{ssm_get_parameter} 4 | \alias{ssm_get_parameter} 5 | \title{Read AWS System Manager's Parameter Store} 6 | \usage{ 7 | ssm_get_parameter(path, decrypt = TRUE) 8 | } 9 | \arguments{ 10 | \item{path}{name/path of the key to be read} 11 | 12 | \item{decrypt}{decrypt the value or return the raw ciphertext} 13 | } 14 | \value{ 15 | (optionally decrypted) value 16 | } 17 | \description{ 18 | Read AWS System Manager's Parameter Store 19 | } 20 | -------------------------------------------------------------------------------- /man/sts_whoami.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/iam.R 3 | \name{sts_whoami} 4 | \alias{sts_whoami} 5 | \title{Returns details about the IAM user or role whose credentials are used to call the operation} 6 | \usage{ 7 | sts_whoami() 8 | } 9 | \value{ 10 | \code{list} with \code{UserId}, \code{Account} and \code{Arn} 11 | } 12 | \description{ 13 | Returns details about the IAM user or role whose credentials are used to call the operation 14 | } 15 | \references{ 16 | \url{https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.get_caller_identity} 17 | } 18 | \seealso{ 19 | \code{\link{iam_whoami}} 20 | } 21 | -------------------------------------------------------------------------------- /man/trypy.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{trypy} 4 | \alias{trypy} 5 | \title{Extract error message from a Python exception} 6 | \usage{ 7 | trypy(expression) 8 | } 9 | \arguments{ 10 | \item{expression}{R expression} 11 | } 12 | \value{ 13 | error 14 | } 15 | \description{ 16 | Extract error message from a Python exception 17 | } 18 | \examples{ 19 | \dontrun{ 20 | trypy(botor()$resource('foobar')) 21 | trypy(sum(1:2)) 22 | trypy(sum(1:1foo)) 23 | } 24 | } 25 | \keyword{internal} 26 | -------------------------------------------------------------------------------- /man/uuid.Rd: -------------------------------------------------------------------------------- 1 | % Generated by roxygen2: do not edit by hand 2 | % Please edit documentation in R/utils.R 3 | \name{uuid} 4 | \alias{uuid} 5 | \title{Generate UUID using Python's uuid module} 6 | \usage{ 7 | uuid() 8 | } 9 | \value{ 10 | string 11 | } 12 | \description{ 13 | Generate UUID using Python's uuid module 14 | } 15 | \keyword{internal} 16 | --------------------------------------------------------------------------------