├── .devcontainer ├── devcontainer.json └── docker-compose.yml ├── .gitattributes ├── .github └── workflows │ └── release.yml ├── .gitignore ├── LICENSE ├── Makefile ├── NOTICE.txt ├── README.md ├── README_previous.md ├── azure-pipelines.yml ├── build_plugin.sh ├── buildspec.yml ├── cmd ├── azure-import │ └── main.go ├── azure-list-versions │ └── main.go ├── changelog-reader │ └── main.go ├── lhsm-plugin-az-core │ ├── archive.go │ ├── remove.go │ ├── restore.go │ ├── zt_archive_test.go │ ├── zt_restore_test.go │ └── zt_test.go ├── lhsm-plugin-az │ ├── main.go │ └── mover.go ├── lhsm-plugin-posix │ ├── config_test.go │ ├── main.go │ ├── posix │ │ ├── mover.go │ │ └── posix_test.go │ └── test-fixtures │ │ ├── lhsm-plugin-posix-badarchive │ │ ├── lhsm-plugin-posix.checksums │ │ └── lhsm-plugin-posix.test ├── lhsmd │ ├── agent │ │ ├── action_stats.go │ │ ├── agent.go │ │ ├── agent_action.go │ │ ├── agent_test.go │ │ ├── config.go │ │ ├── config_test.go │ │ ├── endpoints.go │ │ ├── fileid │ │ │ ├── fileid.go │ │ │ └── testing.go │ │ ├── mountpoints.go │ │ ├── mountpoints_test.go │ │ ├── plugin.go │ │ ├── snapshot.go │ │ └── test-fixtures │ │ │ ├── good-config │ │ │ ├── json-config │ │ │ ├── merge-config │ │ │ └── plugin-config │ ├── agent_e2e_test.go │ ├── config │ │ └── default.go │ ├── main.go │ ├── profile.go │ └── transport │ │ └── grpc │ │ ├── rpc.go │ │ └── stats.go └── util │ ├── logSanitizer.go │ ├── logger.go │ ├── misc.go │ ├── pacer-tokenBucketPacer.go │ └── rateLimiterPolicy.go ├── create-protex-scantree.sh ├── dmplugin ├── config.go ├── dmclient.go ├── dmio │ ├── action.go │ └── progress.go ├── plugin.go └── testing.go ├── doc ├── agent.example └── lhsm-plugin-posix.example ├── go.mod ├── go.sum ├── internal └── testhelpers │ └── helpers.go ├── man ├── lhsm-plugin-posix.1.md ├── lhsm-plugin-s3.1.md └── lhsmd.1.md ├── packaging ├── ci │ ├── bucketsite │ │ ├── ajaxload-circle.gif │ │ ├── index.html │ │ ├── jquery.min.js │ │ └── list.js │ └── lambda │ │ ├── GitPullS3 │ │ ├── LICENCE.txt │ │ ├── Makefile │ │ ├── NOTICE.txt │ │ ├── THIRD_PARTY_LICENSES.html │ │ ├── THIRD_PARTY_LICENSES.md │ │ ├── _cffi_backend.so │ │ ├── _pygit2.so │ │ ├── cffi │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── _cffi_include.h │ │ │ ├── _embedding.h │ │ │ ├── api.py │ │ │ ├── backend_ctypes.py │ │ │ ├── cffi_opcode.py │ │ │ ├── commontypes.py │ │ │ ├── cparser.py │ │ │ ├── ffiplatform.py │ │ │ ├── lock.py │ │ │ ├── model.py │ │ │ ├── parse_c_type.h │ │ │ ├── recompiler.py │ │ │ ├── setuptools_ext.py │ │ │ ├── vengine_cpy.py │ │ │ ├── vengine_gen.py │ │ │ └── verifier.py │ │ ├── ipaddress.py │ │ ├── lambda_function.py │ │ ├── libgit2.so.24 │ │ └── pygit2 │ │ │ ├── COPYING │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── _build.py │ │ │ ├── _libgit2.so │ │ │ ├── _run.py │ │ │ ├── blame.py │ │ │ ├── config.py │ │ │ ├── credentials.py │ │ │ ├── decl.h │ │ │ ├── errors.py │ │ │ ├── ffi.py │ │ │ ├── index.py │ │ │ ├── py2.py │ │ │ ├── py3.py │ │ │ ├── refspec.py │ │ │ ├── remote.py │ │ │ ├── repository.py │ │ │ ├── settings.py │ │ │ ├── submodule.py │ │ │ └── utils.py │ │ ├── Makefile │ │ ├── MonitorGithubBuild │ │ ├── Makefile │ │ ├── lambda_function.py │ │ └── setup.cfg │ │ ├── NotifyGithub │ │ ├── Makefile │ │ ├── lambda_function.py │ │ └── setup.cfg │ │ ├── PublishRepoToBucket │ │ ├── Makefile │ │ ├── lambda_function.py │ │ └── setup.cfg │ │ └── lemur_ci │ │ ├── __init__.py │ │ ├── commit_status.py │ │ └── pipeline.py ├── docker │ ├── Makefile │ ├── README.md │ ├── buildonly-lustre-client │ │ ├── Dockerfile │ │ └── Makefile │ ├── go-el7 │ │ ├── Dockerfile │ │ └── Makefile │ ├── host-kernel │ │ └── Makefile │ ├── lemur-rpm-build │ │ ├── Dockerfile │ │ └── Makefile │ ├── linux-host-kernel │ │ ├── Dockerfile │ │ └── Makefile │ ├── mac-host-kernel │ │ ├── Dockerfile │ │ └── Makefile │ └── native-lustre-client │ │ ├── Dockerfile │ │ └── Makefile └── rpm │ ├── Makefile │ ├── lemur.spec │ ├── lhsmd.conf │ └── lhsmd.service ├── pdm ├── Makefile ├── pdm.pb.go └── pdm.proto ├── pkg ├── checksum │ └── checksum.go ├── fsroot │ ├── client.go │ ├── fsid_darwin.go │ ├── fsid_linux.go │ └── testing.go └── zipcheck │ └── analyze.go └── toolset ├── Dockerfile ├── LustrePack.repo └── README.md /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Lemur dev", 3 | 4 | // Update the 'dockerComposeFile' list if you have more compose files or use different names. 5 | "dockerComposeFile": "docker-compose.yml", 6 | 7 | // Container user to use in VSCode Online and GitHub Codespaces 8 | "containerUser" : "root", 9 | 10 | // The 'service' property is the name of the service for the container that VS Code should 11 | // use. Update this value and .devcontainer/docker-compose.yml to the real service name. 12 | "service": "lemur-dev", 13 | 14 | // The optional 'workspaceFolder' property is the path VS Code should open by default when 15 | // connected. This is typically a volume mount in .devcontainer/docker-compose.yml 16 | "workspaceFolder": "/lemur", 17 | 18 | // Use 'settings' to set *default* container specific settings.json values on container create. 19 | // You can edit these settings after create using File > Preferences > Settings > Remote. 20 | "settings": { 21 | "files.eol": "\n", 22 | "terminal.integrated.shell.linux": "/bin/bash", 23 | "editor.tabSize": 2, 24 | "terminal.integrated.scrollback": 8000, 25 | }, 26 | 27 | // Uncomment the next line if you want start specific services in your Docker Compose config. 28 | // "runServices": [], 29 | 30 | // Uncomment this like if you want to keep your containers running after VS Code shuts down. 31 | // "shutdownAction": "none", 32 | 33 | // Uncomment the next line to run commands after the container is created. 34 | // "postCreateCommand": "cp -R /tmp/.ssh-localhost/* ~/.ssh && sudo chmod 600 ~/.ssh/* && sudo chown -R $(whoami) /hpc && git config --global core.editor vi && pre-commit install && pre-commit autoupdate", 35 | 36 | // Add the IDs of extensions you want installed when the container is created in the array below. 37 | "extensions": [ 38 | "eamodio.gitlens" 39 | ] 40 | } -------------------------------------------------------------------------------- /.devcontainer/docker-compose.yml: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------------------------------------- 2 | # Copyright (c) Microsoft Corporation. All rights reserved. 3 | # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. 4 | #------------------------------------------------------------------------------------------------------------- 5 | 6 | version: '3.7' 7 | services: 8 | lemur-dev: 9 | image: paulmedwards/lemur-dev-lustre-2.12.5:latest 10 | 11 | volumes: 12 | - ..:/lemur 13 | 14 | #- /var/run/docker.sock:/var/run/docker.sock 15 | 16 | # Overrides default command so things don't shut down after the process ends. 17 | command: /bin/sh -c "while sleep 1000; do :; done" 18 | 19 | 20 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | packaging/* linguist-vendored 2 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: release 4 | 5 | # Controls when the workflow will run 6 | on: 7 | # Allows you to run this workflow manually from the Actions tab 8 | #workflow_dispatch: 9 | 10 | push: 11 | tags: 12 | - '*' 13 | 14 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 15 | jobs: 16 | # This workflow contains a single job called "build" 17 | release: 18 | # The type of runner that the job will run on 19 | runs-on: ubuntu-latest 20 | 21 | container: 22 | image: paulmedwards/lemur-dev-lustre-2.12.5:latest 23 | options: --user 0 24 | 25 | # Steps represent a sequence of tasks that will be executed as part of the job 26 | steps: 27 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 28 | - uses: actions/checkout@v2.4.0 29 | with: 30 | fetch-depth: 0 31 | 32 | - name: get version tag 33 | run: echo "version_tag=$(git describe --tags --always --dirty | tr '-' '_')" >> $GITHUB_ENV 34 | 35 | - name: print release name 36 | run: echo "version tag = ${{ env.version_tag }}" 37 | 38 | - name: list current working dir 39 | run: ls -lart 40 | 41 | - name: git status 42 | run: git status 43 | 44 | # Runs a single command using the runners shell 45 | - name: Build RPMs 46 | run: make local-rpm 47 | 48 | - name: Release 49 | uses: softprops/action-gh-release@v1 50 | with: 51 | name: ${{ env.version_tag }} 52 | files: | 53 | /github/home/rpmbuild/RPMS/x86_64/lemur-azure-data-movers-${{ env.version_tag }}-lustre_2.12.x86_64.rpm 54 | /github/home/rpmbuild/RPMS/x86_64/lemur-azure-hsm-agent-${{ env.version_tag }}-lustre_2.12.x86_64.rpm -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac 2 | .DS_Store 3 | .idea 4 | vendor 5 | .cache-go 6 | go 7 | dist 8 | 9 | # vim 10 | *.swp 11 | 12 | # Go litter 13 | *.coverprofile 14 | *.test 15 | .latest 16 | .lhsmd-test 17 | test.log 18 | 19 | # build junk 20 | packaging/docker/lemur-rpm-build/lemur.spec 21 | output 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2018 DDN 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | this software and associated documentation files (the "Software"), to deal in 6 | the Software without restriction, including without limitation the rights to 7 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 8 | the Software, and to permit persons to whom the Software is furnished to do so, 9 | subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 16 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 17 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 18 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 19 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # variable definitions 2 | NAME := lemur 3 | DESC := Lustre HSM Agent and Movers 4 | PREFIX ?= $(PWD)/usr/local 5 | BUILDROOT ?= 6 | VERSION ?= $(shell git describe --tags --always --dirty | tr '-' '_') 7 | BUILDDATE := $(shell date -u +"%B %d, %Y") 8 | GOVERSION := $(shell go version) 9 | PKG_RELEASE ?= lustre_2.12 10 | PROJECT_URL := "https://github.com/edwardsp/$(NAME)" 11 | LDFLAGS := -X 'main.version=$(VERSION)' 12 | 13 | CMD_SOURCES := $(shell find cmd -name main.go) 14 | 15 | TARGETS := $(patsubst cmd/%/main.go,%,$(CMD_SOURCES)) 16 | RACE_TARGETS := $(patsubst cmd/%/main.go,%.race,$(CMD_SOURCES)) 17 | PANDOC_BIN := $(shell if which pandoc >/dev/null 2>&1; then echo pandoc; else echo true; fi) 18 | 19 | $(TARGETS): 20 | go build -v -i -ldflags "$(LDFLAGS)" -o $@ ./cmd/$@ 21 | 22 | $(RACE_TARGETS): 23 | go build -v -i -ldflags "$(LDFLAGS)" --race -o $@ ./cmd/$(basename $@) 24 | 25 | # build tasks 26 | rpm: docker-rpm 27 | docker-rpm: docker 28 | rm -fr $(CURDIR)/output 29 | mkdir -p $(CURDIR)/output/{BUILD,BUILDROOT,RPMS/{noarch,x86_64},SPECS,SRPMS} 30 | docker run --rm -v $(CURDIR):/source:z -v $(CURDIR)/output:/root/rpmbuild:z lemur-rpm-build 31 | 32 | local-rpm: 33 | $(MAKE) -C packaging/rpm NAME=$(NAME) VERSION=$(VERSION) RELEASE=$(PKG_RELEASE) URL=$(PROJECT_URL) 34 | 35 | docker: 36 | $(MAKE) -C packaging/docker 37 | 38 | vendor: 39 | $(MAKE) -C vendor 40 | 41 | # development tasks 42 | coverage: 43 | @-go test -v -coverprofile=cover.out $$(go list ./... | grep -v /vendor/ | grep -v /cmd/) 44 | @-go tool cover -html=cover.out -o cover.html 45 | 46 | benchmark: 47 | @echo "Running tests..." 48 | @go test -bench=. $$(go list ./... | grep -v /vendor/ | grep -v /cmd/) 49 | 50 | all: lint $(TARGETS) 51 | .DEFAULT_GOAL:=all 52 | 53 | # Installation 54 | INSTALLED_TARGETS = $(addprefix $(PREFIX)/bin/, $(TARGETS)) 55 | # test targets 56 | UAT_RACE_TARGETS_DEST := libexec/$(NAME)-testing 57 | INSTALLED_RACE_TARGETS = $(addprefix $(PREFIX)/$(UAT_RACE_TARGETS_DEST)/, $(RACE_TARGETS)) 58 | UAT_FEATURES_DEST := share/$(NAME)/test/features 59 | INSTALLED_FEATURES = $(addprefix $(PREFIX)/$(UAT_FEATURES_DEST)/, $(FEATURE_FILES)) 60 | 61 | # Sample config files 62 | # 63 | EXAMPLES = $(shell find doc -name "*.example") 64 | EXAMPLE_TARGETS = $(patsubst doc/%,%,$(EXAMPLES)) 65 | INSTALLED_EXAMPLES = $(addprefix $(PREFIX)/etc/lhsmd/, $(EXAMPLE_TARGETS)) 66 | 67 | # Cleanliness... 68 | lint: 69 | @ln -sf vendor src 70 | git rev-parse HEAD 71 | GOPATH=$(PWD):$(GOPATH) gometalinter -j2 --vendor -D gotype -D errcheck -D dupl -D gocyclo --deadline 60s ./... --exclude pdm/ 72 | @rm src 73 | 74 | # install tasks 75 | $(PREFIX)/bin/%: % 76 | install -d $$(dirname $@) 77 | install -m 755 $< $@ 78 | 79 | $(PREFIX)/$(UAT_FEATURES_DEST)/%: $(FEATURE_TESTS)/% 80 | install -d $$(dirname $@) 81 | install -m 644 $< $@ 82 | 83 | $(PREFIX)/$(UAT_RACE_TARGETS_DEST)/%: % 84 | install -d $$(dirname $@) 85 | install -m 755 $< $@ 86 | 87 | $(PREFIX)/etc/lhsmd/%: 88 | install -d $$(dirname $@) 89 | install -m 644 doc/$$(basename $@) $@ 90 | 91 | install-example: $(INSTALLED_EXAMPLES) 92 | 93 | install: $(INSTALLED_TARGETS) $(INSTALLED_MAN_TARGETS) 94 | 95 | local-install: 96 | $(MAKE) install PREFIX=usr/local 97 | 98 | 99 | # clean up tasks 100 | clean-docs: 101 | rm -rf ./docs 102 | 103 | clean-deps: 104 | rm -rf $(DEPDIR) 105 | 106 | clean: clean-docs clean-deps 107 | rm -rf ./usr 108 | rm -f $(TARGETS) 109 | rm -f $(RACE_TARGETS) 110 | rm -f $(MAN_TARGETS) 111 | 112 | .PHONY: $(TARGETS) $(RACE_TARGETS) 113 | .PHONY: all check test rpm deb install local-install packages coverage docs jekyll deploy-docs clean-docs clean-deps clean vendor 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## How to build executable for CBLD environment and run tests 2 | 3 | This doc briefly goes over the steps involved in building the plugin for testing inside a deployed Lustre cluster. 4 | 5 | ### Preparation 6 | 7 | - Install Docker 8 | - Run the following commands: 9 | - az login 10 | - az acr login --name [container-registry-name] 11 | - This is to allow us to pull images from a private registry called `[container-registry-name]`. 12 | - docker pull [container-registry-name].azurecr.io/copytoolbuildimage/gobuild 13 | 14 | ### To build the executable 15 | 16 | - docker run --name gobuild -v /{path-to-your-lemur-project-code}:/usr/src/lemur -it [container-registry-name].azurecr.io/copytoolbuildimage/gobuild:latest 17 | - Explanation: 18 | - We are starting a container that has all the right dependencies built (especially Lustre itself) 19 | - The -v flag is mapping a path on your machine (the lemur project) into a specific path on the container, so that the source code shows up inside the container. 20 | - Any changes you make in /usr/src/lemur on the laptop OR in the container should persist. 21 | - cd /usr/src/lemur 22 | - You should see your lemur project's source code now 23 | - ./build_plugin.sh 24 | - This step should run for a while, and the output will be in the `dist` folder 25 | 26 | ### To clean up 27 | 28 | - Break out of the container with ‘exit’. 29 | - docker rm gobuild 30 | - This gets rid of the container so that you can `docker run` the next time, otherwise the name will be occupied. 31 | 32 | ### How to debug if something goes wrong with compilation 33 | 34 | - The environment variables are set appropriately already, but you can examine them with ‘set’ command. 35 | - The most important dependencies to check are: 36 | - The CGO_CFLAGS which specifies where to look for the C header files 37 | - The Lustre binaries themselves have to be available to the C compiler, check to make sure 38 | - Check with Joe to see if the image has changed in unexpected ways. 39 | 40 | ### How to test the compiled executable 41 | 42 | - Deploy a cluster as the guide specifies 43 | - SSH into the HSM VM (the name starts with 'APRI') by looking up its private IP address in the portal 44 | - Transfer your newly compiled build to that machine (you can do so via a storage container) 45 | - See if the current lhsmd is running: 46 | - ps -A | grep lhsmd 47 | - Stop the current lhsmd: 48 | - sudo systemctl stop lhsmd 49 | - Move your new build into /usr/laaso/bin/ with a new name 50 | - Edit the agent config: sudo vi /etc/lhsmd/agent 51 | - point it to use the new build instead 52 | - Add a new config for the new build: cp /etc/lhsmd/lhsm-plugin-az /etc/lhsmd/lhsm-plugin-new-name 53 | - Either: 54 | - restart the lhsmd with: sudo systemctl start lhsmd 55 | - Or better yet run it directly in a separate window: /usr/laaso/bin/lhsmd -config /etc/lhsmd/agent 56 | - This allows you to see the output directly 57 | - cd /lustre/client 58 | - Exercise the copy tool by triggering restore operations 59 | 60 | 61 | ### Extra notes 62 | 63 | - The lhsmd config is located at: sudo vi /etc/systemd/system/lhsmd.service 64 | - The plugin config is located at: sudo vi /etc/lhsmd/lhsm-plugin-az 65 | - The name of the config is the same as the plugin. 66 | - If you've added new fields to the plugin config, you need to update this file before running lhsmd. 67 | - lhsmd is the parent that starts the plugin, so stopping lhsmd also kills the copy tool 68 | - The sys logs are located at /var/log/daemon.log 69 | - You can trigger multiple file restores: 70 | - Example: 71 | ```bash 72 | IFS=$'\n'; set -f 73 | for f in $(find /lustre/ST0202 -type f); do lfs hsm_restore "$f"; done 74 | unset IFS; set +f 75 | ``` 76 | 77 | -------------------------------------------------------------------------------- /README_previous.md: -------------------------------------------------------------------------------- 1 | # Azure HSM Agent and Data Movers for Lustre 2 | 3 | This has been updated for Azure to provide a copy tool back to BLOB storage. 4 | 5 | RPMS are available here for this initial version: 6 | 7 | __Lustre 2.10__ 8 | 9 | * https://azurehpc.azureedge.net/rpms/lemur-azure-hsm-agent-1.0.0-lustre_2.10.x86_64.rpm 10 | * https://azurehpc.azureedge.net/rpms/lemur-azure-data-movers-1.0.0-lustre_2.10.x86_64.rpm 11 | 12 | __Lustre 2.12__ 13 | 14 | * https://azurehpc.azureedge.net/rpms/lemur-azure-hsm-agent-1.0.0-lustre_2.12.x86_64.rpm 15 | * https://azurehpc.azureedge.net/rpms/lemur-azure-data-movers-1.0.0-lustre_2.12.x86_64.rpm 16 | 17 | 18 | ## Building 19 | 20 | Commands used to build RPMS: 21 | 22 | ``` 23 | wget https://dl.google.com/go/go1.12.1.linux-amd64.tar.gz 24 | sudo tar -C /usr/local -xzf go1.12.1.linux-amd64.tar.gz 25 | export PATH=/usr/local/go/bin:$PATH 26 | sudo yum install -y git gcc rpmdevtools rpmlint 27 | 28 | git clone https://github.com/edwardsp/lemur.git 29 | cd lemur 30 | go mod init 31 | go mod vendor 32 | make local-rpm 33 | ``` 34 | 35 | > Note: Lustre 2.12 has an API change in the HSM so the go-lustre needs patching. Change the `int` to `enum_changelog_rec_flages`. 36 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | - master 3 | - dev 4 | - azure-piplines 5 | 6 | pool: 7 | vmImage: 'ubuntu-latest' 8 | 9 | steps: 10 | - task: GoTool@0 11 | name: 'Set_up_Golang' 12 | inputs: 13 | version: '1.14' 14 | 15 | - script: | 16 | echo Running tests... 17 | go test -timeout 25m -race -short -cover ./cmd/lhsm-plugin-az-core 18 | displayName: 'Run lhms-plugin-az-core tests' 19 | env: 20 | ACCOUNT_NAME: $(ACCOUNT_NAME) 21 | ACCOUNT_KEY: $(ACCOUNT_KEY) 22 | -------------------------------------------------------------------------------- /build_plugin.sh: -------------------------------------------------------------------------------- 1 | export CGO_CFLAGS='-I/usr/src/lustre-2.12.5/lustre/include -I/usr/src/lustre-2.12.5/lustre/include/uapi' 2 | export GOPATH=/go 3 | #go build -v -i -ldflags "-X 'main.version=0.6.0_56_g286df59_dirty'" -o dist/lhsm-plugin-az ./cmd/lhsm-plugin-az 4 | go build -o dist/lhsm-plugin-az ./cmd/lhsm-plugin-az 5 | go build -o dist/azure-import ./cmd/azure-import 6 | go build -o dist/changelog-reader ./cmd/changelog-reader 7 | go build -o dist/lhsmd ./cmd/lhsmd 8 | -------------------------------------------------------------------------------- /buildspec.yml: -------------------------------------------------------------------------------- 1 | version: 0.1 2 | 3 | environment_variables: 4 | plaintext: 5 | TOPDIR: "/tmp/rpmbuild" 6 | 7 | phases: 8 | pre_build: 9 | commands: 10 | - echo Pre-build started on `date` 11 | - rm -fr $TOPDIR 12 | - mkdir -p $TOPDIR/{RPMS/{x86_64,noarch},SRPMS,BUILD,BUILDROOT,SPECS,SOURCES} 13 | - mkdir -p $GOPATH/src/github.com/edwardsp/lemur && cp -a . $_ && cd $_ && rm -fr vendor/github.com/wastore/logging/debug/examples && go get -v ./... && go get -v github.com/fortytw2/leaktest' ) 14 | build: 15 | commands: 16 | - echo Build started on `date` 17 | - make local-rpm TOPDIR=$TOPDIR 18 | - make -C packaging/rpm repo TOPDIR=$TOPDIR 19 | - cd $TOPDIR/RPMS && zip -r /tmp/repo.zip . 20 | post_build: 21 | commands: 22 | - echo Build completed on `date` 23 | artifacts: 24 | files: 25 | - /tmp/repo.zip 26 | discard-paths: yes 27 | -------------------------------------------------------------------------------- /cmd/azure-list-versions/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/url" 8 | "os" 9 | 10 | "github.com/Azure/azure-storage-blob-go/azblob" 11 | ) 12 | 13 | const blobEndPoint string = "https://%s.blob.core.windows.net/" 14 | 15 | func main() { 16 | 17 | accountName, accountKey := os.Getenv("STORAGE_ACCOUNT"), os.Getenv("STORAGE_KEY") 18 | if len(accountName) == 0 { 19 | log.Fatal("The STORAGE_ACCOUNT environment variable is not set") 20 | } 21 | if len(accountKey) == 0 { 22 | log.Fatal("The STORAGE_KEY environment variable is not set") 23 | } 24 | 25 | container := os.Args[1] 26 | path := os.Args[2] 27 | 28 | credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) 29 | if err != nil { 30 | log.Fatal("Invalid credentials with error: " + err.Error()) 31 | } 32 | p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) 33 | 34 | ctx := context.Background() 35 | cu, _ := url.Parse(fmt.Sprintf(blobEndPoint+"%s", accountName, container)) 36 | containerURL := azblob.NewContainerURL(*cu, p) 37 | 38 | listBlobsResp, err := containerURL.ListBlobsFlatSegment( 39 | ctx, azblob.Marker{}, 40 | azblob.ListBlobsSegmentOptions{ Prefix: path, Details: azblob.BlobListingDetails{Versions: true}}) 41 | if err != nil { 42 | log.Fatal("Failed to list blobs: " + err.Error()) 43 | } 44 | for _, b := range listBlobsResp.Segment.BlobItems { 45 | fmt.Println(b.Name, *b.VersionID) 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /cmd/lhsm-plugin-az-core/archive.go: -------------------------------------------------------------------------------- 1 | package lhsm_plugin_az_core 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "path" 11 | "strings" 12 | "sync" 13 | "syscall" 14 | 15 | "github.com/Azure/azure-pipeline-go/pipeline" 16 | "github.com/Azure/azure-storage-blob-go/azblob" 17 | "github.com/edwardsp/go-lustre/llapi" 18 | "github.com/edwardsp/lemur/cmd/util" 19 | ) 20 | 21 | type ArchiveOptions struct { 22 | AccountName string 23 | AccountSuffix string 24 | ContainerName string 25 | ResourceSAS string 26 | MountRoot string 27 | BlobName string 28 | SourcePath string 29 | Credential azblob.Credential 30 | Parallelism uint16 31 | BlockSize int64 32 | Pacer util.Pacer 33 | ExportPrefix string 34 | HTTPClient *http.Client 35 | } 36 | 37 | const parallelDirCount = 64 // Number parallel dir metadata uploads 38 | 39 | func upload(ctx context.Context, o ArchiveOptions, blobPath string) (_ int64, err error) { 40 | p := util.NewPipeline(ctx, o.Credential, o.Pacer, azblob.PipelineOptions{HTTPSender: util.HTTPClientFactory(o.HTTPClient)}) 41 | cURL, _ := url.Parse(fmt.Sprintf("https://%s.%s/%s%s", o.AccountName, o.AccountSuffix, o.ContainerName, o.ResourceSAS)) 42 | containerURL := azblob.NewContainerURL(*cURL, p) 43 | blobURL := containerURL.NewBlockBlobURL(blobPath) 44 | meta := azblob.Metadata{} 45 | 46 | //Get owner, group and perms 47 | fileName := path.Join(o.MountRoot, blobPath) 48 | fileInfo, err := os.Stat(fileName) 49 | if err != nil { 50 | util.Log(pipeline.LogError, fmt.Sprintf("Archiving %s. Failed to get fileInfo: %s", blobPath, err.Error())) 51 | return 0, err 52 | } 53 | 54 | owner := fmt.Sprintf("%d", fileInfo.Sys().(*syscall.Stat_t).Uid) 55 | permissions := uint32(fileInfo.Mode().Perm()) 56 | if fileInfo.Mode()&os.ModeSticky != 0 { 57 | permissions |= syscall.S_ISVTX 58 | } 59 | group := fmt.Sprintf("%d", fileInfo.Sys().(*syscall.Stat_t).Gid) 60 | modTime := fileInfo.ModTime().Format("2006-01-02 15:04:05 -0700") 61 | 62 | meta["permissions"] = fmt.Sprintf("%04o", permissions) 63 | meta["modtime"] = modTime 64 | meta["owner"] = owner 65 | meta["group"] = group 66 | 67 | // get lustre stripe info 68 | var layout *llapi.DataLayout 69 | if fileInfo.IsDir() { 70 | layout, err = llapi.DirDataLayout(fileName) 71 | } else { 72 | layout, err = llapi.FileDataLayout(fileName) 73 | } 74 | if err == nil { 75 | meta["lfs_stripe_count"] = fmt.Sprintf("%d", layout.StripeCount) 76 | meta["lfs_stripe_size"] = fmt.Sprintf("%d", layout.StripeSize) 77 | } else { 78 | util.Log(pipeline.LogInfo, fmt.Sprintf("Archiving %s. Using default stripe layout - failed to get lustre stripe layout for %s: %s", blobPath, fileName, err.Error())) 79 | } 80 | 81 | if fileInfo.IsDir() { 82 | meta["hdi_isfolder"] = "true" 83 | _, err = blobURL.Upload(ctx, bytes.NewReader(nil), azblob.BlobHTTPHeaders{}, meta, azblob.BlobAccessConditions{}, azblob.AccessTierNone) 84 | } else { 85 | fi, _ := os.Stat(path.Join(o.MountRoot, blobPath)) 86 | file, _ := os.Open(path.Join(o.MountRoot, blobPath)) 87 | defer file.Close() 88 | 89 | _, err = azblob.UploadFileToBlockBlob( 90 | ctx, file, blobURL, 91 | azblob.UploadToBlockBlobOptions{ 92 | BlockSize: util.GetBlockSize(fi.Size(), o.BlockSize), 93 | Parallelism: o.Parallelism, 94 | Metadata: meta, 95 | }) 96 | } 97 | 98 | if err != nil { 99 | util.Log(pipeline.LogError, fmt.Sprintf("Archiving %s. Failed to upload blob: %s", blobPath, err.Error())) 100 | return 0, err 101 | } 102 | 103 | return fileInfo.Size(), err 104 | } 105 | 106 | //Archive copies local file to HNS 107 | func Archive(o ArchiveOptions) (size int64, err error) { 108 | util.Log(pipeline.LogInfo, fmt.Sprintf("Archiving %s", o.BlobName)) 109 | ctx, cancel := context.WithCancel(context.Background()) 110 | defer cancel() 111 | wg := sync.WaitGroup{} 112 | 113 | parents := strings.Split(o.BlobName, string(os.PathSeparator)) 114 | parents = parents[:len(parents)-1] // Exclude the file itself (processed separately) 115 | wg.Add(len(parents) + 1) // Parent directories + 1 for the file. 116 | 117 | // Upload the file 118 | go func() { 119 | size, err = upload(ctx, o, o.BlobName) 120 | wg.Done() 121 | }() 122 | 123 | //parallely upload directories starting from root. 124 | guard := make(chan struct{}, parallelDirCount) //Guard maintains bounded paralleism for uploading directories 125 | defer close(guard) 126 | 127 | blobPath := "" 128 | for _, currDir := range parents { 129 | blobPath = path.Join(blobPath, currDir) //keep appending path to the url 130 | 131 | guard <- struct{}{} //block till we've enough room. 132 | go func(p string) { 133 | upload(ctx, o, p) 134 | <-guard //release a space in guard. 135 | wg.Done() 136 | }(blobPath) 137 | } 138 | 139 | wg.Wait() 140 | 141 | return size, err 142 | } 143 | -------------------------------------------------------------------------------- /cmd/lhsm-plugin-az-core/remove.go: -------------------------------------------------------------------------------- 1 | package lhsm_plugin_az_core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "path" 8 | "strings" 9 | 10 | "github.com/Azure/azure-pipeline-go/pipeline" 11 | "github.com/Azure/azure-storage-blob-go/azblob" 12 | "github.com/edwardsp/lemur/cmd/util" 13 | ) 14 | 15 | type RemoveOptions struct { 16 | AccountName string 17 | AccountSuffix string 18 | ContainerName string 19 | ResourceSAS string 20 | BlobName string 21 | ExportPrefix string 22 | Credential azblob.Credential 23 | } 24 | 25 | func Remove(o RemoveOptions) error { 26 | ctx := context.TODO() 27 | p := azblob.NewPipeline(o.Credential, azblob.PipelineOptions{}) 28 | blobPath := strings.Replace(url.QueryEscape(path.Join(o.ContainerName, o.ExportPrefix, o.BlobName)), "+", "%20", -1) 29 | u, _ := url.Parse(fmt.Sprintf("https://%s.%s/%s%s", o.AccountName, o.AccountSuffix, blobPath, o.ResourceSAS)) 30 | 31 | util.Log(pipeline.LogInfo, fmt.Sprintf("Removing %s.", u.String())) 32 | 33 | // fetch the properties first so that we know how big the source blob is 34 | blobURL := azblob.NewBlobURL(*u, p) 35 | _, err := blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) 36 | return err 37 | } 38 | -------------------------------------------------------------------------------- /cmd/lhsm-plugin-az-core/restore.go: -------------------------------------------------------------------------------- 1 | package lhsm_plugin_az_core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "os" 9 | "path" 10 | "strings" 11 | 12 | "github.com/Azure/azure-pipeline-go/pipeline" 13 | "github.com/Azure/azure-storage-blob-go/azblob" 14 | "github.com/edwardsp/lemur/cmd/util" 15 | "github.com/pkg/errors" 16 | ) 17 | 18 | type RestoreOptions struct { 19 | AccountName string 20 | AccountSuffix string 21 | ContainerName string 22 | ResourceSAS string 23 | BlobName string 24 | DestinationPath string 25 | Credential azblob.Credential 26 | Parallelism uint16 27 | BlockSize int64 28 | ExportPrefix string 29 | BlobVersionID string 30 | Pacer util.Pacer 31 | HTTPClient *http.Client 32 | } 33 | 34 | var maxRetryPerDownloadBody = 5 35 | 36 | //Restore persists a blob to the local filesystem 37 | func Restore(o RestoreOptions) (int64, error) { 38 | restoreCtx := context.Background() 39 | ctx, cancel := context.WithCancel(restoreCtx) 40 | defer cancel() 41 | 42 | p := util.NewPipeline(ctx, o.Credential, o.Pacer, azblob.PipelineOptions{HTTPSender: util.HTTPClientFactory(o.HTTPClient)}) 43 | blobPath := strings.Replace(url.QueryEscape(path.Join(o.ContainerName, o.ExportPrefix, o.BlobName)), "+", "%20", -1) 44 | 45 | u, _ := url.Parse(fmt.Sprintf("https://%s.%s/%s%s", o.AccountName, o.AccountSuffix, blobPath, o.ResourceSAS)) 46 | 47 | util.Log(pipeline.LogInfo, fmt.Sprintf("Restoring %s to %s.", u.String(), o.DestinationPath)) 48 | 49 | blobURL := azblob.NewBlobURL(*u, p) 50 | if o.BlobVersionID != "" { 51 | blobURL = blobURL.WithVersionID(o.BlobVersionID) 52 | } 53 | blobProp, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{}) 54 | if err != nil { 55 | return 0, errors.Wrapf(err, "GetProperties on %s failed", o.BlobName) 56 | } 57 | contentLen := blobProp.ContentLength() 58 | 59 | file, _ := os.Create(o.DestinationPath) 60 | defer file.Close() 61 | err = azblob.DownloadBlobToFile( 62 | ctx, blobURL, 0, 0, file, 63 | azblob.DownloadFromBlobOptions{ 64 | BlockSize: util.GetBlockSize(contentLen, o.BlockSize), 65 | Parallelism: o.Parallelism, 66 | RetryReaderOptionsPerBlock: azblob.RetryReaderOptions{ 67 | MaxRetryRequests: maxRetryPerDownloadBody, 68 | NotifyFailedRead: util.NewReadLogFunc(u.String()), 69 | }, 70 | }) 71 | 72 | return contentLen, err 73 | } 74 | -------------------------------------------------------------------------------- /cmd/lhsm-plugin-az-core/zt_restore_test.go: -------------------------------------------------------------------------------- 1 | package lhsm_plugin_az_core 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/Azure/azure-pipeline-go/pipeline" 9 | "github.com/Azure/azure-storage-blob-go/azblob" 10 | "github.com/edwardsp/lemur/cmd/util" 11 | chk "gopkg.in/check.v1" 12 | ) 13 | 14 | func (s *cmdIntegrationSuite) TestRestoreSmallBlob(c *chk.C) { 15 | fileSize := 1024 16 | blockSize := 2048 17 | parallelism := 3 18 | performRestoreTest(c, fileSize, blockSize, parallelism) 19 | } 20 | 21 | // test download with the source data uploaded directly to the service from memory 22 | // this is an independent check that download works 23 | func performRestoreTest(c *chk.C, fileSize, blockSize, parallelism int) { 24 | bsu := getBSU() 25 | containerURL, containerName := createNewContainer(c, bsu) 26 | defer deleteContainer(c, containerURL) 27 | blobURL, blobName := createNewBlockBlob(c, containerURL, "") 28 | 29 | // stage the source blob with small amount of data 30 | reader, srcData := getRandomDataAndReader(fileSize) 31 | _, err := blobURL.Upload(ctx, reader, azblob.BlobHTTPHeaders{}, 32 | nil, azblob.BlobAccessConditions{}, azblob.AccessTierNone) 33 | c.Assert(err, chk.IsNil) 34 | 35 | // set up destination file 36 | destination := filepath.Join(os.TempDir(), blobName) 37 | destFile, err := os.Create(destination) 38 | c.Assert(err, chk.Equals, nil) 39 | defer destFile.Close() 40 | defer os.Remove(destination) 41 | 42 | //setup logging 43 | util.InitJobLogger(pipeline.LogDebug) 44 | 45 | // exercise restore 46 | account, key := getAccountAndKey() 47 | credential, err := azblob.NewSharedKeyCredential(account, key) 48 | c.Assert(err, chk.IsNil) 49 | blobName = containerName + "/" + blobName 50 | count, err := Restore(RestoreOptions{ 51 | AccountName: account, 52 | AccountSuffix: "blob.core.windows.net", 53 | ContainerName: "", 54 | BlobName: blobName, 55 | DestinationPath: destination, 56 | Credential: credential, 57 | Parallelism: uint16(parallelism), 58 | BlockSize: int64(blockSize), 59 | HTTPClient: &http.Client{}, 60 | }) 61 | 62 | // make sure we got the right info back 63 | c.Assert(err, chk.IsNil) 64 | c.Assert(count, chk.Equals, int64(len(srcData))) 65 | 66 | // Assert downloaded data is consistent 67 | destBuffer := make([]byte, count) 68 | n, err := destFile.Read(destBuffer) 69 | c.Assert(err, chk.Equals, nil) 70 | c.Assert(n, chk.Equals, len(srcData)) 71 | c.Assert(destBuffer, chk.DeepEquals, srcData) 72 | } 73 | -------------------------------------------------------------------------------- /cmd/lhsm-plugin-posix/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "os" 9 | "os/signal" 10 | "path" 11 | "syscall" 12 | 13 | "github.com/pkg/errors" 14 | 15 | "github.com/edwardsp/lemur/cmd/lhsm-plugin-posix/posix" 16 | "github.com/edwardsp/lemur/dmplugin" 17 | "github.com/edwardsp/lemur/pkg/fsroot" 18 | "github.com/intel-hpdd/logging/alert" 19 | "github.com/intel-hpdd/logging/debug" 20 | ) 21 | 22 | type ( 23 | posixConfig struct { 24 | NumThreads int `hcl:"num_threads"` 25 | Archives posix.ArchiveSet `hcl:"archive"` 26 | Checksums *posix.ChecksumConfig `hcl:"checksums"` 27 | } 28 | ) 29 | 30 | func (c *posixConfig) String() string { 31 | return dmplugin.DisplayConfig(c) 32 | } 33 | 34 | func (c *posixConfig) Merge(other *posixConfig) *posixConfig { 35 | result := new(posixConfig) 36 | 37 | if other.NumThreads > 0 { 38 | result.NumThreads = other.NumThreads 39 | } else { 40 | result.NumThreads = c.NumThreads 41 | 42 | } 43 | 44 | result.Archives = c.Archives.Merge(other.Archives) 45 | result.Checksums = c.Checksums.Merge(other.Checksums) 46 | 47 | return result 48 | } 49 | 50 | func start(plugin *dmplugin.Plugin, cfg *posixConfig) { 51 | // All base filesystem operations will be relative to current directory 52 | err := os.Chdir(plugin.Base()) 53 | if err != nil { 54 | alert.Abort(errors.Wrap(err, "chdir failed")) 55 | } 56 | 57 | interruptHandler(func() { 58 | plugin.Stop() 59 | }) 60 | 61 | for _, a := range cfg.Archives { 62 | mover, err := posix.NewMover(a) 63 | if err != nil { 64 | alert.Abort(errors.Wrap(err, "Unable to create new POSIX mover")) 65 | } 66 | 67 | plugin.AddMover(&dmplugin.Config{ 68 | Mover: mover, 69 | NumThreads: cfg.NumThreads, 70 | ArchiveID: uint32(a.ID), 71 | }) 72 | } 73 | 74 | plugin.Run() 75 | } 76 | 77 | func getMergedConfig(plugin *dmplugin.Plugin) (*posixConfig, error) { 78 | baseCfg := &posixConfig{ 79 | Checksums: &posix.ChecksumConfig{}, 80 | } 81 | 82 | var cfg posixConfig 83 | err := dmplugin.LoadConfig(plugin.ConfigFile(), &cfg) 84 | if err != nil { 85 | return nil, errors.Errorf("Failed to load config: %s", err) 86 | } 87 | 88 | return baseCfg.Merge(&cfg), nil 89 | } 90 | 91 | func main() { 92 | debug.Printf("Starting main...\n") 93 | 94 | plugin, err := dmplugin.New(path.Base(os.Args[0]), func(path string) (fsroot.Client, error) { 95 | return fsroot.New(path) 96 | }) 97 | if err != nil { 98 | alert.Abort(errors.Wrap(err, "failed to initialize plugin")) 99 | } 100 | defer plugin.Close() 101 | 102 | cfg, err := getMergedConfig(plugin) 103 | if err != nil { 104 | alert.Abort(errors.Wrap(err, "Unable to determine plugin configuration")) 105 | } 106 | 107 | debug.Printf("PosixMover configuration:\n%v", cfg) 108 | 109 | if len(cfg.Archives) == 0 { 110 | alert.Abort(errors.New("Invalid configuration: No archives defined")) 111 | } 112 | 113 | for _, archive := range cfg.Archives { 114 | debug.Print(archive) 115 | if err := archive.CheckValid(); err != nil { 116 | alert.Abort(errors.Wrap(err, "Invalid configuration")) 117 | } 118 | } 119 | 120 | posix.DefaultChecksums = *cfg.Checksums 121 | 122 | start(plugin, cfg) 123 | } 124 | 125 | func interruptHandler(once func()) { 126 | c := make(chan os.Signal, 1) 127 | signal.Notify(c, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM) 128 | 129 | go func() { 130 | stopping := false 131 | for sig := range c { 132 | debug.Printf("signal received: %s", sig) 133 | if !stopping { 134 | stopping = true 135 | once() 136 | } 137 | } 138 | }() 139 | } 140 | -------------------------------------------------------------------------------- /cmd/lhsm-plugin-posix/test-fixtures/lhsm-plugin-posix-badarchive: -------------------------------------------------------------------------------- 1 | num_threads = 42 2 | 3 | archive "1" { 4 | root = "/tmp/archives/1" 5 | } 6 | -------------------------------------------------------------------------------- /cmd/lhsm-plugin-posix/test-fixtures/lhsm-plugin-posix.checksums: -------------------------------------------------------------------------------- 1 | checksums { 2 | disabled = true 3 | } 4 | 5 | archive "1" { 6 | id = 1 7 | root = "/tmp/archives/1" 8 | checksums { 9 | disabled = false 10 | } 11 | } 12 | 13 | archive "2" { 14 | id = 2 15 | root = "/tmp/archives/2" 16 | checksums { 17 | disable_compare_on_restore = true 18 | } 19 | } 20 | 21 | archive "3" { 22 | id = 3 23 | root = "/tmp/archives/3" 24 | } 25 | -------------------------------------------------------------------------------- /cmd/lhsm-plugin-posix/test-fixtures/lhsm-plugin-posix.test: -------------------------------------------------------------------------------- 1 | num_threads = 42 2 | 3 | archive "1" { 4 | id = 1 5 | root = "/tmp/archives/1" 6 | } 7 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/action_stats.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package agent 6 | 7 | import ( 8 | "fmt" 9 | "sync" 10 | "sync/atomic" 11 | "time" 12 | 13 | "golang.org/x/net/context" 14 | 15 | "github.com/dustin/go-humanize" 16 | "github.com/rcrowley/go-metrics" 17 | 18 | "github.com/intel-hpdd/logging/audit" 19 | "github.com/intel-hpdd/logging/debug" 20 | ) 21 | 22 | // ActionStats is a synchronized container for ArchiveStats instances 23 | type ActionStats struct { 24 | sync.Mutex 25 | stats map[int]*ArchiveStats 26 | } 27 | 28 | // ArchiveStats is a per-archive container of statistics for that backend 29 | type ArchiveStats struct { 30 | changes uint64 31 | queueLength metrics.Counter 32 | completed metrics.Timer 33 | } 34 | 35 | // NewActionStats initializes a new ActionStats container 36 | func NewActionStats() *ActionStats { 37 | return &ActionStats{ 38 | stats: make(map[int]*ArchiveStats), 39 | } 40 | } 41 | 42 | func (as *ActionStats) update() { 43 | for _, k := range as.Archives() { 44 | archive := as.GetIndex(k) 45 | changes := atomic.LoadUint64(&archive.changes) 46 | if changes != 0 { 47 | atomic.AddUint64(&archive.changes, -changes) 48 | audit.Logf("archive:%d %s", k, archive) 49 | } 50 | } 51 | } 52 | 53 | func (as *ActionStats) run(ctx context.Context) { 54 | for { 55 | select { 56 | case <-ctx.Done(): 57 | debug.Print("Shutting down stats collector") 58 | return 59 | case <-time.After(10 * time.Second): 60 | as.update() 61 | } 62 | } 63 | } 64 | 65 | // Start creates a new goroutine for collecting archive stats 66 | func (as *ActionStats) Start(ctx context.Context) { 67 | go as.run(ctx) 68 | debug.Print("Stats collector started in background") 69 | } 70 | 71 | // StartAction increments stats counters when an action starts 72 | func (as *ActionStats) StartAction(a *Action) { 73 | s := as.GetIndex(int(a.aih.ArchiveID())) 74 | s.queueLength.Inc(1) 75 | atomic.AddUint64(&s.changes, 1) 76 | } 77 | 78 | // CompleteAction updates various stats when an action is complete 79 | func (as *ActionStats) CompleteAction(a *Action, rc int) { 80 | s := as.GetIndex(int(a.aih.ArchiveID())) 81 | s.queueLength.Dec(1) 82 | s.completed.UpdateSince(a.start) 83 | atomic.AddUint64(&s.changes, 1) 84 | } 85 | 86 | // GetIndex returns the *ArchiveStats corresponding to the supplied archive 87 | // number 88 | func (as *ActionStats) GetIndex(i int) *ArchiveStats { 89 | as.Lock() 90 | defer as.Unlock() 91 | s, ok := as.stats[i] 92 | if !ok { 93 | s = &ArchiveStats{ 94 | queueLength: metrics.NewCounter(), 95 | completed: metrics.NewTimer(), 96 | } 97 | metrics.Register(fmt.Sprintf("archive%dCompleted", i), s.completed) 98 | metrics.Register(fmt.Sprintf("archive%dQueueLength", i), s.queueLength) 99 | as.stats[i] = s 100 | } 101 | return s 102 | } 103 | 104 | // Archives returns a slice of archive numbers corresponding to instrumented 105 | // backends 106 | func (as *ActionStats) Archives() (v []int) { 107 | as.Lock() 108 | defer as.Unlock() 109 | for k := range as.stats { 110 | v = append(v, k) 111 | } 112 | return 113 | } 114 | 115 | func (s *ArchiveStats) String() string { 116 | ps := s.completed.Percentiles([]float64{0.5, .75, 0.95, 0.99, 0.999}) 117 | return fmt.Sprintf("total:%v queue:%v %v/%v/%v min:%v max:%v mean:%v median:%v 75%%:%v 95%%:%v 99%%:%v 99.9%%:%v", 118 | humanize.Comma(s.completed.Count()), 119 | humanize.Comma(s.queueLength.Count()), 120 | humanize.Comma(int64(s.completed.Rate1())), 121 | humanize.Comma(int64(s.completed.Rate5())), 122 | humanize.Comma(int64(s.completed.Rate15())), 123 | time.Duration(s.completed.Min()), 124 | time.Duration(s.completed.Max()), 125 | time.Duration(int64(s.completed.Mean())), 126 | time.Duration(int64(ps[0])), 127 | time.Duration(int64(ps[1])), 128 | time.Duration(int64(ps[2])), 129 | time.Duration(int64(ps[3])), 130 | time.Duration(int64(ps[4]))) 131 | } 132 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/agent_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package agent_test 6 | 7 | import ( 8 | "testing" 9 | "time" 10 | 11 | "github.com/edwardsp/lemur/cmd/lhsmd/agent" 12 | _ "github.com/edwardsp/lemur/cmd/lhsmd/transport/grpc" 13 | "github.com/edwardsp/lemur/pkg/fsroot" 14 | "github.com/edwardsp/go-lustre/hsm" 15 | 16 | "golang.org/x/net/context" 17 | ) 18 | 19 | func TestAgentStartStop(t *testing.T) { 20 | cfg := agent.DefaultConfig() 21 | cfg.Transport.SocketDir = "/tmp" 22 | as := hsm.NewTestSource() 23 | ta, err := agent.New(cfg, fsroot.Test(cfg.AgentMountpoint()), as) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | 28 | go func() { 29 | if err := ta.Start(context.Background()); err != nil { 30 | t.Fatalf("Test agent startup failed: %s", err) 31 | } 32 | }() 33 | 34 | // Wait for the agent to signal that it has started 35 | ta.StartWaitFor(5 * time.Second) 36 | 37 | ta.Stop() 38 | } 39 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package agent 6 | 7 | import ( 8 | "io/ioutil" 9 | "os" 10 | "path" 11 | "reflect" 12 | "runtime" 13 | "testing" 14 | 15 | "github.com/edwardsp/lemur/cmd/lhsmd/config" 16 | "github.com/edwardsp/go-lustre/fs/spec" 17 | ) 18 | 19 | func TestConfiguredPlugins(t *testing.T) { 20 | loaded, err := LoadConfig("./test-fixtures/plugin-config") 21 | if err != nil { 22 | t.Fatalf("err: %s", err) 23 | } 24 | 25 | expected := []*PluginConfig{ 26 | { 27 | Name: "lhsm-plugin-posix", 28 | BinPath: config.DefaultPluginDir + "/lhsm-plugin-posix", 29 | AgentConnection: "", 30 | ClientMount: "/mnt/lhsmd/lhsm-plugin-posix", 31 | RestartOnFailure: true, 32 | }, 33 | { 34 | Name: "lhsm-plugin-s3", 35 | BinPath: config.DefaultPluginDir + "/lhsm-plugin-s3", 36 | AgentConnection: "", 37 | ClientMount: "/mnt/lhsmd/lhsm-plugin-s3", 38 | RestartOnFailure: true, 39 | }, 40 | { 41 | Name: "lhsm-plugin-noop", 42 | BinPath: config.DefaultPluginDir + "/lhsm-plugin-noop", 43 | AgentConnection: "", 44 | ClientMount: "/mnt/lhsmd/lhsm-plugin-noop", 45 | RestartOnFailure: true, 46 | }, 47 | } 48 | t.Skip("TODO: Fix test to deal with unix socket") 49 | got := loaded.Plugins() 50 | if !reflect.DeepEqual(got, expected) { 51 | t.Fatalf("\nexpected:\n%s\ngot:\n%s\n", expected, got) 52 | } 53 | } 54 | 55 | func TestLoadConfig(t *testing.T) { 56 | loaded, err := LoadConfig("./test-fixtures/good-config") 57 | if err != nil { 58 | t.Fatalf("err: %s", err) 59 | } 60 | 61 | expectedDevice, err := spec.ClientDeviceFromString("10.211.55.37@tcp0:/testFs") 62 | if err != nil { 63 | t.Fatalf("err: %s", err) 64 | } 65 | expected := &Config{ 66 | MountRoot: "/mnt/lhsmd", 67 | ClientDevice: expectedDevice, 68 | ClientMountOptions: []string{ 69 | "user_xattr", 70 | }, 71 | Processes: runtime.NumCPU(), 72 | InfluxDB: &influxConfig{ 73 | URL: "http://172.17.0.4:8086", 74 | DB: "lhsmd", 75 | }, 76 | EnabledPlugins: []string{ 77 | "lhsm-plugin-posix", 78 | }, 79 | Snapshots: &snapshotConfig{ 80 | Enabled: false, 81 | }, 82 | PluginDir: "/go/bin", 83 | Transport: &transportConfig{ 84 | Type: "grpc", 85 | SocketDir: "/tmp", 86 | }, 87 | } 88 | 89 | if !reflect.DeepEqual(loaded, expected) { 90 | t.Fatalf("\nexpected:\n%s\ngot:\n%s\n", expected, loaded) 91 | } 92 | } 93 | 94 | func TestMergedConfig(t *testing.T) { 95 | defCfg := NewConfig() 96 | loaded, err := LoadConfig("./test-fixtures/merge-config") 97 | if err != nil { 98 | t.Fatalf("err: %s", err) 99 | } 100 | got := defCfg.Merge(loaded) 101 | 102 | expectedDevice, err := spec.ClientDeviceFromString("10.211.55.37@tcp0:/testFs") 103 | if err != nil { 104 | t.Fatalf("err: %s", err) 105 | } 106 | expected := &Config{ 107 | MountRoot: "/mnt/lhsmd", 108 | ClientDevice: expectedDevice, 109 | ClientMountOptions: []string{ 110 | "user_xattr", 111 | }, 112 | Processes: runtime.NumCPU(), 113 | InfluxDB: &influxConfig{ 114 | URL: "http://172.17.0.4:8086", 115 | DB: "lhsmd", 116 | }, 117 | EnabledPlugins: []string{ 118 | "lhsm-plugin-posix", 119 | }, 120 | PluginDir: "/go/bin", 121 | Snapshots: &snapshotConfig{ 122 | Enabled: false, 123 | }, 124 | Transport: &transportConfig{ 125 | Type: "grpc", 126 | SocketDir: "/var/run/lhsmd", 127 | }, 128 | } 129 | 130 | if !reflect.DeepEqual(got, expected) { 131 | t.Fatalf("\nexpected:\n%s\ngot:\n%s\n", expected, got) 132 | } 133 | } 134 | 135 | func TestJsonConfig(t *testing.T) { 136 | cfg, err := LoadConfig("./test-fixtures/json-config") 137 | 138 | if err != nil { 139 | t.Fatalf("Error from LoadConfig(): %s", err) 140 | } 141 | 142 | if cfg.ClientDevice == nil { 143 | t.Fatal("ClientDevice should not be nil") 144 | } 145 | } 146 | 147 | func TestConfigSaveLoad(t *testing.T) { 148 | startCfg := DefaultConfig() 149 | cd, err := spec.ClientDeviceFromString("1.2.3.4@tcp:/foo") 150 | if err != nil { 151 | t.Fatal(err) 152 | } 153 | startCfg.ClientDevice = cd 154 | 155 | td, err := ioutil.TempDir("", "agent-config-test") 156 | if err != nil { 157 | t.Fatal(err) 158 | } 159 | defer os.RemoveAll(td) 160 | 161 | cfgFile := path.Join(td, "cfg") 162 | if err = ioutil.WriteFile(cfgFile, []byte(startCfg.String()), 0644); err != nil { 163 | t.Fatal(err) 164 | } 165 | 166 | loaded, err := LoadConfig(cfgFile) 167 | if err != nil { 168 | t.Fatal(err) 169 | } 170 | 171 | if startCfg.String() != loaded.String() { 172 | t.Fatalf("start cfg != loaded\nstart:\n%s\nloaded:\n%s\n", startCfg, loaded) 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/endpoints.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package agent 6 | 7 | import ( 8 | "errors" 9 | "sync" 10 | "sync/atomic" 11 | ) 12 | 13 | type ( 14 | // Handle is an endpoint handle (unique id) 15 | Handle uint64 16 | 17 | // Endpoints represents a collection of Endpoints and their handles 18 | Endpoints struct { 19 | sync.Mutex 20 | nextHandle int64 21 | endpoints map[uint32]Endpoint 22 | handles map[Handle]uint32 23 | } 24 | 25 | // Endpoint defines an interface for HSM backends 26 | Endpoint interface { 27 | Send(*Action) 28 | } 29 | ) 30 | 31 | // NewEndpoints returns a new *Endpoints instance 32 | func NewEndpoints() *Endpoints { 33 | return &Endpoints{ 34 | endpoints: make(map[uint32]Endpoint), 35 | handles: make(map[Handle]uint32), 36 | } 37 | } 38 | 39 | // Get returns an Endpoint or nil, given a lookup id 40 | func (all *Endpoints) Get(a uint32) (Endpoint, bool) { 41 | all.Lock() 42 | defer all.Unlock() 43 | return all.get(a) 44 | } 45 | 46 | // GetWithHandle returns an Endpoint or nil, given a Handle 47 | func (all *Endpoints) GetWithHandle(h *Handle) (Endpoint, bool) { 48 | all.Lock() 49 | defer all.Unlock() 50 | return all.getWithHandle(h) 51 | } 52 | 53 | func (all *Endpoints) get(a uint32) (Endpoint, bool) { 54 | // all must already be locked. 55 | e, ok := all.endpoints[a] 56 | if !ok { 57 | return nil, ok 58 | } 59 | return e, true 60 | } 61 | 62 | func (all *Endpoints) getWithHandle(h *Handle) (Endpoint, bool) { 63 | // all must already be locked. 64 | a, ok := all.handles[*h] 65 | if !ok { 66 | return nil, ok 67 | } 68 | 69 | return all.get(a) 70 | } 71 | 72 | func (all *Endpoints) newHandle() *Handle { 73 | h := Handle(atomic.AddInt64(&all.nextHandle, 1)) 74 | return &h 75 | } 76 | 77 | // Add registers a new Endpoint 78 | func (all *Endpoints) Add(a uint32, e Endpoint) (*Handle, error) { 79 | h := all.newHandle() 80 | all.Lock() 81 | defer all.Unlock() 82 | 83 | if _, ok := all.get(a); ok { 84 | return nil, errors.New("Endpoint already exists") 85 | } 86 | 87 | all.endpoints[a] = e 88 | all.handles[*h] = a 89 | return h, nil 90 | } 91 | 92 | // NewHandle returns a new *Handle 93 | func (all *Endpoints) NewHandle(a uint32) (*Handle, error) { 94 | all.Lock() 95 | defer all.Unlock() 96 | 97 | if _, ok := all.get(a); !ok { 98 | return nil, errors.New("Endpoint does not exist") 99 | } 100 | 101 | h := all.newHandle() 102 | all.handles[*h] = a 103 | return h, nil 104 | 105 | } 106 | 107 | // RemoveHandle removes the given handle from the collection of handles 108 | func (all *Endpoints) RemoveHandle(h *Handle) { 109 | all.Lock() 110 | defer all.Unlock() 111 | 112 | delete(all.handles, *h) 113 | } 114 | 115 | // Remove removes the given handle and its associated Endpoint 116 | func (all *Endpoints) Remove(h *Handle) Endpoint { 117 | all.Lock() 118 | defer all.Unlock() 119 | a, ok := all.handles[*h] 120 | if !ok { 121 | return nil 122 | } 123 | 124 | if e, ok := all.get(a); ok { 125 | delete(all.handles, *h) 126 | delete(all.endpoints, a) 127 | return e 128 | } 129 | 130 | return nil 131 | } 132 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/fileid/fileid.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package fileid 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/edwardsp/go-lustre" 11 | "github.com/edwardsp/go-lustre/fs" 12 | "github.com/edwardsp/go-lustre/pkg/xattr" 13 | "github.com/intel-hpdd/logging/debug" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | const xattrUUID = "trusted.lhsm_uuid" 18 | const xattrHash = "trusted.lhsm_hash" 19 | const xattrURL = "trusted.lhsm_url" 20 | 21 | type ( 22 | manager interface { 23 | update(string, []byte) error 24 | set(string, []byte) error 25 | get(string) ([]byte, error) 26 | } 27 | attrManager struct { 28 | attr string 29 | } 30 | // Attribute is an interface for managing exctended attributes. 31 | Attribute struct { 32 | mgr manager 33 | } 34 | ) 35 | 36 | var UUID, Hash, URL Attribute 37 | 38 | func init() { 39 | defaultAttrs() 40 | } 41 | func defaultAttrs() { 42 | UUID = Attribute{newManager(xattrUUID)} 43 | Hash = Attribute{newManager(xattrHash)} 44 | URL = Attribute{newManager(xattrURL)} 45 | } 46 | 47 | // Manager returns a new attrManager 48 | func newManager(attr string) *attrManager { 49 | return &attrManager{attr: attr} 50 | } 51 | 52 | func (m *attrManager) String() string { 53 | return m.attr 54 | } 55 | 56 | func (m *attrManager) update(p string, fileID []byte) error { 57 | return m.set(p, fileID) 58 | } 59 | 60 | func (m *attrManager) set(p string, fileID []byte) error { 61 | return xattr.Lsetxattr(p, m.attr, fileID, 0) 62 | } 63 | 64 | func (m *attrManager) get(p string) ([]byte, error) { 65 | buf := make([]byte, 256) 66 | 67 | sz, err := xattr.Lgetxattr(p, m.attr, buf) 68 | if err != nil { 69 | return nil, err 70 | } 71 | return buf[0:sz], nil 72 | } 73 | 74 | func (a Attribute) String() string { 75 | return fmt.Sprintf("%s", a.mgr) 76 | } 77 | 78 | // Update updates an existing fileid attribute with a new value 79 | func (a Attribute) Update(p string, fileID []byte) error { 80 | return a.mgr.update(p, fileID) 81 | } 82 | 83 | // UpdateByFid updates an existing fileid attribute with a new value 84 | func (a Attribute) UpdateByFid(mnt fs.RootDir, fid *lustre.Fid, fileID []byte) error { 85 | p := fs.FidPath(mnt, fid) 86 | return a.Update(p, fileID) 87 | } 88 | 89 | // Set sets a fileid attribute on a file 90 | func (a Attribute) Set(p string, fileID []byte) error { 91 | debug.Printf("setting %s=%s on %s", xattrUUID, fileID, p) 92 | return a.mgr.set(p, fileID) 93 | } 94 | 95 | // Get gets the fileid attribute for a file 96 | func (a Attribute) Get(path string) ([]byte, error) { 97 | val, err := a.mgr.get(path) 98 | if err != nil { 99 | debug.Printf("Error reading attribute: %v (%s) will retry", err, a.mgr) 100 | // WTF, let's try again 101 | //time.Sleep(1 * time.Second) 102 | val, err = a.mgr.get(path) 103 | if err != nil { 104 | return nil, errors.Wrap(err, a.String()) 105 | } 106 | } 107 | return val, nil 108 | } 109 | 110 | // GetByFid fetches attribute by root and FID. 111 | func (a Attribute) GetByFid(mnt fs.RootDir, fid *lustre.Fid) ([]byte, error) { 112 | p := fs.FidPath(mnt, fid) 113 | return a.Get(p) 114 | } 115 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/fileid/testing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package fileid 6 | 7 | import "fmt" 8 | 9 | type ( 10 | fileMap map[string][]byte 11 | 12 | testManager struct { 13 | files fileMap 14 | } 15 | ) 16 | 17 | func (m *testManager) update(p string, fileID []byte) error { 18 | return m.set(p, fileID) 19 | } 20 | 21 | func (m *testManager) set(p string, fileID []byte) error { 22 | m.files[p] = fileID 23 | 24 | return nil 25 | } 26 | 27 | func (m *testManager) get(p string) ([]byte, error) { 28 | if attr, ok := m.files[p]; ok { 29 | return attr, nil 30 | } 31 | return nil, fmt.Errorf("%s was not found in fileAttr map", p) 32 | } 33 | 34 | // EnableTestMode swaps out the real implementation for a test-friendly 35 | // mock. 36 | func EnableTestMode() { 37 | UUID = Attribute{&testManager{ 38 | files: make(fileMap), 39 | }} 40 | Hash = Attribute{&testManager{ 41 | files: make(fileMap), 42 | }} 43 | URL = Attribute{&testManager{ 44 | files: make(fileMap), 45 | }} 46 | } 47 | 48 | // DisableTestMode re-enables normal operation. 49 | func DisableTestMode() { 50 | defaultAttrs() 51 | } 52 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/mountpoints.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package agent 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | "time" 11 | 12 | "github.com/pkg/errors" 13 | 14 | "golang.org/x/net/context" 15 | "golang.org/x/sys/unix" 16 | 17 | "github.com/intel-hpdd/logging/audit" 18 | "github.com/intel-hpdd/logging/debug" 19 | "github.com/edwardsp/go-lustre/pkg/mntent" 20 | ) 21 | 22 | // UnmountTimeout is the time, in seconds, that an unmount will be retried 23 | // before failing with an error. 24 | const UnmountTimeout = 10 25 | 26 | type ( 27 | mountConfig struct { 28 | Device string 29 | Directory string 30 | Type string 31 | Options clientMountOptions 32 | Flags uintptr 33 | } 34 | ) 35 | 36 | func (mc *mountConfig) String() string { 37 | return fmt.Sprintf("%s %s %s %s (%d)", mc.Device, mc.Directory, mc.Type, mc.Options, mc.Flags) 38 | } 39 | 40 | func mountClient(cfg *mountConfig) error { 41 | if err := os.MkdirAll(cfg.Directory, 0700); err != nil { 42 | return errors.Wrap(err, "mkdir failed") 43 | } 44 | 45 | return unix.Mount(cfg.Device, cfg.Directory, cfg.Type, cfg.Flags, cfg.Options.String()) 46 | } 47 | 48 | func createMountConfigs(cfg *Config) []*mountConfig { 49 | device := cfg.ClientDevice.String() 50 | // this is what mount_lustre.c does... 51 | opts := append(cfg.ClientMountOptions, "device="+device) 52 | 53 | var flags uintptr 54 | // LU-1783 -- force strictatime until a kernel vfs bug is fixed 55 | flags |= unix.MS_STRICTATIME 56 | 57 | // Create the agent mountpoint first, then add per-plugin mountpoints 58 | configs := []*mountConfig{ 59 | &mountConfig{ 60 | Device: device, 61 | Directory: cfg.AgentMountpoint(), 62 | Type: "lustre", 63 | Options: opts, 64 | Flags: flags, 65 | }, 66 | } 67 | 68 | for _, plugin := range cfg.Plugins() { 69 | configs = append(configs, &mountConfig{ 70 | Device: device, 71 | Directory: plugin.ClientMount, 72 | Type: "lustre", 73 | Options: opts, 74 | Flags: flags, 75 | }) 76 | } 77 | 78 | return configs 79 | } 80 | 81 | // ConfigureMounts configures a set of Lustre client mounts; one for the agent 82 | // and one for each configure data mover. 83 | func ConfigureMounts(cfg *Config) error { 84 | entries, err := mntent.GetMounted() 85 | if err != nil { 86 | return errors.Wrap(err, "failed to get list of mounted filesystems") 87 | } 88 | 89 | for _, mc := range createMountConfigs(cfg) { 90 | if _, err := entries.ByDir(mc.Directory); err == nil { 91 | continue 92 | } 93 | 94 | debug.Printf("Mounting client at %s", mc.Directory) 95 | if err := mountClient(mc); err != nil { 96 | return errors.Wrap(err, "mount client failed") 97 | } 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func doTimedUnmount(dir string) error { 104 | done := make(chan struct{}) 105 | lastError := make(chan error) 106 | 107 | // This feels a little baroque, but it accomplishes two goals: 108 | // 1) Don't leak this goroutine if we time out 109 | // 2) Make sure that we safely get an error from unix.Unmount 110 | // back out to the caller 111 | ctx, cancel := context.WithCancel(context.Background()) 112 | go func(ctx context.Context) { 113 | var err error 114 | for { 115 | select { 116 | case <-ctx.Done(): 117 | lastError <- err 118 | return 119 | default: 120 | err = unix.Unmount(dir, 0) 121 | if err == nil { 122 | close(done) 123 | lastError <- err 124 | return 125 | } 126 | audit.Logf("Waiting for %s to be unmounted", dir) 127 | time.Sleep(1 * time.Second) 128 | } 129 | } 130 | }(ctx) 131 | 132 | for { 133 | select { 134 | case <-done: 135 | return <-lastError 136 | case <-time.After(time.Duration(UnmountTimeout) * time.Second): 137 | cancel() 138 | return errors.Wrapf(<-lastError, "Unmount of %s timed out after %d seconds", dir, UnmountTimeout) 139 | } 140 | } 141 | } 142 | 143 | // CleanupMounts unmounts the Lustre client mounts configured by 144 | // ConfigureMounts(). 145 | func CleanupMounts(cfg *Config) error { 146 | entries, err := mntent.GetMounted() 147 | if err != nil { 148 | return errors.Wrap(err, "failed to get list of mounted filesystems") 149 | } 150 | 151 | // Reverse the generated slice to perform mover unmounts first, 152 | // finishing with the agent unmount. 153 | mcList := createMountConfigs(cfg) 154 | revList := make([]*mountConfig, len(mcList)) 155 | for i := range mcList { 156 | revList[i] = mcList[len(mcList)-1-i] 157 | } 158 | 159 | for _, mc := range revList { 160 | if _, err := entries.ByDir(mc.Directory); err != nil { 161 | continue 162 | } 163 | 164 | debug.Printf("Cleaning up %s", mc.Directory) 165 | if err := doTimedUnmount(mc.Directory); err != nil { 166 | return errors.Wrapf(err, "Failed to unmount %s", mc.Directory) 167 | } 168 | } 169 | 170 | return nil 171 | } 172 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/mountpoints_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package agent 6 | 7 | import ( 8 | "reflect" 9 | "testing" 10 | 11 | "golang.org/x/sys/unix" 12 | 13 | "github.com/edwardsp/lemur/cmd/lhsmd/config" 14 | "github.com/edwardsp/go-lustre/fs/spec" 15 | ) 16 | 17 | func TestMountConfigs(t *testing.T) { 18 | cfg, err := LoadConfig("./test-fixtures/plugin-config") 19 | if err != nil { 20 | t.Fatalf("err: %s", err) 21 | } 22 | 23 | d, err := spec.ClientDeviceFromString("0@lo:/test") 24 | if err != nil { 25 | t.Fatalf("err: %s", err) 26 | } 27 | expectedDevice := d.String() 28 | expectedOptions := clientMountOptions{"user_xattr", "device=" + expectedDevice} 29 | var expectedFlags uintptr 30 | expectedFlags |= unix.MS_STRICTATIME 31 | expected := []*mountConfig{ 32 | { 33 | Device: expectedDevice, 34 | Directory: config.DefaultAgentMountRoot + "/agent", 35 | Type: "lustre", 36 | Options: expectedOptions, 37 | Flags: expectedFlags, 38 | }, 39 | { 40 | Device: expectedDevice, 41 | Directory: config.DefaultAgentMountRoot + "/lhsm-plugin-posix", 42 | Type: "lustre", 43 | Options: expectedOptions, 44 | Flags: expectedFlags, 45 | }, 46 | { 47 | Device: expectedDevice, 48 | Directory: config.DefaultAgentMountRoot + "/lhsm-plugin-s3", 49 | Type: "lustre", 50 | Options: expectedOptions, 51 | Flags: expectedFlags, 52 | }, 53 | { 54 | Device: expectedDevice, 55 | Directory: config.DefaultAgentMountRoot + "/lhsm-plugin-noop", 56 | Type: "lustre", 57 | Options: expectedOptions, 58 | Flags: expectedFlags, 59 | }, 60 | } 61 | 62 | got := createMountConfigs(cfg) 63 | 64 | if !reflect.DeepEqual(got, expected) { 65 | t.Fatalf("\nexpected:\n%s\ngot:\n%s\n", expected, got) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/plugin.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package agent 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | "os" 11 | "os/exec" 12 | "path" 13 | "time" 14 | 15 | "github.com/pkg/errors" 16 | 17 | "golang.org/x/net/context" 18 | 19 | "github.com/edwardsp/lemur/cmd/lhsmd/config" 20 | "github.com/intel-hpdd/logging/alert" 21 | "github.com/intel-hpdd/logging/audit" 22 | "github.com/intel-hpdd/logging/debug" 23 | ) 24 | 25 | var backoff = []time.Duration{ 26 | 0 * time.Second, 27 | 1 * time.Second, 28 | 10 * time.Second, 29 | 30 * time.Second, 30 | 1 * time.Minute, 31 | } 32 | var maxBackoff = len(backoff) - 1 33 | 34 | type ( 35 | // PluginConfig represents configuration for a single plugin 36 | PluginConfig struct { 37 | Name string 38 | BinPath string 39 | AgentConnection string 40 | ClientMount string 41 | Args []string 42 | RestartOnFailure bool 43 | 44 | lastRestart time.Time 45 | restartCount int 46 | } 47 | 48 | // PluginMonitor watches monitored plugins and restarts 49 | // them as needed. 50 | PluginMonitor struct { 51 | processChan ppChan 52 | processStateChan psChan 53 | } 54 | 55 | pluginProcess struct { 56 | plugin *PluginConfig 57 | cmd *exec.Cmd 58 | } 59 | 60 | pluginStatus struct { 61 | ps *os.ProcessState 62 | err error 63 | } 64 | 65 | ppChan chan *pluginProcess 66 | psChan chan *pluginStatus 67 | ) 68 | 69 | func (p *PluginConfig) String() string { 70 | data, err := json.Marshal(p) 71 | if err != nil { 72 | alert.Abort(errors.Wrap(err, "marshal failed")) 73 | } 74 | 75 | var out bytes.Buffer 76 | json.Indent(&out, data, "", "\t") 77 | return out.String() 78 | } 79 | 80 | // NoRestart optionally sets a plugin to not be restarted on failure 81 | func (p *PluginConfig) NoRestart() *PluginConfig { 82 | p.RestartOnFailure = false 83 | return p 84 | } 85 | 86 | // RestartDelay returns a time.Duration to delay restarts based on 87 | // the number of restarts and the last restart time. 88 | func (p *PluginConfig) RestartDelay() time.Duration { 89 | // If it's been a decent amount of time since the last restart, 90 | // reset the backoff mechanism for a quick restart. 91 | if time.Since(p.lastRestart) > backoff[maxBackoff]*2 { 92 | p.restartCount = 0 93 | } 94 | 95 | if p.restartCount > maxBackoff { 96 | return backoff[maxBackoff] 97 | } 98 | return backoff[p.restartCount] 99 | } 100 | 101 | // NewPlugin returns a plugin configuration 102 | func NewPlugin(name, binPath, conn, mountRoot string, args ...string) *PluginConfig { 103 | return &PluginConfig{ 104 | Name: name, 105 | BinPath: binPath, 106 | AgentConnection: conn, 107 | ClientMount: path.Join(mountRoot, name), 108 | Args: args, 109 | RestartOnFailure: true, 110 | } 111 | } 112 | 113 | // NewMonitor creates a new plugin monitor 114 | func NewMonitor() *PluginMonitor { 115 | return &PluginMonitor{ 116 | processChan: make(ppChan), 117 | processStateChan: make(psChan), 118 | } 119 | } 120 | 121 | func (m *PluginMonitor) run(ctx context.Context) { 122 | processMap := make(map[int]*PluginConfig) 123 | 124 | var waitForCmd = func(cmd *exec.Cmd) { 125 | debug.Printf("Waiting for %s (%d) to exit", cmd.Path, cmd.Process.Pid) 126 | ps, err := cmd.Process.Wait() 127 | if err != nil { 128 | audit.Logf("Err after Wait() for %d: %s", cmd.Process.Pid, err) 129 | } 130 | 131 | debug.Printf("PID %d finished: %s", cmd.Process.Pid, ps) 132 | m.processStateChan <- &pluginStatus{ps, err} 133 | } 134 | 135 | for { 136 | select { 137 | case p := <-m.processChan: 138 | processMap[p.cmd.Process.Pid] = p.plugin 139 | go waitForCmd(p.cmd) 140 | case s := <-m.processStateChan: 141 | cfg, found := processMap[s.ps.Pid()] 142 | if !found { 143 | debug.Printf("Received disp of unknown pid: %d", s.ps.Pid()) 144 | break 145 | } 146 | 147 | delete(processMap, s.ps.Pid()) 148 | audit.Logf("Process %d for %s died: %s", s.ps.Pid(), cfg.Name, s.ps) 149 | if cfg.RestartOnFailure { 150 | delay := cfg.RestartDelay() 151 | audit.Logf("Restarting plugin %s after delay of %s (attempt %d)", cfg.Name, delay, cfg.restartCount) 152 | 153 | cfg.restartCount++ 154 | cfg.lastRestart = time.Now() 155 | // Restart in a different goroutine to 156 | // avoid deadlocking this one. 157 | go func(cfg *PluginConfig, delay time.Duration) { 158 | <-time.After(delay) 159 | 160 | err := m.StartPlugin(cfg) 161 | if err != nil { 162 | audit.Logf("Failed to restart plugin %s: %s", cfg.Name, err) 163 | } 164 | }(cfg, delay) 165 | } 166 | case <-ctx.Done(): 167 | return 168 | } 169 | } 170 | } 171 | 172 | // Start creates a new plugin monitor 173 | func (m *PluginMonitor) Start(ctx context.Context) { 174 | go m.run(ctx) 175 | } 176 | 177 | // StartPlugin starts the plugin and monitors it 178 | func (m *PluginMonitor) StartPlugin(cfg *PluginConfig) error { 179 | debug.Printf("Starting %s for %s", cfg.BinPath, cfg.Name) 180 | 181 | cmd := exec.Command(cfg.BinPath, cfg.Args...) // #nosec 182 | 183 | prefix := path.Base(cfg.BinPath) 184 | cmd.Stdout = audit.Writer().Prefix(prefix + " ") 185 | cmd.Stderr = audit.Writer().Prefix(prefix + "-stderr ") 186 | 187 | cmd.Env = append(os.Environ(), config.AgentConnEnvVar+"="+cfg.AgentConnection) 188 | cmd.Env = append(cmd.Env, config.PluginMountpointEnvVar+"="+cfg.ClientMount) 189 | 190 | if err := cmd.Start(); err != nil { 191 | return errors.Wrapf(err, "cmd failed %q", cmd) 192 | } 193 | 194 | audit.Logf("Started %s (PID: %d)", cmd.Path, cmd.Process.Pid) 195 | m.processChan <- &pluginProcess{cfg, cmd} 196 | 197 | return nil 198 | } 199 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package agent 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | "path" 11 | "time" 12 | 13 | "github.com/pkg/errors" 14 | 15 | "github.com/edwardsp/go-lustre" 16 | "github.com/edwardsp/go-lustre/fs" 17 | "github.com/edwardsp/go-lustre/hsm" 18 | "github.com/edwardsp/go-lustre/llapi" 19 | "github.com/edwardsp/go-lustre/status" 20 | "github.com/edwardsp/lemur/cmd/lhsmd/agent/fileid" 21 | "github.com/intel-hpdd/logging/alert" 22 | "github.com/intel-hpdd/logging/debug" 23 | ) 24 | 25 | func createSnapDir(p string) (string, error) { 26 | fi, err := os.Lstat(p) 27 | if err != nil { 28 | return "", errors.Wrap(err, "lstat failed") 29 | } 30 | snapDir := path.Join(p, ".hsmsnap") 31 | err = os.MkdirAll(snapDir, fi.Mode()) 32 | if err != nil { 33 | return "", errors.Wrap(err, "mkdir all failed") 34 | } 35 | return snapDir, nil 36 | } 37 | 38 | func createStubFile(f string, fi os.FileInfo, archive uint, layout *llapi.DataLayout) error { 39 | _, err := hsm.Import(f, archive, fi, layout) 40 | if err != nil { 41 | os.Remove(f) 42 | return errors.Wrapf(err, "%s: import failed", f) 43 | } 44 | return nil 45 | } 46 | 47 | func snapName(fi os.FileInfo) string { 48 | return fmt.Sprintf("%s^%s", fi.Name(), fi.ModTime().Format(time.RFC3339)) 49 | } 50 | 51 | func createSnapshots(mnt fs.RootDir, archive uint, fileID []byte, names []string) error { 52 | var firstPath string 53 | first := true 54 | for _, p := range names { 55 | absPath := mnt.Join(p) 56 | snapDir, err := createSnapDir(path.Dir(absPath)) 57 | if err != nil { 58 | return errors.Wrap(err, "create snapdir failed") 59 | } 60 | fi, err := os.Lstat(absPath) 61 | if err != nil { 62 | return errors.Wrap(err, "lstat failed") 63 | } 64 | f := path.Join(snapDir, snapName(fi)) 65 | if first { 66 | var layout *llapi.DataLayout 67 | layout, err = llapi.FileDataLayout(absPath) 68 | if err != nil { 69 | alert.Warnf("%s: unable to get layout: %v", f, err) 70 | return errors.Wrap(err, "get layout") 71 | } 72 | debug.Printf("%s: layout: %#v", absPath, layout) 73 | err = createStubFile(f, fi, archive, layout) 74 | if err != nil { 75 | return errors.Wrap(err, "create stub file") 76 | } 77 | err = fileid.UUID.Set(f, fileID) 78 | if err != nil { 79 | return errors.Wrapf(err, "%s: set fileid", f) 80 | } 81 | firstPath = f 82 | first = false 83 | } else { 84 | err = os.Link(firstPath, f) 85 | if err != nil { 86 | return errors.Wrapf(err, "%s: link to %s failed", f, firstPath) 87 | } 88 | } 89 | 90 | } 91 | return nil 92 | } 93 | 94 | func createSnapshot(mnt fs.RootDir, archive uint, fid *lustre.Fid, fileID []byte) error { 95 | names, err := status.FidPathnames(mnt, fid) 96 | if err != nil { 97 | return errors.Wrapf(err, "%s: fidpathname failed", fid) 98 | } 99 | 100 | return createSnapshots(mnt, archive, fileID, names) 101 | } 102 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/test-fixtures/good-config: -------------------------------------------------------------------------------- 1 | mount_root = "/mnt/lhsmd" 2 | agent_mountpoint = "/mnt/lhsmd/agent" 3 | client_device = "10.211.55.37@tcp:/testFs" 4 | client_mount_options = ["user_xattr"] 5 | plugin_dir = "/go/bin" 6 | 7 | influxdb { 8 | url = "http://172.17.0.4:8086" 9 | db = "lhsmd" 10 | } 11 | 12 | snapshots { 13 | enabled = false 14 | } 15 | 16 | transport { 17 | socket_dir = "/tmp" 18 | } 19 | 20 | enabled_plugins = ["lhsm-plugin-posix"] 21 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/test-fixtures/json-config: -------------------------------------------------------------------------------- 1 | { 2 | "MountRoot": "", 3 | "AgentMountpoint": "", 4 | "client_device": "10.211.55.37@tcp0:/testFs", 5 | "ClientMountOptions": null, 6 | "Processes": 0, 7 | "InfluxDB": { 8 | "URL": "", 9 | "DB": "", 10 | "User": "", 11 | "Password": "" 12 | }, 13 | "EnabledPlugins": null, 14 | "PluginDir": "", 15 | "Snapshots": { 16 | "Enabled": false 17 | }, 18 | "Transport": { 19 | "Type": "", 20 | "SocketDir": "/tmp", 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/test-fixtures/merge-config: -------------------------------------------------------------------------------- 1 | mount_root = "/mnt/lhsmd" 2 | agent_mountpoint = "/mnt/lhsmd/agent" 3 | client_device = "10.211.55.37@tcp:/testFs" 4 | client_mount_options = ["user_xattr"] 5 | plugin_dir = "/go/bin" 6 | 7 | handler_count = 2 8 | 9 | influxdb { 10 | url = "http://172.17.0.4:8086" 11 | db = "lhsmd" 12 | } 13 | 14 | snapshots { 15 | enabled = false 16 | } 17 | 18 | enabled_plugins = ["lhsm-plugin-posix"] 19 | 20 | transport { 21 | type = "grpc" 22 | port = 9000 23 | } 24 | -------------------------------------------------------------------------------- /cmd/lhsmd/agent/test-fixtures/plugin-config: -------------------------------------------------------------------------------- 1 | client_device = "0@lo:/test" 2 | 3 | enabled_plugins = ["lhsm-plugin-posix", "lhsm-plugin-s3", "lhsm-plugin-noop"] 4 | -------------------------------------------------------------------------------- /cmd/lhsmd/config/default.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package config 6 | 7 | const ( 8 | // DefaultConfigDir is the default agent config directory 9 | DefaultConfigDir = "/etc/lhsmd" 10 | // AgentConfigFile is the agent config file in config dir 11 | AgentConfigFile = "agent" 12 | // DefaultConfigPath is the default path to the agent config file 13 | DefaultConfigPath = DefaultConfigDir + "/" + AgentConfigFile 14 | 15 | // ConfigDirEnvVar is the name of an environment variable which 16 | // can be set to change the location of config files 17 | // (e.g. for development) 18 | ConfigDirEnvVar = "LHSMD_CONFIG_DIR" 19 | 20 | // AgentConnEnvVar is the environment variable containing a connect 21 | // string for plugins to use when registering with the agent 22 | AgentConnEnvVar = "LHSMD_AGENT_CONNECTION" 23 | 24 | // PluginMountpointEnvVar is the environment variable containing 25 | // a Lustre client mountpoint to be used by the plugin 26 | PluginMountpointEnvVar = "LHSMD_CLIENT_MOUNTPOINT" 27 | 28 | // DefaultTransport is the default agent<->plugin transport 29 | DefaultTransport = "grpc" 30 | 31 | // DefaultTransportSocketDir is default directory to store the unix socket 32 | DefaultTransportSocketDir = "/var/run/lhsmd" 33 | 34 | // DefaultAgentMountRoot is the root directory for agent client mounts 35 | DefaultAgentMountRoot = "/mnt/lhsmd" 36 | 37 | // DefaultPluginDir is the default location for plugin binaries 38 | DefaultPluginDir = "/usr/libexec/lhsmd" 39 | ) 40 | 41 | // DefaultClientMountOptions is the default set of Lustre client 42 | // mount options 43 | var DefaultClientMountOptions = []string{"user_xattr"} 44 | -------------------------------------------------------------------------------- /cmd/lhsmd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "log" 10 | "os" 11 | "os/signal" 12 | "syscall" 13 | "time" 14 | 15 | "github.com/intel-hpdd/go-metrics-influxdb" 16 | "github.com/pkg/errors" 17 | "github.com/rcrowley/go-metrics" 18 | 19 | "golang.org/x/net/context" 20 | 21 | "github.com/edwardsp/lemur/cmd/lhsmd/agent" 22 | "github.com/edwardsp/lemur/pkg/fsroot" 23 | "github.com/intel-hpdd/logging/alert" 24 | "github.com/intel-hpdd/logging/audit" 25 | "github.com/intel-hpdd/logging/debug" 26 | "github.com/edwardsp/go-lustre/hsm" 27 | 28 | // Register the supported transports 29 | _ "github.com/edwardsp/lemur/cmd/lhsmd/transport/grpc" 30 | ) 31 | 32 | func init() { 33 | flag.Var(debug.FlagVar()) 34 | } 35 | 36 | func interruptHandler(once func()) { 37 | c := make(chan os.Signal, 1) 38 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 39 | 40 | go func() { 41 | stopping := false 42 | for sig := range c { 43 | debug.Printf("signal received: %s", sig) 44 | if !stopping { 45 | stopping = true 46 | once() 47 | } 48 | } 49 | }() 50 | 51 | } 52 | 53 | func run(conf *agent.Config) error { 54 | debug.Printf("current configuration:\n%v", conf.String()) 55 | if err := agent.ConfigureMounts(conf); err != nil { 56 | return errors.Wrap(err, "Error while creating Lustre mountpoints") 57 | } 58 | 59 | if conf.InfluxDB != nil && conf.InfluxDB.URL != "" { 60 | debug.Print("Configuring InfluxDB stats target") 61 | go influxdb.InfluxDB( 62 | metrics.DefaultRegistry, // metrics registry 63 | time.Second*10, // interval 64 | conf.InfluxDB.URL, 65 | conf.InfluxDB.DB, // your InfluxDB database 66 | conf.InfluxDB.User, // your InfluxDB user 67 | conf.InfluxDB.Password, // your InfluxDB password 68 | ) 69 | } 70 | 71 | client, err := fsroot.New(conf.AgentMountpoint()) 72 | if err != nil { 73 | return errors.Wrap(err, "Could not get fs client") 74 | } 75 | as := hsm.NewActionSource(client.Root()) 76 | 77 | ct, err := agent.New(conf, client, as) 78 | if err != nil { 79 | return errors.Wrap(err, "Error creating agent") 80 | } 81 | 82 | interruptHandler(func() { 83 | ct.Stop() 84 | }) 85 | 86 | return errors.Wrap(ct.Start(context.Background()), 87 | "Error in HsmAgent.Start()") 88 | } 89 | 90 | func main() { 91 | flag.Parse() 92 | 93 | if debug.Enabled() { 94 | // Set this so that plugins can use it without needing 95 | // to mess around with plugin args. 96 | os.Setenv(debug.EnableEnvVar, "true") 97 | } 98 | 99 | // Setting the prefix helps us to track down deprecated calls to log.* 100 | log.SetFlags(log.LstdFlags | log.Lshortfile) 101 | log.SetOutput(audit.Writer().Prefix("DEPRECATED ")) 102 | 103 | conf := agent.ConfigInitMust() 104 | err := run(conf) 105 | 106 | // Ensure that we always clean up. 107 | if err := agent.CleanupMounts(conf); err != nil { 108 | alert.Warn(errors.Wrap(err, "Error while cleaning up Lustre mountpoints")) 109 | } 110 | 111 | if err != nil { 112 | alert.Abort(err) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /cmd/lhsmd/profile.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build profile 6 | 7 | package main 8 | 9 | import ( 10 | "log" 11 | "net/http" 12 | _ "net/http/pprof" 13 | ) 14 | 15 | func init() { 16 | go func() { 17 | log.Println(http.ListenAndServe("localhost:6060", nil)) 18 | }() 19 | 20 | } 21 | -------------------------------------------------------------------------------- /cmd/lhsmd/transport/grpc/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package rpc 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "runtime" 11 | "time" 12 | 13 | "github.com/dustin/go-humanize" 14 | "github.com/rcrowley/go-metrics" 15 | ) 16 | 17 | type messageStats struct { 18 | StartTime time.Time 19 | StartSysMem uint64 20 | MaxSysMem uint64 21 | Count metrics.Counter 22 | Rate metrics.Meter 23 | Latencies metrics.Histogram 24 | } 25 | 26 | func (s *messageStats) String() string { 27 | var buf bytes.Buffer 28 | 29 | ms := runtime.MemStats{} 30 | runtime.ReadMemStats(&ms) 31 | if ms.Sys > s.MaxSysMem { 32 | s.MaxSysMem = ms.Sys 33 | } 34 | 35 | fmt.Fprintf(&buf, "mem usage (start/cur/max): %s/%s/%s\n", 36 | humanize.Bytes(s.StartSysMem), 37 | humanize.Bytes(ms.Sys), 38 | humanize.Bytes(s.MaxSysMem), 39 | ) 40 | fmt.Fprintf(&buf, "runtime: %s\n", time.Since(s.StartTime)) 41 | fmt.Fprintf(&buf, " count: %s\n", humanize.Comma(s.Count.Count())) 42 | fmt.Fprintf(&buf, "msg/sec (1 min/5 min/15 min/inst): %s/%s/%s/%s\n", 43 | humanize.Comma(int64(s.Rate.Rate1())), 44 | humanize.Comma(int64(s.Rate.Rate5())), 45 | humanize.Comma(int64(s.Rate.Rate15())), 46 | humanize.Comma(int64(s.Rate.RateMean())), 47 | ) 48 | fmt.Fprintln(&buf, "latencies:") 49 | fmt.Fprintf(&buf, " min: %s\n", time.Duration(s.Latencies.Min())) 50 | fmt.Fprintf(&buf, " mean: %s\n", time.Duration(int64(s.Latencies.Mean()))) 51 | fmt.Fprintf(&buf, " max: %s\n", time.Duration(s.Latencies.Max())) 52 | 53 | return buf.String() 54 | } 55 | 56 | func newMessageStats() *messageStats { 57 | ms := runtime.MemStats{} 58 | runtime.ReadMemStats(&ms) 59 | 60 | return &messageStats{ 61 | StartTime: time.Now(), 62 | StartSysMem: ms.Sys, 63 | MaxSysMem: ms.Sys, 64 | Count: metrics.NewCounter(), 65 | Rate: metrics.NewMeter(), 66 | Latencies: metrics.NewHistogram( 67 | metrics.NewUniformSample(1024), 68 | ), 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /cmd/util/logger.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2020 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package util 22 | 23 | import ( 24 | "fmt" 25 | "log" 26 | "log/syslog" 27 | "runtime" 28 | "time" 29 | 30 | "github.com/Azure/azure-pipeline-go/pipeline" 31 | ) 32 | 33 | type ILogger interface { 34 | OpenLog() 35 | CloseLog() 36 | Log(level pipeline.LogLevel, msg string) 37 | Panic(err error) 38 | } 39 | 40 | //Default logger instance 41 | var globalLogger jobLogger 42 | 43 | //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// 44 | 45 | type jobLogger struct { 46 | // maximum loglevel represents the maximum severity of log messages which can be logged to Job Log file. 47 | // any message with severity higher than this will be ignored. 48 | minimumLevelToLog pipeline.LogLevel // The maximum customer-desired log level for this job 49 | logger *log.Logger // The Job's logger 50 | sanitizer pipeline.LogSanitizer 51 | } 52 | 53 | func InitJobLogger(minimumLevelToLog pipeline.LogLevel) { 54 | 55 | globalLogger = jobLogger{ 56 | minimumLevelToLog: minimumLevelToLog, 57 | sanitizer: &azCopyLogSanitizer{}, 58 | } 59 | 60 | globalLogger.OpenLog() 61 | } 62 | 63 | func (jl *jobLogger) OpenLog() { 64 | if jl.minimumLevelToLog == pipeline.LogNone { 65 | return 66 | } 67 | 68 | logwriter, err := syslog.New(syslog.LOG_NOTICE, "lhsm-plugin-az") 69 | if err != nil { 70 | panic(err) 71 | } 72 | 73 | log.SetOutput(logwriter) 74 | flags := log.LstdFlags | log.LUTC 75 | utcMessage := fmt.Sprintf("Log times are in UTC. Local time is " + time.Now().Format("2 Jan 2006 15:04:05")) 76 | 77 | jl.logger = log.New(logwriter, "", flags) 78 | // Log the OS Environment and OS Architecture 79 | jl.logger.Println("OS-Environment ", runtime.GOOS) 80 | jl.logger.Println("OS-Architecture ", runtime.GOARCH) 81 | jl.logger.Println(utcMessage) 82 | } 83 | 84 | func (jl *jobLogger) CloseLog() { 85 | return 86 | } 87 | 88 | func (jl jobLogger) Log(loglevel pipeline.LogLevel, msg string) { 89 | // ensure all secrets are redacted 90 | msg = jl.sanitizer.SanitizeLogMessage(msg) 91 | jl.logger.Println(msg) 92 | } 93 | 94 | func Log(logLevel pipeline.LogLevel, msg string) { 95 | globalLogger.Log(logLevel, msg) 96 | } 97 | 98 | const tryEquals string = "Try=" // TODO: refactor so that this can be used by the retry policies too? So that when you search the logs for Try= you are guaranteed to find both types of retry (i.e. request send retries, and body read retries) 99 | 100 | func NewReadLogFunc(redactedURL string) func(int, error, int64, int64, bool) { 101 | 102 | return func(failureCount int, err error, offset int64, count int64, willRetry bool) { 103 | retryMessage := "Will retry" 104 | if !willRetry { 105 | retryMessage = "Will NOT retry" 106 | } 107 | globalLogger.Log(pipeline.LogInfo, fmt.Sprintf( 108 | "Error reading body of reply. Next try (if any) will be %s%d. %s. Error: %s. Offset: %d Count: %d URL: %s", 109 | tryEquals, // so that retry wording for body-read retries is similar to that for URL-hitting retries 110 | 111 | // We log the number of the NEXT try, not the failure just done, so that users searching the log for "Try=2" 112 | // will find ALL retries, both the request send retries (which are logged as try 2 when they are made) and 113 | // body read retries (for which only the failure is logged - so if we did the actual failure number, there would be 114 | // not Try=2 in the logs if the retries work). 115 | failureCount+1, 116 | 117 | retryMessage, 118 | err, 119 | offset, 120 | count, 121 | redactedURL)) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /cmd/util/misc.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2020 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package util 22 | 23 | import ( 24 | "context" 25 | "net/http" 26 | "time" 27 | 28 | "github.com/Azure/azure-pipeline-go/pipeline" 29 | kvauth "github.com/Azure/azure-sdk-for-go/services/keyvault/auth" 30 | "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault" 31 | "github.com/Azure/azure-storage-blob-go/azblob" 32 | ) 33 | 34 | //HTTPClientFactory returns http sender with given client 35 | func HTTPClientFactory(client *http.Client) pipeline.FactoryFunc { 36 | return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { 37 | return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 38 | r, err := client.Do(request.WithContext(ctx)) 39 | if err != nil { 40 | err = pipeline.NewError(err, "HTTP request failed") 41 | } 42 | return pipeline.NewHTTPResponse(r), err 43 | } 44 | }) 45 | } 46 | 47 | // NewPipeline creates a blobpipeline with these options 48 | func NewPipeline(ctx context.Context, c azblob.Credential, p Pacer, o azblob.PipelineOptions) pipeline.Pipeline { 49 | const tryTimeout = time.Minute * 15 50 | const retryDelay = time.Second * 1 51 | const maxRetryDelay = time.Second * 6 52 | const maxTries = 20 53 | 54 | r := azblob.RetryOptions{ 55 | Policy: 0, 56 | MaxTries: 20, 57 | TryTimeout: tryTimeout, 58 | RetryDelay: retryDelay, 59 | MaxRetryDelay: maxRetryDelay, 60 | } 61 | // Closest to API goes first; closest to the wire goes last 62 | var f []pipeline.Factory 63 | 64 | if p != nil { 65 | f = append(f, NewRateLimiterPolicy(ctx, p)) 66 | } 67 | f = append(f, 68 | azblob.NewTelemetryPolicyFactory(o.Telemetry), 69 | azblob.NewUniqueRequestIDPolicyFactory(), 70 | azblob.NewRetryPolicyFactory(r), 71 | c, 72 | azblob.NewRequestLogPolicyFactory(o.RequestLog), 73 | pipeline.MethodFactoryMarker()) // indicates at what stage in the pipeline the method factory is invoked 74 | 75 | return pipeline.NewPipeline(f, pipeline.Options{HTTPSender: o.HTTPSender, Log: o.Log}) 76 | } 77 | 78 | //GetKVSecret returns string secret by name 'kvSecretName' in keyvault 'kvName' 79 | //Uses MSI auth to login 80 | func GetKVSecret(kvName, kvSuffix, kvSecretName string) (secret string, err error) { 81 | authorizer, err := kvauth.NewAuthorizerFromEnvironment() 82 | if err != nil { 83 | return "", err 84 | } 85 | 86 | basicClient := keyvault.New() 87 | basicClient.Authorizer = authorizer 88 | 89 | ctx, _ := context.WithTimeout(context.Background(), 3*time.Minute) 90 | secretResp, err := basicClient.GetSecret(ctx, "https://"+kvName+"."+kvSuffix, kvSecretName, "") 91 | if err != nil { 92 | return "", err 93 | } 94 | 95 | return *secretResp.Value, nil 96 | } 97 | 98 | func GetBlockSize(filesize int64, minBlockSize int64) (blockSize int64) { 99 | blockSizeThreshold := int64(256 * 1024 * 1024) /* 256 MB */ 100 | blockSize = minBlockSize 101 | 102 | /* We should not perform checks on filesize, block size limitation here. Those are performed in SDK 103 | * and take care of themselves when limits change 104 | */ 105 | 106 | for ; uint32(filesize/blockSize) > azblob.BlockBlobMaxBlocks; blockSize = 2 * blockSize { 107 | if blockSize > blockSizeThreshold { 108 | /* 109 | * For a RAM usage of 0.5G/core, we would have 4G memory on typical 8 core device, meaning at a blockSize of 256M, 110 | * we can have 4 blocks in core, waiting for a disk or n/w operation. Any higher block size would *sort of* 111 | * serialize n/w and disk operations, and is better avoided. 112 | */ 113 | blockSize = filesize / azblob.BlockBlobMaxBlocks 114 | break 115 | } 116 | } 117 | 118 | return blockSize 119 | } 120 | -------------------------------------------------------------------------------- /cmd/util/rateLimiterPolicy.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2020 Microsoft 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | package util 22 | 23 | import ( 24 | "context" 25 | "errors" 26 | "fmt" 27 | "time" 28 | 29 | "github.com/Azure/azure-pipeline-go/pipeline" 30 | ) 31 | 32 | func logThroughput(ctx context.Context, p Pacer) { 33 | interval := 4 * time.Second 34 | intervalStartTime := time.Now() 35 | prevBytesTransferred := p.GetTotalTraffic() 36 | 37 | for { 38 | select { 39 | case <-ctx.Done(): 40 | return 41 | case <-time.After(interval): 42 | bytesOnWireMb := float64(float64(p.GetTotalTraffic()-prevBytesTransferred) / (1000 * 1000)) 43 | timeElapsed := time.Since(intervalStartTime).Seconds() 44 | if timeElapsed != 0 { 45 | throughput := bytesOnWireMb / float64(timeElapsed) 46 | Log(pipeline.LogInfo, fmt.Sprintf("4-sec throughput: %v MBPS", throughput)) 47 | } 48 | // reset the interval timer and byte count 49 | intervalStartTime = time.Now() 50 | prevBytesTransferred = p.GetTotalTraffic() 51 | } 52 | } 53 | } 54 | 55 | type rateLimiterPolicy struct { 56 | next pipeline.Policy 57 | pacer Pacer 58 | } 59 | 60 | func (r *rateLimiterPolicy) Do(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { 61 | err := r.pacer.RequestTrafficAllocation(ctx, request.ContentLength) 62 | if err != nil { 63 | return nil, errors.New("failed to pace request block") 64 | } 65 | 66 | resp, err := r.next.Do(ctx, request) 67 | 68 | err = r.pacer.RequestTrafficAllocation(ctx, resp.Response().ContentLength) 69 | if err != nil { 70 | return nil, errors.New("failed to pace response block") 71 | } 72 | 73 | return resp, err 74 | } 75 | 76 | func NewRateLimiterPolicy(ctx context.Context, pacer Pacer) pipeline.Factory { 77 | go logThroughput(ctx, pacer) 78 | return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { 79 | r := rateLimiterPolicy{next: next, pacer: pacer} 80 | return r.Do 81 | }) 82 | } 83 | -------------------------------------------------------------------------------- /create-protex-scantree.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | EXCLUDES="golang.org/x github.com/mjmac/go-ps github.com/stathat github.com/urfave github.com/DATA-DOG/godog" # stuff that isn't really 3rd-party, or is test-only 4 | SCANDIR=${SCANDIR:-$(mktemp -d)} 5 | 6 | imports=$(go list -f '{{.Deps}}' ./... | \ 7 | tr "[" " " | tr "]" " " | \ 8 | xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}' | \ 9 | sort | uniq) 10 | 11 | for import in $imports; do 12 | dest=$SCANDIR/$(dirname $import) 13 | if [[ $import == *yaml.v2* ]]; then 14 | dest=$SCANDIR/$import 15 | fi 16 | for excl in $EXCLUDES; do 17 | if [[ $import == *$excl* ]]; then 18 | echo "Skipping $import (matches $excl)." 19 | continue 2 20 | fi 21 | done 22 | 23 | mkdir -p $dest 24 | echo -n "Copying $import -> $dest... " 25 | cp -a $GOPATH/src/$import $dest 26 | echo Done. 27 | done 28 | 29 | echo "SCANDIR: $SCANDIR" 30 | -------------------------------------------------------------------------------- /dmplugin/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package dmplugin 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | "io/ioutil" 11 | "os" 12 | 13 | "github.com/hashicorp/hcl" 14 | "github.com/pkg/errors" 15 | "github.com/edwardsp/lemur/cmd/lhsmd/config" 16 | "github.com/intel-hpdd/logging/alert" 17 | ) 18 | 19 | type pluginConfig struct { 20 | AgentAddress string 21 | ClientRoot string 22 | ConfigDir string 23 | } 24 | 25 | // LoadConfig reads this plugin's config file and decodes it into the passed 26 | // config struct. 27 | func LoadConfig(cfgFile string, cfg interface{}) error { 28 | // Ensure config file is private 29 | fi, err := os.Stat(cfgFile) 30 | if err != nil { 31 | return errors.Wrap(err, "stat config file failed") 32 | } 33 | if (int(fi.Mode()) & 077) != 0 { 34 | return errors.Errorf("file permissions on %s are insecure (%#o)", cfgFile, int(fi.Mode())) 35 | } 36 | 37 | data, err := ioutil.ReadFile(cfgFile) 38 | if err != nil { 39 | return errors.Wrap(err, "read config file failed") 40 | } 41 | 42 | if err := hcl.Decode(cfg, string(data)); err != nil { 43 | return errors.Wrap(err, "decode config file failed") 44 | } 45 | 46 | return nil 47 | } 48 | 49 | // DisplayConfig formats the configuration into string for display 50 | // purposes. A helper function to remove some duplicate code in 51 | // movers. 52 | func DisplayConfig(cfg interface{}) string { 53 | data, err := json.Marshal(cfg) 54 | if err != nil { 55 | alert.Abort(errors.Wrap(err, "marshal config failed")) 56 | } 57 | 58 | var out bytes.Buffer 59 | json.Indent(&out, data, "", "\t") 60 | return out.String() 61 | } 62 | 63 | func getAgentEnvSetting(name string) (value string) { 64 | if value = os.Getenv(name); value == "" { 65 | alert.Fatal("This plugin is intended to be launched by the agent.") 66 | } 67 | return 68 | } 69 | 70 | // mustInitConfig looks for the plugin environment variables and 71 | // returns the configuratino. Will fail the process with hlpeful 72 | // message if any of the env variables are not seet. 73 | func mustInitConfig() *pluginConfig { 74 | pc := &pluginConfig{ 75 | AgentAddress: getAgentEnvSetting(config.AgentConnEnvVar), 76 | ClientRoot: getAgentEnvSetting(config.PluginMountpointEnvVar), 77 | ConfigDir: getAgentEnvSetting(config.ConfigDirEnvVar), 78 | } 79 | return pc 80 | } 81 | -------------------------------------------------------------------------------- /dmplugin/dmio/action.go: -------------------------------------------------------------------------------- 1 | package dmio 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "os" 7 | 8 | "github.com/edwardsp/go-lustre" 9 | "github.com/edwardsp/lemur/dmplugin" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // BufferSize is the buffered reader size 14 | var BufferSize = 1024 * 1024 15 | 16 | type ( 17 | statter interface { 18 | Stat() (os.FileInfo, error) 19 | } 20 | 21 | writerWriterAt interface { 22 | io.WriterAt 23 | io.Writer 24 | } 25 | 26 | // ActionReader wraps an io.SectionReader and also implements 27 | // io.Closer by closing the embedded io.Closer. 28 | ActionReader struct { 29 | sr *io.SectionReader 30 | closer io.Closer 31 | } 32 | 33 | // BufferedActionReader wraps a buffered ActionReader and 34 | // also implements io.Closer by closing the embedded io.Closer. 35 | BufferedActionReader struct { 36 | br *bufio.Reader 37 | closer io.Closer 38 | } 39 | 40 | // ActionWriter wraps an io.WriterAt but imposes a base offset 41 | // determined by the action's extent. 42 | ActionWriter struct { 43 | baseOffset int64 44 | wwa writerWriterAt 45 | statter statter 46 | closer io.Closer 47 | } 48 | ) 49 | 50 | // Close calls the embedded io.Closer's Close() 51 | func (ar *ActionReader) Close() error { 52 | return ar.closer.Close() 53 | } 54 | 55 | // Read calls the embedded *io.SectionReader's Read() 56 | func (ar *ActionReader) Read(p []byte) (int, error) { 57 | return ar.sr.Read(p) 58 | } 59 | 60 | // Seek calls the embedded *io.SectionReader's Seek() 61 | func (ar *ActionReader) Seek(offset int64, whence int) (int64, error) { 62 | return ar.sr.Seek(offset, whence) 63 | } 64 | 65 | // Close calls the embedded io.Closer's Close() 66 | func (bar *BufferedActionReader) Close() error { 67 | return bar.closer.Close() 68 | } 69 | 70 | // Read calls the embedded *bufio.Reader's Read() 71 | func (bar *BufferedActionReader) Read(p []byte) (int, error) { 72 | return bar.br.Read(p) 73 | } 74 | 75 | // Close calls the embedded io.Closer's Close() 76 | func (aw *ActionWriter) Close() error { 77 | return aw.closer.Close() 78 | } 79 | 80 | // WriteAt calls the embedded io.WriterAt's WriteAt(), with the 81 | // offset adjusted by the base offset. 82 | func (aw *ActionWriter) WriteAt(p []byte, off int64) (int, error) { 83 | return aw.wwa.WriteAt(p, aw.baseOffset+off) 84 | } 85 | 86 | // Write calls the embedded io.Writer's Write(). 87 | func (aw *ActionWriter) Write(p []byte) (int, error) { 88 | return aw.wwa.Write(p) 89 | } 90 | 91 | // Stat calls the embedded statter's Stat(). 92 | func (aw *ActionWriter) Stat() (os.FileInfo, error) { 93 | return aw.statter.Stat() 94 | } 95 | 96 | // ActualLength returns the length embedded in the action if it is not 97 | // Inf (i.e. when it's an extent). Otherwise, interpret it as EOF 98 | // and stat the actual file to determine the length on disk. 99 | func ActualLength(action dmplugin.Action, fp statter) (int64, error) { 100 | var length int64 101 | if action.Length() == lustre.MaxExtentLength { 102 | fi, err := fp.Stat() 103 | if err != nil { 104 | return 0, errors.Wrap(err, "stat failed") 105 | } 106 | 107 | length = fi.Size() - int64(action.Offset()) 108 | } else { 109 | // TODO: Sanity check length + offset with actual file size? 110 | length = int64(action.Length()) 111 | } 112 | return length, nil 113 | } 114 | 115 | // NewBufferedActionReader returns a *BufferedActionReader for the supplied 116 | // action. 117 | func NewBufferedActionReader(action dmplugin.Action) (*BufferedActionReader, int64, error) { 118 | ar, length, err := NewActionReader(action) 119 | if err != nil { 120 | return nil, 0, errors.Wrapf(err, "Failed to create ActionReader from %s", action) 121 | } 122 | 123 | return &BufferedActionReader{ 124 | br: bufio.NewReaderSize(ar, BufferSize), 125 | closer: ar, 126 | }, length, nil 127 | } 128 | 129 | // NewActionReader returns an *ActionReader for the supplied action. 130 | func NewActionReader(action dmplugin.Action) (*ActionReader, int64, error) { 131 | src, err := os.Open(action.PrimaryPath()) 132 | if err != nil { 133 | return nil, 0, errors.Wrapf(err, "Failed to open %s for read", action.PrimaryPath()) 134 | } 135 | 136 | length, err := ActualLength(action, src) 137 | if err != nil { 138 | return nil, 0, errors.Wrapf(err, "Could not determine extent length for %s", action) 139 | } 140 | 141 | return &ActionReader{ 142 | sr: io.NewSectionReader(src, int64(action.Offset()), length), 143 | closer: src, 144 | }, length, nil 145 | } 146 | 147 | // NewActionWriter returns an *ActionWriter for the supplied action. 148 | func NewActionWriter(action dmplugin.Action) (*ActionWriter, error) { 149 | dst, err := os.OpenFile(action.WritePath(), os.O_WRONLY, 0644) 150 | if err != nil { 151 | return nil, errors.Wrapf(err, "Failed to open %s for write", action.WritePath()) 152 | } 153 | 154 | // Set up for simple Write() 155 | dst.Seek(int64(action.Offset()), 0) 156 | 157 | return &ActionWriter{ 158 | baseOffset: int64(action.Offset()), 159 | wwa: dst, 160 | statter: dst, 161 | closer: dst, 162 | }, nil 163 | } 164 | -------------------------------------------------------------------------------- /dmplugin/plugin.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package dmplugin 6 | 7 | import ( 8 | "net" 9 | "path" 10 | "sync" 11 | "time" 12 | 13 | "github.com/pkg/errors" 14 | 15 | "golang.org/x/net/context" 16 | 17 | pb "github.com/edwardsp/lemur/pdm" 18 | "github.com/edwardsp/lemur/pkg/fsroot" 19 | "google.golang.org/grpc" 20 | ) 21 | 22 | // Plugin manages communication between the HSM agent and the datamover 23 | type Plugin struct { 24 | name string 25 | ctx context.Context 26 | cancelContext context.CancelFunc 27 | rpcConn *grpc.ClientConn 28 | cli pb.DataMoverClient 29 | movers []*DataMoverClient 30 | fsClient fsroot.Client 31 | config *pluginConfig 32 | } 33 | 34 | func unixDialer(addr string, timeout time.Duration) (net.Conn, error) { 35 | return net.DialTimeout("unix", addr, timeout) 36 | } 37 | 38 | // New returns a new *Plugin, or error 39 | func New(name string, initClient func(string) (fsroot.Client, error)) (*Plugin, error) { 40 | config := mustInitConfig() 41 | 42 | fsClient, err := initClient(config.ClientRoot) 43 | if err != nil { 44 | return nil, errors.Wrap(err, "client init failed") 45 | } 46 | 47 | ctx, cancel := context.WithCancel(context.Background()) 48 | conn, err := grpc.Dial(config.AgentAddress, grpc.WithDialer(unixDialer), grpc.WithInsecure()) 49 | if err != nil { 50 | return nil, errors.Wrap(err, "dial gprc server failed") 51 | } 52 | return &Plugin{ 53 | name: name, 54 | rpcConn: conn, 55 | ctx: ctx, 56 | cancelContext: cancel, 57 | cli: pb.NewDataMoverClient(conn), 58 | fsClient: fsClient, 59 | config: config, 60 | }, nil 61 | } 62 | 63 | // FsName returns the associated Lustre filesystem name 64 | func (a *Plugin) FsName() string { 65 | return a.fsClient.FsName() 66 | } 67 | 68 | // Base returns the root directory for plugin. 69 | func (a *Plugin) Base() string { 70 | return a.fsClient.Path() 71 | } 72 | 73 | // ConfigFile returns path to the plugin config file. 74 | func (a *Plugin) ConfigFile() string { 75 | return path.Join(a.config.ConfigDir, a.name) 76 | } 77 | 78 | // AddMover registers a new data mover with the plugin 79 | func (a *Plugin) AddMover(config *Config) { 80 | dm := NewMover(a, a.cli, config) 81 | a.movers = append(a.movers, dm) 82 | } 83 | 84 | // Run starts the data mover threads and blocks until they complete. 85 | func (a *Plugin) Run() { 86 | var wg sync.WaitGroup 87 | for _, dm := range a.movers { 88 | wg.Add(1) 89 | go func(dm *DataMoverClient) { 90 | dm.Run(a.ctx) 91 | wg.Done() 92 | }(dm) 93 | } 94 | wg.Wait() 95 | } 96 | 97 | // Stop signals to all registered data movers that they should stop processing 98 | // and shut down 99 | func (a *Plugin) Stop() { 100 | a.cancelContext() 101 | } 102 | 103 | // Close closes the connection to the agent 104 | func (a *Plugin) Close() error { 105 | return errors.Wrap(a.rpcConn.Close(), "closed failed") 106 | } 107 | -------------------------------------------------------------------------------- /dmplugin/testing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package dmplugin 6 | 7 | import ( 8 | lustre "github.com/edwardsp/go-lustre" 9 | "github.com/intel-hpdd/logging/alert" 10 | ) 11 | 12 | // Fataler provides Fatal and Fatalf 13 | type Fataler interface { 14 | Fatal(args ...interface{}) 15 | Fatalf(format string, args ...interface{}) 16 | } 17 | 18 | // TestAction is an Action implementation used for testing Movers. 19 | type TestAction struct { 20 | t Fataler 21 | id uint64 22 | path string 23 | offset int64 24 | length int64 25 | data []byte 26 | uuid string 27 | hash []byte 28 | url string 29 | ActualLength int 30 | Updates int 31 | } 32 | 33 | // NewTestAction returns a stub action that can be used for testing. 34 | func NewTestAction(t Fataler, path string, offset int64, length int64, uuid string, data []byte) *TestAction { 35 | return &TestAction{ 36 | t: t, 37 | id: 1, 38 | path: path, 39 | offset: offset, 40 | length: length, 41 | uuid: uuid, 42 | data: data, 43 | } 44 | } 45 | 46 | // Update sends an action status update 47 | func (a *TestAction) Update(offset, length, max int64) error { 48 | a.Updates++ 49 | return nil 50 | } 51 | 52 | // Complete signals that the action has completed 53 | func (a *TestAction) Complete() error { 54 | return nil 55 | } 56 | 57 | // Fail signals that the action has failed 58 | func (a *TestAction) Fail(err error) error { 59 | alert.Warnf("fail: id:%d %v", a.id, err) 60 | return nil 61 | } 62 | 63 | // ID returns the action item's ID 64 | func (a *TestAction) ID() uint64 { 65 | return a.id 66 | } 67 | 68 | // Offset returns the current offset of the action item 69 | func (a *TestAction) Offset() int64 { 70 | return a.offset 71 | } 72 | 73 | // Length returns the expected length of the action item's file 74 | func (a *TestAction) Length() int64 { 75 | return a.length 76 | } 77 | 78 | // Data returns a byte slice of the action item's data 79 | func (a *TestAction) Data() []byte { 80 | return a.data 81 | } 82 | 83 | // PrimaryPath returns the action item's primary file path 84 | func (a *TestAction) PrimaryPath() string { 85 | return a.path 86 | } 87 | 88 | // WritePath returns the action item's write path (e.g. for restores) 89 | func (a *TestAction) WritePath() string { 90 | return a.path 91 | } 92 | 93 | // UUID returns the action item's file id 94 | func (a *TestAction) UUID() string { 95 | return a.uuid 96 | } 97 | 98 | // Hash returns the action item's file id 99 | func (a *TestAction) Hash() []byte { 100 | return a.hash 101 | } 102 | 103 | // URL returns the action item's file id 104 | func (a *TestAction) URL() string { 105 | return a.url 106 | } 107 | 108 | // SetUUID returns the action item's file id 109 | func (a *TestAction) SetUUID(u string) { 110 | a.uuid = u 111 | } 112 | 113 | // SetHash sets the action's file id 114 | func (a *TestAction) SetHash(id []byte) { 115 | a.hash = id 116 | } 117 | 118 | // SetURL returns the action item's file id 119 | func (a *TestAction) SetURL(u string) { 120 | a.url = u 121 | } 122 | 123 | // SetActualLength sets the action's actual file length 124 | func (a *TestAction) SetActualLength(length int64) { 125 | if a.length != lustre.MaxExtentLength && length != a.length { 126 | a.t.Fatalf("actual length does not match original %d != %d", length, a.length) 127 | } 128 | a.ActualLength = int(length) 129 | } 130 | -------------------------------------------------------------------------------- /doc/agent.example: -------------------------------------------------------------------------------- 1 | ## Sample configuration for the Lustre HSM Agent. See lhsmd(1) for more information. 2 | ## 3 | ## The mount target for the Lustre filesystem that will be used with this agent. 4 | ## 5 | # client_device= "10.0.2.15@tcp:/lustre" 6 | 7 | client_device = "required" 8 | 9 | ## 10 | ## Base directory used for the Lustre mount points created by the agent 11 | ## 12 | # mount_root= "/var/lib/lhsmd/roots" 13 | 14 | ## 15 | ## List of enabled plugins 16 | ## 17 | # enabled_plugins = ["lhsm-plugin-posix", "lhsm-plugin-s3"] 18 | 19 | ## 20 | ## Directory to look for the plugins 21 | ## 22 | # plugin_dir = "/usr/libexec/lhsmd" 23 | 24 | ## 25 | ## Number of threads handling incoming HSM requests. 26 | ## 27 | # handler_count = 4 28 | 29 | ## 30 | ## Enable expeimental snapshot feature. 31 | ## 32 | # snapshots { 33 | # enabled = false 34 | # } 35 | 36 | 37 | ## 38 | ## Metric Storage 39 | ## 40 | # influxdb { 41 | # url = "http://10.0.1.123:8086" 42 | # db = "lhsmd" 43 | # user = "*user*" 44 | # password = "*password*" 45 | # } 46 | -------------------------------------------------------------------------------- /doc/lhsm-plugin-posix.example: -------------------------------------------------------------------------------- 1 | ## Sample configuration for the POSIX data mover plugin. 2 | ## 3 | ## Maximum number of concurrent copies. 4 | ## 5 | # num_threads = 8 6 | 7 | ## 8 | ## One or more archive definition is required. 9 | ## 10 | # archive "s3-test" { 11 | # id = 2 # Must be unique to this endpoint 12 | # root = "/path/to/archvie" # The base directory of the archive 13 | # compression = "off" # Enable data compression with "on" 14 | # 15 | # checksums { 16 | # disabled = false # Generating checksums is enabled by default 17 | # disable_compare_on_restore = false # Ignore existing checksums during restore 18 | # } 19 | # } 20 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/edwardsp/lemur 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/Azure/azure-pipeline-go v0.2.3 7 | github.com/Azure/azure-sdk-for-go v48.2.0+incompatible 8 | github.com/Azure/azure-storage-azcopy v10.0.2+incompatible 9 | github.com/Azure/azure-storage-blob-go v0.10.1-0.20201027154117-4e8f2d48550b 10 | github.com/Azure/go-autorest/autorest v0.11.20 // indirect 11 | github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect 12 | github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect 13 | github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect 14 | github.com/dustin/go-humanize v1.0.0 15 | github.com/edwardsp/go-lustre v0.0.23 16 | github.com/fortytw2/leaktest v1.3.0 17 | github.com/golang/protobuf v1.4.2 18 | github.com/hashicorp/hcl v1.0.0 19 | github.com/influxdata/influxdb v1.9.3 // indirect 20 | github.com/intel-hpdd/go-metrics-influxdb v0.0.0-20160818143319-2b4497452ba3 21 | github.com/intel-hpdd/logging v0.0.0-20170320184255-0ce155fc7407 22 | github.com/pborman/uuid v1.2.0 23 | github.com/pkg/errors v0.9.1 24 | github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 25 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 26 | golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 27 | google.golang.org/grpc v1.31.0 28 | google.golang.org/protobuf v1.23.0 29 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f 30 | ) 31 | -------------------------------------------------------------------------------- /internal/testhelpers/helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package testhelpers 6 | 7 | import ( 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "testing" 12 | ) 13 | 14 | var testPrefix = "ptest" 15 | 16 | // TempDir returns path to a new temporary directory and function that will 17 | // forcibly remove it. 18 | func TempDir(t *testing.T) (string, func()) { 19 | tdir, err := ioutil.TempDir("", testPrefix) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | return tdir, func() { 24 | err = os.RemoveAll(tdir) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | } 29 | } 30 | 31 | // ChdirTemp changes the working directory to a new TempDir. The cleanup 32 | // function returns to the previous working directoy and removes the temp 33 | // directory. 34 | func ChdirTemp(t *testing.T) func() { 35 | tdir, cleanDir := TempDir(t) 36 | 37 | cwd, err := os.Getwd() 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | err = os.Chdir(tdir) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | 47 | return func() { 48 | err := os.Chdir(cwd) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | cleanDir() 53 | } 54 | } 55 | 56 | // Fill writes size amount of bytes to the file. 57 | func Fill(t *testing.T, fp io.Writer, size int64) { 58 | var bs int64 = 1024 * 1024 59 | buf := make([]byte, bs) 60 | 61 | for i := 0; i < len(buf); i++ { 62 | buf[i] = byte(i) 63 | } 64 | 65 | for i := int64(0); i < size; i += bs { 66 | if size < bs { 67 | bs = size 68 | } 69 | fp.Write(buf[:bs]) 70 | 71 | } 72 | } 73 | 74 | // CorruptFile writes an string to the beginning of the file. 75 | func CorruptFile(t *testing.T, path string) { 76 | fp, err := os.OpenFile(path, os.O_RDWR, 0644) 77 | if err != nil { 78 | t.Fatal(err) 79 | } 80 | _, err = fp.Write([]byte("Silent data corruption. :)")) 81 | if err != nil { 82 | t.Fatal(err) 83 | 84 | } 85 | err = fp.Close() 86 | if err != nil { 87 | t.Fatal(err) 88 | 89 | } 90 | } 91 | 92 | // TempFile creates a temporary file. If size is >0 then that amount of bytes 93 | // will be written to the file. 94 | func TempFile(t *testing.T, size int64) (string, func()) { 95 | fp, err := ioutil.TempFile(".", testPrefix) 96 | if err != nil { 97 | t.Fatal(err) 98 | } 99 | defer fp.Close() 100 | 101 | if size > 0 { 102 | Fill(t, fp, size) 103 | } 104 | name := fp.Name() 105 | return name, func() { 106 | err := os.Remove(name) 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | } 111 | } 112 | 113 | // CopyFile copies data from one file another. If the target file does not 114 | // exist then it will be created with the given mode. This is a non-optimal copy 115 | // and not intended to be used for very large files. 116 | func CopyFile(t *testing.T, src string, dest string, mode os.FileMode) { 117 | buf, err := ioutil.ReadFile(src) 118 | if err != nil { 119 | t.Fatal(err) 120 | } 121 | 122 | err = ioutil.WriteFile(dest, buf, mode) 123 | if err != nil { 124 | t.Fatal(err) 125 | } 126 | } 127 | 128 | // TempCopy copies provided file to a new temp file that will be assigned the 129 | // provided mode after the copy. (So the mode can specify a read-only file.) 130 | func TempCopy(t *testing.T, src string, mode os.FileMode) (string, func()) { 131 | tmpFile, cleanup := TempFile(t, 0) 132 | CopyFile(t, src, tmpFile, mode) 133 | 134 | // ensure file has correct mode, in case we're overwriting 135 | err := os.Chmod(tmpFile, mode) 136 | if err != nil { 137 | t.Fatal(err) 138 | } 139 | 140 | return tmpFile, cleanup 141 | } 142 | 143 | // Action yields "action". 144 | // Srsly, wtf? 145 | func Action(t *testing.T) string { 146 | return "action" 147 | } 148 | -------------------------------------------------------------------------------- /man/lhsm-plugin-posix.1.md: -------------------------------------------------------------------------------- 1 | % LHSMD (1) User Manual 2 | % Intel Corporation 3 | % REPLACE_DATE 4 | 5 | # NAME 6 | 7 | lhsm-plugin-posix - Lhsmd plugin for POSIX archives 8 | 9 | # DESCRIPTION 10 | 11 | `lhsm-plugin-posix` is a data mover that supports archiving data to a POSIX file system. It is not intended 12 | to be run directly, and should only be run by `lhsmd`. It is configured using the 13 | configuration file. 14 | 15 | # GENERAL USAGE 16 | 17 | The default location for the mover configuration file is `/etc/lhsmd/lhsm-plugin-posix`. 18 | These are the configuration options available. 19 | 20 | `num_threads` 21 | : The maximum number of concurrent copy requests the plugin will allow. 22 | 23 | `archive` 24 | : Each `archive` section configures an archive endpoint that will be registered with the agent 25 | and corresponds with a Lustre Archive ID. It is important that each Archive ID be used with the 26 | same endpoint on each data mover. 27 | 28 | `id` 29 | : The ID associated with this archive. 30 | 31 | `root` 32 | : The base directory of the archive. Must be accessible on the mover node. 33 | 34 | `compression` 35 | : If set to "on", all data will be compressed when written to the backend. Compressed 36 | data objects will be automatically uncompressed during restore even if this option has been 37 | subsequently disabled. 38 | If set to "auto", then a small portion of each file will be compressed to determine 39 | if the file should be compressed or not. If the initial check yields better than 30% 40 | compression, the the file will be compressed. This still an experimental feature 41 | and will likely need further refinement and optimization. 42 | 43 | `checksums` 44 | : By default, data checksums are created when a file is archived and validated on restore. 45 | These options can be used to disable checksums entirely or just disable restore validation (useful 46 | if checksums have been corrupted or lost). 47 | 48 | `disabled` 49 | : This inhibits creation of file checksums. 50 | 51 | `disable_compare_on_restore` 52 | : This prevents checking file checksums on restore. 53 | 54 | 55 | # EXAMPLES 56 | 57 | A sample S3 plugin configuration with one archive: 58 | 59 | num_threads = 8 60 | 61 | archive "posix-test" { 62 | id = 1 63 | root = "/tmp/archive" 64 | compression = "off" 65 | checksums { 66 | disabled = false 67 | } 68 | } 69 | 70 | # SEE ALSO 71 | 72 | `lhsmd` (1) 73 | -------------------------------------------------------------------------------- /man/lhsm-plugin-s3.1.md: -------------------------------------------------------------------------------- 1 | % LHSMD (1) User Manual 2 | % Intel Corporation 3 | % REPLACE_DATE 4 | 5 | # NAME 6 | 7 | lhsm-plugin-s3 - Lhsmd plugin for AWS S3 8 | 9 | # DESCRIPTION 10 | 11 | `lhsm-plugin-s3` is a data mover that supports archiving data in AWS S3. It is not intended 12 | to be run directly, and should only be run by `lhsmd`. 13 | 14 | # GENERAL USAGE 15 | 16 | The default location for the mover configuration file is `/etc/lhsmd/lhsm-plugin-s3`. 17 | These are the configuration options available. 18 | 19 | The four S3 service connection parameters can be set globally or customized for each 20 | archive. 21 | 22 | `region` 23 | : The AWS region to use. The default is `us-east-1`. 24 | 25 | `endpoint` 26 | : The full URL of the S3 service. The service must support auth V4 signed 27 | authentication. The default value will be the AWS S3 endpoint for the 28 | current region so this is only needed when using non-AWS S3 services. 29 | 30 | `aws_access_key_id` 31 | : The access key with permissions to write to the provide S3 32 | bucket. This option is provided for convenience. Typically keys are 33 | provided though a standard mechanism for AWS tools, such as 34 | ~/.aws/credentials, AWS_ACCESS_KEY_ID environment variable, or 35 | an IAM Role. If this is set, then this will take priority over 36 | other keys found in the environment. 37 | 38 | `aws_secret_access_key` 39 | : The AWS secret key. This option is provided for convenience. Typically keys are 40 | provided though a standard mechanism for AWS tools, such as ~/.aws/credentials, 41 | AWS_SECRET_ACCESS_KEY environment variable, or an IAM Role. If this is set, then 42 | this will take priority over other keys found in the environment. 43 | 44 | 45 | 46 | `num_threads` 47 | : The maximum number of concurrent copy requests the plugin will allow. 48 | 49 | `archive` 50 | : Each `archive` section configures an archive endpoint that will be registered with the agent 51 | and corresponds with a Lustre Archive ID. It is important that each Archive ID be used with the 52 | same endpoint on each data mover. 53 | 54 | `id` 55 | : The ID associated with this archive. 56 | 57 | `bucket` 58 | : The AWS S3 bucket that will be used. 59 | 60 | `prefix` 61 | : An optional prefix key for the archive objects. 62 | 63 | # EXAMPLES 64 | 65 | A sample S3 plugin configuration with one archive: 66 | 67 | num_threads = 8 68 | 69 | archive "s3-test" { 70 | id = 2 71 | bucket = "*bucket*" 72 | prefix = "s3-test-archive" 73 | } 74 | 75 | # SEE ALSO 76 | 77 | `lhsmd` (1) 78 | -------------------------------------------------------------------------------- /man/lhsmd.1.md: -------------------------------------------------------------------------------- 1 | % LHSMD (1) User Manual 2 | % Intel Corporation 3 | % REPLACE_DATE 4 | 5 | # NAME 6 | 7 | lhsmd - Lustre HSM Agent 8 | 9 | # SYNOPSIS 10 | 11 | lhsmd [-config *FILE*] [-debug] 12 | 13 | # DESCRIPTION 14 | 15 | Lhsmd is a Lustre HSM Agent. It handles HSM requests from the Lustre 16 | coordinator, and forwards the requests to the configured data mover 17 | plugins based on the archive id of the request. The configuration of 18 | the plugins specifies which Lustre Archive ID is associated with an a 19 | each archive endpoint. More than one plugin can be used at the same 20 | time, and each data mover can support multiple archive IDs and 21 | endpoints. 22 | 23 | The agent configuration file specifies which Lustre filesystem is being managed, 24 | which plugins to start, and options storing 25 | metrics in an InfluxDB database. By default, example config files are 26 | provided in `/etc/lhsmd`. These can be copied to the correct, non 27 | ".example" name and customized accordingly. 28 | 29 | Although the agent can be run directly on the command for debugging 30 | purposes, for production use we recommend using systemd (or equivalent) to 31 | manage and run the lhsmd service to ensure only one agent runs per 32 | host. 33 | 34 | # systemctl enable lhsmd 35 | # systemctl start lhsmd 36 | 37 | # OPTIONS 38 | 39 | -config *FILE* 40 | : Specify configuration file instead of using default 41 | `/etc/lhsmd/agent`. 42 | 43 | -debug 44 | : Enable debug logging. 45 | 46 | # GENERAL USAGE 47 | 48 | The default location for the agent configuration file is `/etc/lhsmd/agent`. These are the configuration options available. 49 | 50 | `client_device` 51 | : Required option, the `client_device` the mount target for the Lustre filesystem the agent will be using. The 52 | agent will create mount points of the filesystem for itself and for each of the configured plugins. 53 | 54 | `mount_root` 55 | : The `mount_root` is the location for the Lustre mount points created by the agent. 56 | 57 | `enabled_plugins` 58 | : A list of plugins to start. If the plugin name is not an absolute path, the agent will search for a binary 59 | matching the plugin name provided here. 60 | 61 | `plugin_dir` 62 | : An additional directory to search for plugins. 63 | 64 | `handler_count` 65 | : Number of threads that will be used to process HSM requests in the agent. (The number of threads in the 66 | plugins is configured separately) 67 | 68 | `snapshots` 69 | : Optional section to enable the HSM Snapshot feature. When this is enabled, 70 | then each time a file is archived, the agent will create a released copy of file in 71 | `.hsmsnapshot` which corresponds to archived version of the file. If the original file 72 | is changed or deleted, then the snapshot can be used to retrieve the archived version. 73 | 74 | `enabled` 75 | : If true, then the experimental HSM snapshot feature is enabled. 76 | 77 | `influxdb` 78 | : Optional section for storing `lhsmd` metrics in an InfluxDB database. 79 | 80 | `url` 81 | : Optional URL used for sending metrics to an InfluxDB. If not set, the metrics will not be saved. 82 | 83 | `db` 84 | : Name for the database for metrics. 85 | 86 | `user` 87 | : InfluxDB user name. 88 | 89 | `password` 90 | : InfluxDB password. 91 | 92 | # EXAMPLES 93 | 94 | A sample agent configuration that enables the snapshot feature: 95 | 96 | mount_root= "/var/lib/lhsmd/roots" 97 | client_device= "10.0.2.15@tcp:/lustre" 98 | enabled_plugins = ["lhsm-plugin-posix", "lhsm-plugin-s3"] 99 | handler_count = 4 100 | snapshots { 101 | enabled = true 102 | } 103 | 104 | influxdb { 105 | url = "http://10.0.1.123:8086" 106 | db = "lhsmd" 107 | user = "*user*" 108 | password = "*password*" 109 | } 110 | 111 | # SEE ALSO 112 | 113 | `lhsm-plugin-s3` (1), `lhsm-plugin-posix` (1), `lfs-hsm` (1) 114 | -------------------------------------------------------------------------------- /packaging/ci/bucketsite/ajaxload-circle.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edwardsp/lemur/de15cc61af28e78b8ca18dbc7fc6be1fe30cdde1/packaging/ci/bucketsite/ajaxload-circle.gif -------------------------------------------------------------------------------- /packaging/ci/bucketsite/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 |
7 | 8 | 9 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/Makefile: -------------------------------------------------------------------------------- 1 | PYTHON ?= python2.7 2 | FUNCTION_NAME = $(notdir $(CURDIR)) 3 | OUTDIR ?= $(dir $(CURDIR)) 4 | ZIPFILE ?= $(OUTDIR)/$(FUNCTION_NAME).zip 5 | 6 | $(ZIPFILE): *.py 7 | rm -f $@ 8 | zip -r $@ * 9 | 10 | clean: 11 | rm -f $(ZIPFILE) 12 | 13 | .PHONY: clean 14 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/NOTICE.txt: -------------------------------------------------------------------------------- 1 | Git2S3-GitPullS3 2 | Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/_cffi_backend.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edwardsp/lemur/de15cc61af28e78b8ca18dbc7fc6be1fe30cdde1/packaging/ci/lambda/GitPullS3/_cffi_backend.so -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/_pygit2.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edwardsp/lemur/de15cc61af28e78b8ca18dbc7fc6be1fe30cdde1/packaging/ci/lambda/GitPullS3/_pygit2.so -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/cffi/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Except when otherwise stated (look for LICENSE files in directories or 3 | information at the beginning of each file) all software and 4 | documentation is licensed as follows: 5 | 6 | The MIT License 7 | 8 | Permission is hereby granted, free of charge, to any person 9 | obtaining a copy of this software and associated documentation 10 | files (the "Software"), to deal in the Software without 11 | restriction, including without limitation the rights to use, 12 | copy, modify, merge, publish, distribute, sublicense, and/or 13 | sell copies of the Software, and to permit persons to whom the 14 | Software is furnished to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included 17 | in all copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | 27 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/cffi/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError', 2 | 'FFIError'] 3 | 4 | from .api import FFI, CDefError, FFIError 5 | from .ffiplatform import VerificationError, VerificationMissing 6 | 7 | __version__ = "1.7.0" 8 | __version_info__ = (1, 7, 0) 9 | 10 | # The verifier module file names are based on the CRC32 of a string that 11 | # contains the following version number. It may be older than __version__ 12 | # if nothing is clearly incompatible. 13 | __version_verifier_modules__ = "0.8.6" 14 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/cffi/commontypes.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from . import api, model 3 | 4 | 5 | COMMON_TYPES = {} 6 | 7 | try: 8 | # fetch "bool" and all simple Windows types 9 | from _cffi_backend import _get_common_types 10 | _get_common_types(COMMON_TYPES) 11 | except ImportError: 12 | pass 13 | 14 | COMMON_TYPES['FILE'] = model.unknown_type('FILE', '_IO_FILE') 15 | COMMON_TYPES['bool'] = '_Bool' # in case we got ImportError above 16 | 17 | for _type in model.PrimitiveType.ALL_PRIMITIVE_TYPES: 18 | if _type.endswith('_t'): 19 | COMMON_TYPES[_type] = _type 20 | del _type 21 | 22 | _CACHE = {} 23 | 24 | def resolve_common_type(parser, commontype): 25 | try: 26 | return _CACHE[commontype] 27 | except KeyError: 28 | cdecl = COMMON_TYPES.get(commontype, commontype) 29 | if not isinstance(cdecl, str): 30 | result, quals = cdecl, 0 # cdecl is already a BaseType 31 | elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES: 32 | result, quals = model.PrimitiveType(cdecl), 0 33 | elif cdecl == 'set-unicode-needed': 34 | raise api.FFIError("The Windows type %r is only available after " 35 | "you call ffi.set_unicode()" % (commontype,)) 36 | else: 37 | if commontype == cdecl: 38 | raise api.FFIError( 39 | "Unsupported type: %r. Please look at " 40 | "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations " 41 | "and file an issue if you think this type should really " 42 | "be supported." % (commontype,)) 43 | result, quals = parser.parse_type_and_quals(cdecl) # recursive 44 | 45 | assert isinstance(result, model.BaseTypeByIdentity) 46 | _CACHE[commontype] = result, quals 47 | return result, quals 48 | 49 | 50 | # ____________________________________________________________ 51 | # extra types for Windows (most of them are in commontypes.c) 52 | 53 | 54 | def win_common_types(): 55 | return { 56 | "UNICODE_STRING": model.StructType( 57 | "_UNICODE_STRING", 58 | ["Length", 59 | "MaximumLength", 60 | "Buffer"], 61 | [model.PrimitiveType("unsigned short"), 62 | model.PrimitiveType("unsigned short"), 63 | model.PointerType(model.PrimitiveType("wchar_t"))], 64 | [-1, -1, -1]), 65 | "PUNICODE_STRING": "UNICODE_STRING *", 66 | "PCUNICODE_STRING": "const UNICODE_STRING *", 67 | 68 | "TBYTE": "set-unicode-needed", 69 | "TCHAR": "set-unicode-needed", 70 | "LPCTSTR": "set-unicode-needed", 71 | "PCTSTR": "set-unicode-needed", 72 | "LPTSTR": "set-unicode-needed", 73 | "PTSTR": "set-unicode-needed", 74 | "PTBYTE": "set-unicode-needed", 75 | "PTCHAR": "set-unicode-needed", 76 | } 77 | 78 | if sys.platform == 'win32': 79 | COMMON_TYPES.update(win_common_types()) 80 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/cffi/ffiplatform.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | 3 | 4 | class VerificationError(Exception): 5 | """ An error raised when verification fails 6 | """ 7 | 8 | class VerificationMissing(Exception): 9 | """ An error raised when incomplete structures are passed into 10 | cdef, but no verification has been done 11 | """ 12 | 13 | 14 | LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', 15 | 'extra_objects', 'depends'] 16 | 17 | def get_extension(srcfilename, modname, sources=(), **kwds): 18 | from distutils.core import Extension 19 | allsources = [srcfilename] 20 | for src in sources: 21 | allsources.append(os.path.normpath(src)) 22 | return Extension(name=modname, sources=allsources, **kwds) 23 | 24 | def compile(tmpdir, ext, compiler_verbose=0): 25 | """Compile a C extension module using distutils.""" 26 | 27 | saved_environ = os.environ.copy() 28 | try: 29 | outputfilename = _build(tmpdir, ext, compiler_verbose) 30 | outputfilename = os.path.abspath(outputfilename) 31 | finally: 32 | # workaround for a distutils bugs where some env vars can 33 | # become longer and longer every time it is used 34 | for key, value in saved_environ.items(): 35 | if os.environ.get(key) != value: 36 | os.environ[key] = value 37 | return outputfilename 38 | 39 | def _build(tmpdir, ext, compiler_verbose=0): 40 | # XXX compact but horrible :-( 41 | from distutils.core import Distribution 42 | import distutils.errors, distutils.log 43 | # 44 | dist = Distribution({'ext_modules': [ext]}) 45 | dist.parse_config_files() 46 | options = dist.get_option_dict('build_ext') 47 | options['force'] = ('ffiplatform', True) 48 | options['build_lib'] = ('ffiplatform', tmpdir) 49 | options['build_temp'] = ('ffiplatform', tmpdir) 50 | # 51 | try: 52 | old_level = distutils.log.set_threshold(0) or 0 53 | try: 54 | distutils.log.set_verbosity(compiler_verbose) 55 | dist.run_command('build_ext') 56 | cmd_obj = dist.get_command_obj('build_ext') 57 | [soname] = cmd_obj.get_outputs() 58 | finally: 59 | distutils.log.set_threshold(old_level) 60 | except (distutils.errors.CompileError, 61 | distutils.errors.LinkError) as e: 62 | raise VerificationError('%s: %s' % (e.__class__.__name__, e)) 63 | # 64 | return soname 65 | 66 | try: 67 | from os.path import samefile 68 | except ImportError: 69 | def samefile(f1, f2): 70 | return os.path.abspath(f1) == os.path.abspath(f2) 71 | 72 | def maybe_relative_path(path): 73 | if not os.path.isabs(path): 74 | return path # already relative 75 | dir = path 76 | names = [] 77 | while True: 78 | prevdir = dir 79 | dir, name = os.path.split(prevdir) 80 | if dir == prevdir or not dir: 81 | return path # failed to make it relative 82 | names.append(name) 83 | try: 84 | if samefile(dir, os.curdir): 85 | names.reverse() 86 | return os.path.join(*names) 87 | except OSError: 88 | pass 89 | 90 | # ____________________________________________________________ 91 | 92 | try: 93 | int_or_long = (int, long) 94 | import cStringIO 95 | except NameError: 96 | int_or_long = int # Python 3 97 | import io as cStringIO 98 | 99 | def _flatten(x, f): 100 | if isinstance(x, str): 101 | f.write('%ds%s' % (len(x), x)) 102 | elif isinstance(x, dict): 103 | keys = sorted(x.keys()) 104 | f.write('%dd' % len(keys)) 105 | for key in keys: 106 | _flatten(key, f) 107 | _flatten(x[key], f) 108 | elif isinstance(x, (list, tuple)): 109 | f.write('%dl' % len(x)) 110 | for value in x: 111 | _flatten(value, f) 112 | elif isinstance(x, int_or_long): 113 | f.write('%di' % (x,)) 114 | else: 115 | raise TypeError( 116 | "the keywords to verify() contains unsupported object %r" % (x,)) 117 | 118 | def flatten(x): 119 | f = cStringIO.StringIO() 120 | _flatten(x, f) 121 | return f.getvalue() 122 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/cffi/lock.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | if sys.version_info < (3,): 4 | try: 5 | from thread import allocate_lock 6 | except ImportError: 7 | from dummy_thread import allocate_lock 8 | else: 9 | try: 10 | from _thread import allocate_lock 11 | except ImportError: 12 | from _dummy_thread import allocate_lock 13 | 14 | 15 | ##import sys 16 | ##l1 = allocate_lock 17 | 18 | ##class allocate_lock(object): 19 | ## def __init__(self): 20 | ## self._real = l1() 21 | ## def __enter__(self): 22 | ## for i in range(4, 0, -1): 23 | ## print sys._getframe(i).f_code 24 | ## print 25 | ## return self._real.__enter__() 26 | ## def __exit__(self, *args): 27 | ## return self._real.__exit__(*args) 28 | ## def acquire(self, f): 29 | ## assert f is False 30 | ## return self._real.acquire(f) 31 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/libgit2.so.24: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edwardsp/lemur/de15cc61af28e78b8ca18dbc7fc6be1fe30cdde1/packaging/ci/lambda/GitPullS3/libgit2.so.24 -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/_build.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | """ 29 | This is an special module, it provides stuff used by setup.py at build time. 30 | But also used by pygit2 at run time. 31 | """ 32 | 33 | # Import from the Standard Library 34 | import os 35 | from os import getenv 36 | 37 | # 38 | # The version number of pygit2 39 | # 40 | __version__ = '0.24.1' 41 | 42 | 43 | # 44 | # Utility functions to get the paths required for bulding extensions 45 | # 46 | def _get_libgit2_path(): 47 | # LIBGIT2 environment variable takes precedence 48 | libgit2_path = getenv("LIBGIT2") 49 | if libgit2_path is not None: 50 | return libgit2_path 51 | 52 | # Default 53 | if os.name == 'nt': 54 | return '%s\libgit2' % getenv("ProgramFiles") 55 | return '/usr/local' 56 | 57 | 58 | def get_libgit2_paths(): 59 | libgit2_path = _get_libgit2_path() 60 | return ( 61 | os.path.join(libgit2_path, 'bin'), 62 | os.path.join(libgit2_path, 'include'), 63 | getenv('LIBGIT2_LIB', os.path.join(libgit2_path, 'lib')), 64 | ) 65 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/_libgit2.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edwardsp/lemur/de15cc61af28e78b8ca18dbc7fc6be1fe30cdde1/packaging/ci/lambda/GitPullS3/pygit2/_libgit2.so -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/_run.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | """ 29 | This is an special module, it provides stuff used by by pygit2 at run-time. 30 | """ 31 | 32 | # Import from the Standard Library 33 | import codecs 34 | import os 35 | from os.path import abspath, dirname 36 | import sys 37 | 38 | # Import from cffi 39 | from cffi import FFI 40 | 41 | # Import from pygit2 42 | from _build import get_libgit2_paths 43 | 44 | 45 | # C_HEADER_SRC 46 | if getattr(sys, 'frozen', False): 47 | dir_path = getattr(sys, '_MEIPASS', None) 48 | if dir_path is None: 49 | dir_path = dirname(abspath(sys.executable)) 50 | else: 51 | dir_path = dirname(abspath(__file__)) 52 | 53 | decl_path = os.path.join(dir_path, 'decl.h') 54 | with codecs.open(decl_path, 'r', 'utf-8') as header: 55 | C_HEADER_SRC = header.read() 56 | 57 | # C_KEYWORDS 58 | libgit2_bin, libgit2_include, libgit2_lib = get_libgit2_paths() 59 | C_KEYWORDS = dict(libraries=['git2'], 60 | library_dirs=[libgit2_lib], 61 | include_dirs=[libgit2_include]) 62 | 63 | # preamble 64 | preamble = "#include " 65 | 66 | # ffi 67 | ffi = FFI() 68 | set_source = getattr(ffi, 'set_source', None) 69 | if set_source is not None: 70 | set_source("pygit2._libgit2", preamble, **C_KEYWORDS) 71 | 72 | ffi.cdef(C_HEADER_SRC) 73 | 74 | 75 | if __name__ == '__main__': 76 | ffi.compile() 77 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/blame.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | # Import from the future 29 | from __future__ import absolute_import, unicode_literals 30 | 31 | # Import from pygit2 32 | from .errors import check_error 33 | from .ffi import ffi, C 34 | from .utils import to_bytes, is_string, to_str 35 | from .utils import GenericIterator 36 | from _pygit2 import Signature, Oid 37 | 38 | 39 | def wrap_signature(csig): 40 | if not csig: 41 | return None 42 | 43 | return Signature(ffi.string(csig.name).decode('utf-8'), 44 | ffi.string(csig.email).decode('utf-8'), 45 | csig.when.time, csig.when.offset, 'utf-8') 46 | 47 | class BlameHunk(object): 48 | 49 | @classmethod 50 | def _from_c(cls, blame, ptr): 51 | hunk = cls.__new__(cls) 52 | hunk._blame = blame 53 | hunk._hunk = ptr 54 | return hunk 55 | 56 | @property 57 | def lines_in_hunk(self): 58 | """Number of lines""" 59 | return self._hunk.lines_in_hunk 60 | 61 | @property 62 | def boundary(self): 63 | """Tracked to a boundary commit""" 64 | # Casting directly to bool via cffi does not seem to work 65 | return int(ffi.cast('int', self._hunk.boundary)) != 0 66 | 67 | @property 68 | def final_start_line_number(self): 69 | """Final start line number""" 70 | return self._hunk.final_start_line_number 71 | 72 | @property 73 | def final_committer(self): 74 | """Final committer""" 75 | return wrap_signature(self._hunk.final_signature) 76 | 77 | @property 78 | def final_commit_id(self): 79 | return Oid(raw=bytes(ffi.buffer(ffi.addressof(self._hunk, 'final_commit_id'))[:])) 80 | 81 | @property 82 | def orig_start_line_number(self): 83 | """Origin start line number""" 84 | return self._hunk.orig_start_line_number 85 | 86 | @property 87 | def orig_committer(self): 88 | """Original committer""" 89 | return wrap_signature(self._hunk.orig_signature) 90 | 91 | @property 92 | def orig_commit_id(self): 93 | return Oid(raw=bytes(ffi.buffer(ffi.addressof(self._hunk, 'orig_commit_id'))[:])) 94 | 95 | @property 96 | def orig_path(self): 97 | """Original path""" 98 | path = self._hunk.orig_path 99 | if not path: 100 | return None 101 | 102 | return ffi.string(path).decode() 103 | 104 | 105 | class Blame(object): 106 | 107 | @classmethod 108 | def _from_c(cls, repo, ptr): 109 | blame = cls.__new__(cls) 110 | blame._repo = repo 111 | blame._blame = ptr 112 | return blame 113 | 114 | def __del__(self): 115 | C.git_blame_free(self._blame) 116 | 117 | def __len__(self): 118 | return C.git_blame_get_hunk_count(self._blame) 119 | 120 | def __getitem__(self, index): 121 | chunk = C.git_blame_get_hunk_byindex(self._blame, index) 122 | if not chunk: 123 | raise IndexError 124 | 125 | return BlameHunk._from_c(self, chunk) 126 | 127 | def for_line(self, line_no): 128 | """Returns the object for a given line given its number 129 | in the current Blame. 130 | 131 | Arguments: 132 | 133 | line_no 134 | Line number, starts at 1. 135 | """ 136 | if line_no < 0: 137 | raise IndexError 138 | 139 | chunk = C.git_blame_get_hunk_byline(self._blame, line_no) 140 | if not chunk: 141 | raise IndexError 142 | 143 | return BlameHunk._from_c(self, chunk) 144 | 145 | def __iter__(self): 146 | return GenericIterator(self) 147 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/credentials.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | from .ffi import C 29 | 30 | GIT_CREDTYPE_USERPASS_PLAINTEXT = C.GIT_CREDTYPE_USERPASS_PLAINTEXT 31 | GIT_CREDTYPE_SSH_KEY = C.GIT_CREDTYPE_SSH_KEY 32 | 33 | 34 | class UserPass(object): 35 | """Username/Password credentials 36 | 37 | This is an object suitable for passing to a remote's credentials 38 | callback and for returning from said callback. 39 | """ 40 | 41 | def __init__(self, username, password): 42 | self._username = username 43 | self._password = password 44 | 45 | @property 46 | def credential_type(self): 47 | return GIT_CREDTYPE_USERPASS_PLAINTEXT 48 | 49 | @property 50 | def credential_tuple(self): 51 | return (self._username, self._password) 52 | 53 | def __call__(self, _url, _username, _allowed): 54 | return self 55 | 56 | 57 | class Keypair(object): 58 | """SSH key pair credentials 59 | 60 | This is an object suitable for passing to a remote's credentials 61 | callback and for returning from said callback. 62 | 63 | :param str username: the username being used to authenticate with the 64 | remote server 65 | :param str pubkey: the path to the user's public key file 66 | :param str privkey: the path to the user's private key file 67 | :param str passphrase: the password used to decrypt the private key file, 68 | or empty string if no passphrase is required. 69 | """ 70 | 71 | def __init__(self, username, pubkey, privkey, passphrase): 72 | self._username = username 73 | self._pubkey = pubkey 74 | self._privkey = privkey 75 | self._passphrase = passphrase 76 | 77 | @property 78 | def credential_type(self): 79 | return GIT_CREDTYPE_SSH_KEY 80 | 81 | @property 82 | def credential_tuple(self): 83 | return (self._username, self._pubkey, self._privkey, self._passphrase) 84 | 85 | def __call__(self, _url, _username, _allowed): 86 | return self 87 | 88 | 89 | class KeypairFromAgent(Keypair): 90 | def __init__(self, username): 91 | super(KeypairFromAgent, self).__init__(username, None, None, None) 92 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/errors.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | # Import from pygit2 29 | from .ffi import ffi, C 30 | from _pygit2 import GitError 31 | 32 | 33 | value_errors = set([C.GIT_EEXISTS, C.GIT_EINVALIDSPEC, C.GIT_EEXISTS, 34 | C.GIT_EAMBIGUOUS]) 35 | 36 | def check_error(err, io=False): 37 | if err >= 0: 38 | return 39 | 40 | # Error message 41 | giterr = C.giterr_last() 42 | if giterr != ffi.NULL: 43 | message = ffi.string(giterr.message).decode() 44 | else: 45 | message = "err %d (no message provided)" % err 46 | 47 | # Translate to Python errors 48 | if err in value_errors: 49 | raise ValueError(message) 50 | 51 | if err == C.GIT_ENOTFOUND: 52 | if io: 53 | raise IOError(message) 54 | 55 | raise KeyError(message) 56 | 57 | if err == C.GIT_EINVALIDSPEC: 58 | raise ValueError(message) 59 | 60 | if err == C.GIT_ITEROVER: 61 | raise StopIteration() 62 | 63 | # Generic Git error 64 | raise GitError(message) 65 | 66 | # Indicate that we want libgit2 to pretend a function was not set 67 | Passthrough = Exception("The function asked for pass-through") 68 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/ffi.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | # Import from the future 29 | from __future__ import absolute_import 30 | 31 | # Import from pygit2 32 | try: 33 | from ._libgit2 import ffi, lib as C 34 | except ImportError: 35 | from ._run import ffi, preamble, C_KEYWORDS 36 | C = ffi.verify(preamble, **C_KEYWORDS) 37 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/py2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | # Import from the future 29 | from __future__ import absolute_import 30 | 31 | # Import from pygit2 32 | from .ffi import ffi 33 | 34 | 35 | def to_bytes(s, encoding='utf-8', errors='strict'): 36 | if s == ffi.NULL or s is None: 37 | return ffi.NULL 38 | 39 | if isinstance(s, unicode): 40 | encoding = encoding or 'utf-8' 41 | return s.encode(encoding, errors) 42 | 43 | return s 44 | 45 | 46 | def is_string(s): 47 | return isinstance(s, basestring) 48 | 49 | 50 | def to_str(s): 51 | if type(s) is str: 52 | return s 53 | 54 | if type(s) is unicode: 55 | return s.encode() 56 | 57 | raise TypeError('unexpected type "%s"' % repr(s)) 58 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/py3.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | # Import from the future 29 | from __future__ import absolute_import 30 | 31 | # Import from pygit2 32 | from .ffi import ffi 33 | 34 | 35 | def to_bytes(s, encoding='utf-8', errors='strict'): 36 | if s == ffi.NULL or s is None: 37 | return ffi.NULL 38 | 39 | if isinstance(s, bytes): 40 | return s 41 | 42 | return s.encode(encoding, errors) 43 | 44 | 45 | def is_string(s): 46 | return isinstance(s, str) 47 | 48 | 49 | def to_str(s): 50 | if type(s) is str: 51 | return s 52 | 53 | if type(s) is bytes: 54 | return s.decode() 55 | 56 | raise TypeError('unexpected type "%s"' % repr(s)) 57 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/refspec.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | # Import from the future 29 | from __future__ import absolute_import 30 | 31 | # Import from pygit2 32 | from .errors import check_error 33 | from .ffi import ffi, C 34 | from .utils import to_bytes 35 | 36 | 37 | class Refspec(object): 38 | """The constructor is for internal use only""" 39 | def __init__(self, owner, ptr): 40 | self._owner = owner 41 | self._refspec = ptr 42 | 43 | @property 44 | def src(self): 45 | """Source or lhs of the refspec""" 46 | return ffi.string(C.git_refspec_src(self._refspec)).decode() 47 | 48 | @property 49 | def dst(self): 50 | """Destinaton or rhs of the refspec""" 51 | return ffi.string(C.git_refspec_dst(self._refspec)).decode() 52 | 53 | @property 54 | def force(self): 55 | """Whether this refspeca llows non-fast-forward updates""" 56 | return bool(C.git_refspec_force(self._refspec)) 57 | 58 | @property 59 | def string(self): 60 | """String which was used to create this refspec""" 61 | return ffi.string(C.git_refspec_string(self._refspec)).decode() 62 | 63 | @property 64 | def direction(self): 65 | """Direction of this refspec (fetch or push)""" 66 | return C.git_refspec_direction(self._refspec) 67 | 68 | def src_matches(self, ref): 69 | """Return True if the given string matches the source of this refspec, 70 | False otherwise. 71 | """ 72 | return bool(C.git_refspec_src_matches(self._refspec, to_bytes(ref))) 73 | 74 | def dst_matches(self, ref): 75 | """Return True if the given string matches the destination of this 76 | refspec, False otherwise.""" 77 | return bool(C.git_refspec_dst_matches(self._refspec, to_bytes(ref))) 78 | 79 | def _transform(self, ref, fn): 80 | buf = ffi.new('git_buf *', (ffi.NULL, 0)) 81 | err = fn(buf, self._refspec, to_bytes(ref)) 82 | check_error(err) 83 | 84 | try: 85 | return ffi.string(buf.ptr).decode() 86 | finally: 87 | C.git_buf_free(buf) 88 | 89 | def transform(self, ref): 90 | """Transform a reference name according to this refspec from the lhs to 91 | the rhs. Return an string. 92 | """ 93 | return self._transform(ref, C.git_refspec_transform) 94 | 95 | def rtransform(self, ref): 96 | """Transform a reference name according to this refspec from the lhs to 97 | the rhs. Return an string. 98 | """ 99 | return self._transform(ref, C.git_refspec_rtransform) 100 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | from _pygit2 import option 29 | from _pygit2 import GIT_OPT_GET_SEARCH_PATH, GIT_OPT_SET_SEARCH_PATH 30 | from _pygit2 import GIT_OPT_GET_MWINDOW_SIZE, GIT_OPT_SET_MWINDOW_SIZE 31 | 32 | 33 | class SearchPathList(object): 34 | 35 | def __getitem__(self, key): 36 | return option(GIT_OPT_GET_SEARCH_PATH, key) 37 | 38 | def __setitem__(self, key, value): 39 | option(GIT_OPT_SET_SEARCH_PATH, key, value) 40 | 41 | 42 | class Settings(object): 43 | """Library-wide settings""" 44 | 45 | __slots__ = [] 46 | 47 | _search_path = SearchPathList() 48 | 49 | @property 50 | def search_path(self): 51 | """Configuration file search path. 52 | 53 | This behaves like an array whose indices correspond to the 54 | GIT_CONFIG_LEVEL_* values. The local search path cannot be 55 | changed. 56 | """ 57 | return self._search_path 58 | 59 | @property 60 | def mwindow_size(self): 61 | """Maximum mmap window size""" 62 | return option(GIT_OPT_GET_MWINDOW_SIZE) 63 | 64 | @mwindow_size.setter 65 | def mwindow_size(self, value): 66 | option(GIT_OPT_SET_MWINDOW_SIZE, value) 67 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/submodule.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | 29 | # Import from the future 30 | from __future__ import absolute_import, unicode_literals 31 | 32 | from .errors import check_error 33 | from .ffi import ffi, C 34 | 35 | class Submodule(object): 36 | 37 | @classmethod 38 | def _from_c(cls, repo, cptr): 39 | subm = cls.__new__(cls) 40 | 41 | subm._repo = repo 42 | subm._subm = cptr 43 | 44 | return subm 45 | 46 | def __del__(self): 47 | C.git_submodule_free(self._subm) 48 | 49 | def open(self): 50 | """Open the repository for a submodule.""" 51 | crepo = ffi.new('git_repository **') 52 | err = C.git_submodule_open(crepo, self._subm) 53 | check_error(err) 54 | 55 | return self._repo._from_c(crepo[0], True) 56 | 57 | @property 58 | def name(self): 59 | """Name of the submodule.""" 60 | name = C.git_submodule_name(self._subm) 61 | return ffi.string(name).decode('utf-8') 62 | 63 | @property 64 | def path(self): 65 | """Path of the submodule.""" 66 | path = C.git_submodule_path(self._subm) 67 | return ffi.string(path).decode('utf-8') 68 | 69 | @property 70 | def url(self): 71 | """URL of the submodule.""" 72 | url = C.git_submodule_url(self._subm) 73 | return ffi.string(url).decode('utf-8') 74 | 75 | @property 76 | def branch(self): 77 | """Branch that is to be tracked by the submodule.""" 78 | branch = C.git_submodule_branch(self._subm) 79 | return ffi.string(branch).decode('utf-8') 80 | -------------------------------------------------------------------------------- /packaging/ci/lambda/GitPullS3/pygit2/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2010-2015 The pygit2 contributors 4 | # 5 | # This file is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License, version 2, 7 | # as published by the Free Software Foundation. 8 | # 9 | # In addition to the permissions in the GNU General Public License, 10 | # the authors give you unlimited permission to link the compiled 11 | # version of this file into combinations with other programs, 12 | # and to distribute those combinations without any restriction 13 | # coming from the use of this file. (The General Public License 14 | # restrictions do apply in other respects; for example, they cover 15 | # modification of the file, and distribution when not linked into 16 | # a combined executable.) 17 | # 18 | # This file is distributed in the hope that it will be useful, but 19 | # WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU General Public License 24 | # along with this program; see the file COPYING. If not, write to 25 | # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 | # Boston, MA 02110-1301, USA. 27 | 28 | # Import from the future 29 | from __future__ import absolute_import 30 | 31 | # Import from the Standard Library 32 | from sys import version_info 33 | 34 | # Import from pygit2 35 | from .ffi import ffi 36 | 37 | 38 | if version_info[0] < 3: 39 | from .py2 import is_string, to_bytes, to_str 40 | else: 41 | from .py3 import is_string, to_bytes, to_str 42 | 43 | 44 | 45 | def strarray_to_strings(arr): 46 | l = [None] * arr.count 47 | for i in range(arr.count): 48 | l[i] = ffi.string(arr.strings[i]).decode() 49 | 50 | return l 51 | 52 | 53 | class StrArray(object): 54 | """A git_strarray wrapper 55 | 56 | Use this in order to get a git_strarray* to pass to libgit2 out of a 57 | list of strings. This has a context manager, which you should use, e.g. 58 | 59 | with StrArray(list_of_strings) as arr: 60 | C.git_function_that_takes_strarray(arr) 61 | """ 62 | 63 | def __init__(self, l): 64 | # Allow passing in None as lg2 typically considers them the same as empty 65 | if l is None: 66 | self.array = ffi.NULL 67 | return 68 | 69 | if not isinstance(l, list): 70 | raise TypeError("Value must be a list") 71 | 72 | strings = [None] * len(l) 73 | for i in range(len(l)): 74 | if not is_string(l[i]): 75 | raise TypeError("Value must be a string") 76 | 77 | strings[i] = ffi.new('char []', to_bytes(l[i])) 78 | 79 | self._arr = ffi.new('char *[]', strings) 80 | self._strings = strings 81 | self.array = ffi.new('git_strarray *', [self._arr, len(strings)]) 82 | 83 | def __enter__(self): 84 | return self.array 85 | 86 | def __exit__(self, type, value, traceback): 87 | pass 88 | 89 | 90 | class GenericIterator(object): 91 | """Helper to easily implement an iterator. 92 | 93 | The constructor gets a container which must implement __len__ and 94 | __getitem__ 95 | """ 96 | 97 | def __init__(self, container): 98 | self.container = container 99 | self.length = len(container) 100 | self.idx = 0 101 | 102 | def next(self): 103 | return self.__next__() 104 | 105 | def __next__(self): 106 | idx = self.idx 107 | if idx >= self.length: 108 | raise StopIteration 109 | 110 | self.idx += 1 111 | return self.container[idx] 112 | -------------------------------------------------------------------------------- /packaging/ci/lambda/Makefile: -------------------------------------------------------------------------------- 1 | SUBDIRS = $(dir $(wildcard */Makefile)) 2 | 3 | .PHONY: subdirs $(SUBDIRS) 4 | 5 | subdirs: $(SUBDIRS) 6 | 7 | $(SUBDIRS): 8 | $(MAKE) -C $@ OUTDIR=$(CURDIR) 9 | 10 | clean: 11 | for subdir in $(SUBDIRS); do \ 12 | $(MAKE) -C $$subdir clean; \ 13 | done 14 | -------------------------------------------------------------------------------- /packaging/ci/lambda/MonitorGithubBuild/Makefile: -------------------------------------------------------------------------------- 1 | PYTHON ?= python2.7 2 | FUNCTION_NAME = $(notdir $(CURDIR)) 3 | OUTDIR ?= $(dir $(CURDIR)) 4 | ZIPFILE ?= $(OUTDIR)/$(FUNCTION_NAME).zip 5 | 6 | $(ZIPFILE): dist 7 | rm -f $@ 8 | pushd dist >/dev/null && \ 9 | zip -r $@ * && \ 10 | popd >/dev/null 11 | 12 | dist: *.py $(dir $(CURDIR))/lemur_ci/*.py 13 | rm -fr dist 14 | mkdir dist 15 | cp -a *.py $(dir $(CURDIR))/lemur_ci dist 16 | 17 | clean: 18 | rm -fr dist $(ZIPFILE) 19 | 20 | .PHONY: clean 21 | -------------------------------------------------------------------------------- /packaging/ci/lambda/MonitorGithubBuild/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import boto3 5 | import traceback 6 | import os 7 | 8 | from dateutil import parser 9 | from datetime import datetime as dt 10 | from dateutil.tz import tzlocal 11 | 12 | from lemur_ci import pipeline, commit_status 13 | 14 | s3 = boto3.client('s3') 15 | sns = boto3.client('sns') 16 | 17 | def lambda_handler(event, context): 18 | try: 19 | # for debugging 20 | print json.dumps(event) 21 | 22 | # Extract the Job 23 | job = pipeline.Job(event['CodePipeline.job']) 24 | 25 | token = None 26 | sourceStartTime = None 27 | sourceUrl = None 28 | sourceRevision = None 29 | if 'continuationToken' in job.data: 30 | token = json.loads(job.data['continuationToken']) 31 | sourceStartTime = parser.parse(token['sourceStartTime']) 32 | sourceUrl = token['sourceUrl'] 33 | sourceRevision = token['sourceRevision'] 34 | 35 | # Get the pipeline object for details about the pipeline 36 | pl = job.pipeline 37 | 38 | if 'continuationToken' not in job.data: 39 | # Get the pipeline config so that we can check the source object 40 | # metadata. 41 | # NB: This is somewhat racy -- If a new commit comes in before 42 | # we have a chance to do this, we'll get the wrong commit 43 | # metadata. There must be a better way to communicate 44 | # between pipeline stages! 45 | stages = pl.stages 46 | sourceConfig = stages['Source']['actions'][0]['configuration'] 47 | sourceMeta = s3.head_object(Bucket=sourceConfig['S3Bucket'], Key=sourceConfig['S3ObjectKey'])['Metadata'] 48 | sourceUrl = sourceMeta['source_html_url'] 49 | sourceRevision = sourceMeta['source_revision'] 50 | 51 | # Get the pipeline states in order to figure out where we are 52 | states = pl.states 53 | 54 | # NB: There can only be 1 source action in the source stage 55 | sourceAction = [s for s in states['Source']['actionStates'] if 'currentRevision' in s][0] 56 | if sourceStartTime is None and 'latestExecution' in sourceAction: 57 | sourceStartTime = sourceAction['latestExecution']['lastStatusChange'] 58 | 59 | buildAction = [s for s in states['Build']['actionStates'] if s['actionName'] == 'CodeBuild'][0] 60 | lastBuild = buildAction['latestExecution'] 61 | 62 | now = dt.now().replace(tzinfo=tzlocal()) 63 | state = 'pending' 64 | targetUrl = None 65 | if lastBuild['lastStatusChange'] > sourceStartTime: 66 | if 'externalExecutionUrl' in lastBuild: 67 | targetUrl = lastBuild['externalExecutionUrl'] 68 | if lastBuild['status'] == 'Succeeded': 69 | state = 'success' 70 | elif lastBuild['status'] == 'Failed': 71 | state = 'failure' 72 | elif lastBuild['status'] != 'InProgress': 73 | state = 'error' 74 | 75 | if state == 'pending' and 'continuationToken' in job.data: 76 | # We've already set the github status and now we're waiting 77 | # for the build to finish. 78 | return pipeline.continue_job_later(job.id, token, "Waiting for build to %s" % ("start" if lastBuild['status'] != 'InProgress' else "finish")) 79 | 80 | message = commit_status.Message( 81 | repoUrl=sourceUrl, 82 | sha=sourceRevision, 83 | state=state, 84 | context="lemur-ci/build", 85 | statusUrl=targetUrl 86 | ) 87 | print sns.publish( 88 | TopicArn=os.environ['STATUS_TOPIC_ARN'], 89 | Message=message.json 90 | ) 91 | if state == 'pending': 92 | token = dict( 93 | previous_job_id = job.id, 94 | sourceStartTime = sourceStartTime.isoformat(), 95 | sourceUrl = sourceUrl, 96 | sourceRevision = sourceRevision 97 | ) 98 | return pipeline.continue_job_later(job.id, token, "Waiting for build to start") 99 | pipeline.put_job_success(job.id, 'Notified %s/%s of success' % (sourceUrl, sourceRevision)) 100 | 101 | except Exception as e: 102 | # If any other exceptions which we didn't expect are raised 103 | # then fail the job and log the exception message. 104 | print('Function failed due to exception.') 105 | print(e) 106 | traceback.print_exc() 107 | pipeline.put_job_failure(job.id, 'Function exception: ' + str(e)) 108 | 109 | print('Function complete.') 110 | return "Complete." 111 | -------------------------------------------------------------------------------- /packaging/ci/lambda/MonitorGithubBuild/setup.cfg: -------------------------------------------------------------------------------- 1 | # just makes pip happy to install into a directory with no prefix 2 | [install] 3 | prefix= 4 | -------------------------------------------------------------------------------- /packaging/ci/lambda/NotifyGithub/Makefile: -------------------------------------------------------------------------------- 1 | PYTHON ?= python2.7 2 | PIP_PACKAGES ?= PyGithub 3 | FUNCTION_NAME = $(notdir $(CURDIR)) 4 | OUTDIR ?= $(dir $(CURDIR)) 5 | ZIPFILE ?= $(OUTDIR)/$(FUNCTION_NAME).zip 6 | 7 | $(ZIPFILE): dist 8 | rm -f $@ 9 | pushd dist >/dev/null && \ 10 | zip -r $@ * && \ 11 | popd >/dev/null 12 | 13 | dist: *.py $(dir $(CURDIR))/lemur_ci/*.py 14 | rm -fr dist 15 | mkdir dist 16 | cp -a *.py $(dir $(CURDIR))/lemur_ci dist 17 | for pkg in $(PIP_PACKAGES); do \ 18 | pip install --no-compile -t dist $$pkg; \ 19 | rm -fr dist/*/tests; \ 20 | done 21 | 22 | clean: 23 | rm -fr dist $(ZIPFILE) 24 | 25 | .PHONY: clean 26 | -------------------------------------------------------------------------------- /packaging/ci/lambda/NotifyGithub/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import boto3 5 | import traceback 6 | import re 7 | import os 8 | 9 | from base64 import b64decode 10 | 11 | from lemur_ci import commit_status 12 | from github import Github 13 | from github.GithubObject import NotSet 14 | 15 | 16 | def lambda_handler(event, context): 17 | try: 18 | # for debugging 19 | print json.dumps(event) 20 | 21 | message = commit_status.Message(**json.loads(event['Records'][0]['Sns']['Message'])) 22 | 23 | authToken = boto3.client('kms').decrypt(CiphertextBlob=b64decode(os.environ['GITHUB_TOKEN']))['Plaintext'] 24 | g = Github(authToken, base_url=message.api_url) 25 | u = g.get_user() 26 | repo = None 27 | if u.login == message.repoOrgOrUser: 28 | repo = u.get_repo(message.repo) 29 | else: 30 | o = g.get_organization(message.repoOrgOrUser) 31 | repo = o.get_repo(message.repo) 32 | 33 | commit = repo.get_commit(message.sha) 34 | print commit.create_status(**message.as_status()) 35 | 36 | except Exception as e: 37 | # If any other exceptions which we didn't expect are raised 38 | # then fail the job and log the exception message. 39 | print('Function failed due to exception.') 40 | print(e) 41 | traceback.print_exc() 42 | 43 | print('Function complete.') 44 | return "Complete." 45 | -------------------------------------------------------------------------------- /packaging/ci/lambda/NotifyGithub/setup.cfg: -------------------------------------------------------------------------------- 1 | # just makes pip happy to install into a directory with no prefix 2 | [install] 3 | prefix= 4 | -------------------------------------------------------------------------------- /packaging/ci/lambda/PublishRepoToBucket/Makefile: -------------------------------------------------------------------------------- 1 | PYTHON ?= python2.7 2 | FUNCTION_NAME = $(notdir $(CURDIR)) 3 | OUTDIR ?= $(dir $(CURDIR)) 4 | ZIPFILE ?= $(OUTDIR)/$(FUNCTION_NAME).zip 5 | 6 | $(ZIPFILE): dist 7 | rm -f $@ 8 | pushd dist >/dev/null && \ 9 | zip -r $@ * && \ 10 | popd >/dev/null 11 | 12 | dist: *.py $(dir $(CURDIR))/lemur_ci/*.py 13 | rm -fr dist 14 | mkdir dist 15 | cp -a *.py $(dir $(CURDIR))/lemur_ci dist 16 | 17 | clean: 18 | rm -fr dist $(ZIPFILE) 19 | 20 | .PHONY: clean 21 | -------------------------------------------------------------------------------- /packaging/ci/lambda/PublishRepoToBucket/lambda_function.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import re 4 | import json 5 | import boto3 6 | import traceback 7 | import tempfile 8 | import shutil 9 | import contextlib 10 | import zipfile 11 | import gzip 12 | import xml.etree.ElementTree as ET 13 | from os import path 14 | 15 | from lemur_ci import pipeline 16 | 17 | from botocore.client import Config 18 | from boto3 import Session, client 19 | 20 | @contextlib.contextmanager 21 | def tempdir(): 22 | dirpath = tempfile.mkdtemp() 23 | def cleanup(): 24 | shutil.rmtree(dirpath) 25 | yield dirpath 26 | 27 | def lambda_handler(event, context): 28 | try: 29 | # for debugging 30 | print json.dumps(event) 31 | 32 | job = pipeline.Job(event['CodePipeline.job']) 33 | artifact = job.data['inputArtifacts'][0] 34 | config = job.data['actionConfiguration']['configuration'] 35 | creds = job.data['artifactCredentials'] 36 | from_bucket = artifact['location']['s3Location']['bucketName'] 37 | from_key = artifact['location']['s3Location']['objectKey'] 38 | to_bucket = config['UserParameters'] 39 | 40 | session = Session(aws_access_key_id=creds['accessKeyId'], 41 | aws_secret_access_key=creds['secretAccessKey'], 42 | aws_session_token=creds['sessionToken']) 43 | s3 = session.client('s3', config=Config(signature_version='s3v4')) 44 | 45 | keyPrefix = 'devel' 46 | version = 'UNKNOWN' 47 | zipMembers = [] 48 | with tempdir() as td: 49 | with tempfile.NamedTemporaryFile() as tf: 50 | s3.download_file(from_bucket, from_key, tf.name) 51 | with zipfile.ZipFile(tf.name, 'r') as zf: 52 | zf.extractall(td) 53 | zipMembers = zf.namelist() 54 | # TODO: Figure out how to avoid double-wrapping this 55 | if 'repo.zip' in zipMembers: 56 | with zipfile.ZipFile(path.join(td, 'repo.zip'), 'r') as zf: 57 | zf.extractall(td) 58 | zipMembers = zf.namelist() 59 | 60 | # extract the RPM version 61 | r = ET.parse(path.join(td, 'repodata/repomd.xml')).getroot() 62 | pf = r.find(".//*[@type='primary']/{http://linux.duke.edu/metadata/repo}location").attrib['href'] 63 | with gzip.open(path.join(td, pf)) as gz: 64 | pr = ET.parse(gz).getroot() 65 | version = pr.find('.//{http://linux.duke.edu/metadata/common}package/{http://linux.duke.edu/metadata/common}version').attrib['ver'] 66 | 67 | if re.match(r'^\d+\.\d+\.\d+$', version): 68 | keyPrefix = 'release' 69 | 70 | # Get a new s3 client to use IAM Role 71 | s3 = client('s3') 72 | for fileName in zipMembers: 73 | if path.isdir(path.join(td, fileName)): 74 | continue 75 | s3.upload_file(path.join(td, fileName), to_bucket, path.join(keyPrefix, version, fileName)) 76 | 77 | pipeline.put_job_success(job.id, "Published repo") 78 | 79 | except Exception as e: 80 | # If any other exceptions which we didn't expect are raised 81 | # then fail the job and log the exception message. 82 | print('Function failed due to exception.') 83 | print(e) 84 | traceback.print_exc() 85 | pipeline.put_job_failure(job.id, 'Function exception: ' + str(e)) 86 | 87 | print('Function complete.') 88 | return "Complete." 89 | -------------------------------------------------------------------------------- /packaging/ci/lambda/PublishRepoToBucket/setup.cfg: -------------------------------------------------------------------------------- 1 | # just makes pip happy to install into a directory with no prefix 2 | [install] 3 | prefix= 4 | -------------------------------------------------------------------------------- /packaging/ci/lambda/lemur_ci/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edwardsp/lemur/de15cc61af28e78b8ca18dbc7fc6be1fe30cdde1/packaging/ci/lambda/lemur_ci/__init__.py -------------------------------------------------------------------------------- /packaging/ci/lambda/lemur_ci/commit_status.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | 4 | 5 | class Message: 6 | def __init__(self, repoUrl, sha, state, statusUrl=None, context=None, description=None): 7 | self.repoUrl = repoUrl 8 | self.sha = sha 9 | self.state = state 10 | self.statusUrl = statusUrl 11 | self.context = context 12 | self.description = description 13 | 14 | (self.repoScheme, self.repoHost, self.repoOrgOrUser, self.repo) = re.match(r'^(https?://)([^/]+)/([^/]+)/([^/]+)$', self.repoUrl).groups() 15 | 16 | @property 17 | def api_url(self): 18 | return self.repoScheme+'api.'+self.repoHost 19 | 20 | @property 21 | def json(self): 22 | return json.dumps(self.as_kwargs()) 23 | 24 | def as_kwargs(self): 25 | return dict((k,v) for k,v in dict( 26 | repoUrl=self.repoUrl, 27 | sha=self.sha, 28 | state=self.state, 29 | statusUrl=self.statusUrl, 30 | context=self.context, 31 | description=self.description 32 | ).iteritems() if v is not None) 33 | 34 | def as_status(self): 35 | status = self.as_kwargs() 36 | if 'statusUrl' in status: 37 | status['target_url'] = status['statusUrl'] 38 | del status['statusUrl'] 39 | del status['repoUrl'] 40 | del status['sha'] 41 | return status 42 | -------------------------------------------------------------------------------- /packaging/ci/lambda/lemur_ci/pipeline.py: -------------------------------------------------------------------------------- 1 | import json 2 | import boto3 3 | 4 | client = boto3.client('codepipeline') 5 | 6 | class Pipeline: 7 | def __init__(self, name): 8 | self.name = name 9 | 10 | @property 11 | def stages(self): 12 | return dict([(s['name'], s) for s in client.get_pipeline(name=self.name)['pipeline']['stages']]) 13 | 14 | @property 15 | def states(self): 16 | return dict((s['stageName'], s) for s in client.get_pipeline_state(name=self.name)['stageStates']) 17 | 18 | 19 | class Job: 20 | def __init__(self, val): 21 | if isinstance(val, dict): 22 | self.id = val['id'] 23 | self.data = val['data'] 24 | else: 25 | self.id = val 26 | self.data = {} 27 | 28 | @property 29 | def details(self): 30 | return client.get_job_details(jobId=self.id) 31 | 32 | @property 33 | def pipeline(self): 34 | return Pipeline(self.details['jobDetails']['data']['pipelineContext']['pipelineName']) 35 | 36 | 37 | def continue_job_later(job, token, message): 38 | """Notify CodePipeline of a continuing job 39 | 40 | This will cause CodePipeline to invoke the function again with the 41 | supplied continuation token. 42 | 43 | Args: 44 | job: The JobID 45 | message: A message to be logged relating to the job status 46 | continuation_token: The continuation token 47 | 48 | Raises: 49 | Exception: Any exception thrown by .put_job_success_result() 50 | 51 | """ 52 | 53 | # Use the continuation token to keep track of any job execution state 54 | # This data will be available when a new job is scheduled to continue the current execution 55 | continuation_token = json.dumps(token) 56 | 57 | print('Putting job continuation (%s)' % token) 58 | print(message) 59 | client.put_job_success_result(jobId=job, continuationToken=continuation_token) 60 | 61 | def put_job_success(job, message): 62 | """Notify CodePipeline of a successful job 63 | 64 | Args: 65 | job: The CodePipeline job ID 66 | message: A message to be logged relating to the job status 67 | 68 | Raises: 69 | Exception: Any exception thrown by .put_job_success_result() 70 | 71 | """ 72 | print('Putting job success') 73 | print(message) 74 | client.put_job_success_result(jobId=job) 75 | 76 | def put_job_failure(job, message): 77 | """Notify CodePipeline of a failed job 78 | 79 | Args: 80 | job: The CodePipeline job ID 81 | message: A message to be logged relating to the job status 82 | 83 | Raises: 84 | Exception: Any exception thrown by .put_job_failure_result() 85 | 86 | """ 87 | print('Putting job failure') 88 | print(message) 89 | client.put_job_failure_result(jobId=job, failureDetails={'message': message, 'type': 'JobFailed'}) 90 | 91 | def get_user_params(job_data, required_params=[]): 92 | """Decodes the JSON user parameters and validates the required properties. 93 | 94 | Args: 95 | job_data: The job data structure containing the UserParameters string which should be a valid JSON structure 96 | 97 | Returns: 98 | The JSON parameters decoded as a dictionary. 99 | 100 | Raises: 101 | Exception: The JSON can't be decoded or a property is missing. 102 | 103 | """ 104 | try: 105 | # Get the user parameters which contain the stack, artifact and file settings 106 | user_parameters = job_data['actionConfiguration']['configuration']['UserParameters'] 107 | decoded_parameters = json.loads(user_parameters) 108 | 109 | except Exception as e: 110 | # We're expecting the user parameters to be encoded as JSON 111 | # so we can pass multiple values. If the JSON can't be decoded 112 | # then fail the job with a helpful message. 113 | if user_parameters != "": 114 | raise Exception('UserParameters could not be decoded as JSON') 115 | 116 | for param in required_params: 117 | if param not in decoded_parameters: 118 | raise Exception('Your UserParameters JSON must include "%s"' % param) 119 | 120 | return decoded_parameters 121 | -------------------------------------------------------------------------------- /packaging/docker/Makefile: -------------------------------------------------------------------------------- 1 | PROXY_VARS := HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY http_proxy https_proxy ftp_proxy no_proxy 2 | BUILD_VARS := $(shell bv=; for pvar in $(PROXY_VARS); do if [ x"$${!pvar}" == "x" ]; then continue; fi; bv="$$bv --build-arg=$$pvar=$${!pvar}"; done; echo $$bv) 3 | SUBDIRS := go-el7 buildonly-lustre-client lemur-rpm-build 4 | 5 | subdirs: $(SUBDIRS) 6 | 7 | $(SUBDIRS): 8 | @BUILD_VARS="$(BUILD_VARS)" make -C $@ 9 | 10 | host-kernel: go-el7 11 | buildonly-lustre-client: go-el7 12 | native-lustre-client: host-kernel 13 | lemur-rpm-build: buildonly-lustre-client 14 | 15 | .PHONY: $(SUBDIRS) subdirs 16 | -------------------------------------------------------------------------------- /packaging/docker/README.md: -------------------------------------------------------------------------------- 1 | # Docker Images 2 | 3 | In this directory are several recipes for creating Docker images. The go-el7 image is generally useful as a base EL7 image with the latest go packages installed (at time of writing, go 1.7.1-1 is latest). After running `make -C go-el7`, an image with the tags go-el7:latest and go-el7:(version) will be created. 4 | 5 | ### go-el7 6 | * Creates an EL7 image with the most recent version of go packaged by the Fedora project 7 | * Image tags: go-el7:latest, go-el7:(version) 8 | 9 | ### host-kernel 10 | * Delegates to linux-host-kernel or mac-host-kernel, depending on host platform 11 | 12 | ### mac-host-kernel 13 | * Builds on the latest go-el7 image to create a kernel-devel image suitable for building and installing a Lustre client (on Mac, the targeted Linux kernel is the Docker-provided Moby kernel) 14 | * Prerequisite image(s): go-el7:latest 15 | * Image tags: mac-host-kernel:latest, mac-host-kernel:(moby version), host-kernel:latest 16 | 17 | ### linux-host-kernel 18 | * Builds on the latest go-el7 image to create a kernel-devel image suitable for building and installing a Lustre client 19 | * Prerequisite image(s): go-el7:latest 20 | * Image tags: linux-host-kernel:latest, linux-host-kernel:(uname -r), host-kernel:latest 21 | 22 | ### native-lustre-client 23 | * Builds on the host-kernel image to create a lustre-client image suitable for building against liblustreapi and/or mounting a Lustre client from within a container (pulls from latest successful master build on jenkins) 24 | * Prerequisite image(s): go-el7:latest, host-kernel:latest 25 | * Image tags: native-lustre-client:latest, native-lustre-client:(lustre version), lustre-client:latest 26 | 27 | ### buildonly-lustre-client 28 | * Simple image which just installs a downloaded lustre-client RPM without attempting to match it with the host kernel -- only useful for builds against liblustreapi 29 | * Prerequisite image(s): go-el7:latest 30 | * Image tags: buildonly-lustre-client:latest, buildonly-lustre-client:(lustre version), lustre-client:latest 31 | 32 | ### lemur-rpm-build 33 | * Image which can be used to produce lemur RPMs from the source tree 34 | * Prerequisite image(s): go-el7:latest, lustre-client:latest 35 | * Image tags: lemur-rpm-build:latest, lemur-rpm-build:(version) -------------------------------------------------------------------------------- /packaging/docker/buildonly-lustre-client/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM go-el7:latest 2 | MAINTAINER Michael MacDonald 3 | 4 | env REPO_NAME lustre-client 5 | 6 | ARG package_url 7 | 8 | RUN echo -e "[${REPO_NAME}]\nname=${REPO_NAME}\ngpgcheck=0\nbaseurl=${package_url}\n" | sed -e 's/,/%2C/g' > /etc/yum.repos.d/${REPO_NAME}.repo \ 9 | && unset no_proxy NO_PROXY \ 10 | && yum install -y lustre-client 11 | -------------------------------------------------------------------------------- /packaging/docker/buildonly-lustre-client/Makefile: -------------------------------------------------------------------------------- 1 | 2 | REPO ?= $(notdir $(CURDIR)) 3 | 4 | BUILDER_URL ?= https://build.hpdd.intel.com 5 | LUSTRE_JOB ?= lustre-b2_9 6 | LUSTRE_BUILD ?= lastSuccessfulBuild 7 | CLIENT_PACKAGE ?= lustre-client 8 | 9 | PACKAGE_URL := $(BUILDER_URL)/job/$(LUSTRE_JOB)/arch=x86_64,build_type=client,distro=el7,ib_stack=inkernel/$(LUSTRE_BUILD) 10 | CLIENT_VERSION := $(shell curl -sf $(PACKAGE_URL)/api/json | python -c 'import sys, json, re; pkg=[a for a in json.load(sys.stdin)["artifacts"] if re.match(r"^$(CLIENT_PACKAGE)-\d+.*\.rpm", a["fileName"])][0]["fileName"]; print(re.sub(r"$(CLIENT_PACKAGE)-(.*)\.x86_64(\.x86_64)?\.rpm",r"\1",pkg))') 11 | IMAGE := $(shell latest=$$(docker images | awk "/$(REPO).*$(CLIENT_VERSION)/ {print \$$2}"); if [ "$$latest" == $(CLIENT_VERSION) ]; then true; else echo $(REPO)/$(CLIENT_VERSION); fi) 12 | 13 | $(CLIENT_VERSION): $(IMAGE) 14 | 15 | $(IMAGE): Dockerfile 16 | @echo "Building $(IMAGE) for $(CLIENT_VERSION)" 17 | docker build -t $(subst /,:,$(IMAGE)) -t $(REPO):latest -t lustre-client:latest --build-arg=package_url=$(PACKAGE_URL)/artifact/artifacts/ $(BUILD_VARS) . 18 | 19 | clean: 20 | docker rmi $(subst /,:,$(IMAGE)) $(REPO):latest lustre-client:latest 21 | 22 | .PHONY: $(CLIENT_VERSION) $(IMAGE) 23 | -------------------------------------------------------------------------------- /packaging/docker/go-el7/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | MAINTAINER Robert Read 3 | 4 | # Setup go build environment 5 | RUN yum install -y @development golang pcre-devel glibc-static which 6 | 7 | RUN mkdir -p /go/src /go/bin && chmod -R 777 /go 8 | ENV GOPATH=/go \ 9 | PATH=$GOPATH/bin:$PATH 10 | 11 | RUN go get github.com/tools/godep && cp /go/bin/godep /usr/local/bin 12 | 13 | ARG go_version 14 | ARG go_macros_version 15 | 16 | # Bootstrap a golang RPM build from Fedora Rawhide, but disable tests because they require privileged mode 17 | RUN rpm -ivh http://mirrors.kernel.org/fedora/development/rawhide/Everything/x86_64/os/Packages/g/go-srpm-macros-${go_macros_version}.noarch.rpm \ 18 | && ln -s /usr/lib/rpm/macros.d/macros.go-srpm /etc/rpm/ \ 19 | && rpmbuild --define '%check exit 0' --rebuild http://mirrors.kernel.org/fedora/development/rawhide/Everything/source/tree/Packages/g/golang-${go_version}.src.rpm \ 20 | && cd /root/rpmbuild/RPMS/x86_64 && rpm -Uvh golang-*.rpm ../noarch/golang-src-*.rpm 21 | -------------------------------------------------------------------------------- /packaging/docker/go-el7/Makefile: -------------------------------------------------------------------------------- 1 | FEDORA_GOLANG ?= $(shell curl -s http://mirrors.kernel.org/fedora/development/rawhide/Everything/source/tree/Packages/g/ | awk '/>golang-1.*\.src\.rpmgo-srpm-macros.*\.noarch\.rpm 3 | 4 | RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm 5 | 6 | RUN yum install -y sudo yum-utils 7 | 8 | RUN sed -i -e "s/^\(Defaults\s\+requiretty.*\)/#\1/" /etc/sudoers 9 | 10 | ADD ./lemur.spec /tmp/lemur.spec 11 | 12 | # prep the image with some build deps, but this will be run again 13 | # for the actual build to catch any changes since the image was built 14 | RUN yum-builddep -y /tmp/lemur.spec && rm /tmp/lemur.spec 15 | 16 | VOLUME ["/source", "/root/rpmbuild"] 17 | CMD make -C /source local-rpm 18 | -------------------------------------------------------------------------------- /packaging/docker/lemur-rpm-build/Makefile: -------------------------------------------------------------------------------- 1 | REPO ?= $(notdir $(CURDIR)) 2 | 3 | VERSION := 1 4 | IMAGE := $(shell latest=$$(docker images | awk "/$(REPO).*$(VERSION)/ {print \$$2}"); if [ "$$latest" == $(VERSION) ]; then true; else echo $(REPO)/$(VERSION); fi) 5 | 6 | $(VERSION): $(IMAGE) 7 | 8 | $(IMAGE): Dockerfile lemur.spec 9 | @echo "Building $(IMAGE)" 10 | docker build -t $(subst /,:,$(IMAGE)) -t $(REPO):latest $(BUILD_VARS) . 11 | 12 | lemur.spec: ../../rpm/lemur.spec 13 | cp -a ../../rpm/lemur.spec . 14 | 15 | clean: 16 | rm -f lemur.spec 17 | docker rmi $(REPO):latest $(subst /,:,$(IMAGE)) 18 | 19 | .PHONY: $(VERSION) $(IMAGE) 20 | -------------------------------------------------------------------------------- /packaging/docker/linux-host-kernel/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM go-el7:latest 2 | MAINTAINER Michael MacDonald 3 | 4 | RUN yum install -y kernel-devel 5 | -------------------------------------------------------------------------------- /packaging/docker/linux-host-kernel/Makefile: -------------------------------------------------------------------------------- 1 | REPO ?= $(notdir $(CURDIR)) 2 | 3 | HOST_KERNEL_VERSION := $(shell docker run --rm go-el7 uname -r) 4 | IMAGE := $(shell latest=$$(docker images | awk "/$(REPO).*$(HOST_KERNEL_VERSION)/ {print \$$2}"); if [ "$$latest" == $(HOST_KERNEL_VERSION) ]; then true; else echo $(REPO)/$(HOST_KERNEL_VERSION); fi) 5 | 6 | $(HOST_KERNEL_VERSION): $(IMAGE) 7 | 8 | $(IMAGE): Dockerfile 9 | @echo "Building $(IMAGE) for $(HOST_KERNEL_VERSION)" 10 | docker build -t $(subst /,:,$(IMAGE)) -t $(REPO):latest -t host-kernel:latest . 11 | 12 | clean: 13 | docker rmi $(subst /,:,$(IMAGE)) $(REPO):latest host-kernel:latest 14 | -------------------------------------------------------------------------------- /packaging/docker/mac-host-kernel/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM go-el7:latest 2 | MAINTAINER Michael MacDonald 3 | 4 | # Grab host kernel source and prepare symlinks. 5 | RUN export KERNEL_VERSION=$(uname -r | cut -d '-' -f 1) \ 6 | && mkdir -p /usr/src/kernels \ 7 | && curl -L https://www.kernel.org/pub/linux/kernel/v${KERNEL_VERSION%%.*}.x/linux-$KERNEL_VERSION.tar.xz | tar -C /usr/src/kernels -xJ \ 8 | && mv /usr/src/kernels/linux-$KERNEL_VERSION /usr/src/kernels/$KERNEL_VERSION \ 9 | && mkdir -p /lib/modules/$(uname -r) \ 10 | && ln -sf /usr/src/kernels/$KERNEL_VERSION /lib/modules/$(uname -r)/build \ 11 | && ln -sf build /lib/modules/$(uname -r)/source 12 | 13 | RUN yum install -y bc 14 | 15 | # Set up host kernel source for building DKMS client. 16 | # Notes: 17 | # 1) We have to pretend that it's a RHEL kernel in order to make DKMS happy 18 | RUN export KERNEL_VERSION=$(uname -r | cut -d '-' -f 1) \ 19 | && yum install -y bc \ 20 | && RHEL_RELEASE=($(awk '{gsub(/\./, " ", $4); print $4}' /etc/redhat-release)) \ 21 | && cd /usr/src/kernels/$KERNEL_VERSION \ 22 | && zcat /proc/1/root/proc/config.gz > .config \ 23 | && make modules_prepare \ 24 | && echo -e "#define RHEL_MAJOR ${RHEL_RELEASE[0]}\n#define RHEL_MINOR ${RHEL_RELEASE[1]}\n#define RHEL_RELEASE \"${RHEL_RELEASE[0]}.${RHEL_RELEASE[1]}.${RHEL_RELEASE[2]}\"\n" >> include/generated/uapi/linux/version.h 25 | -------------------------------------------------------------------------------- /packaging/docker/mac-host-kernel/Makefile: -------------------------------------------------------------------------------- 1 | REPO ?= $(notdir $(CURDIR)) 2 | 3 | HOST_KERNEL_VERSION := $(shell docker run --rm go-el7 uname -r) 4 | IMAGE := $(shell latest=$$(docker images | awk "/$(REPO).*$(HOST_KERNEL_VERSION)/ {print \$$2}"); if [ "$$latest" == $(HOST_KERNEL_VERSION) ]; then true; else echo $(REPO)/$(HOST_KERNEL_VERSION); fi) 5 | 6 | $(HOST_KERNEL_VERSION): $(IMAGE) 7 | 8 | $(IMAGE): Dockerfile 9 | @echo "Building $(IMAGE) for $(HOST_KERNEL_VERSION)" 10 | docker build -t $(subst /,:,$(IMAGE)) -t $(REPO):latest -t host-kernel:latest $(BUILD_VARS) . 11 | 12 | clean: 13 | docker rmi $(subst /,:,$(IMAGE)) $(REPO):latest host-kernel:latest 14 | -------------------------------------------------------------------------------- /packaging/docker/native-lustre-client/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM host-kernel:latest 2 | MAINTAINER Michael MacDonald 3 | 4 | env REPO_NAME lustre-client 5 | 6 | ARG package_url 7 | 8 | RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm 9 | 10 | # Now, build the client modules and install the userspace utils 11 | RUN echo -e "[${REPO_NAME}]\nname=${REPO_NAME}\ngpgcheck=0\nbaseurl=${package_url}\n" | sed -e 's/,/%2C/g' > /etc/yum.repos.d/${REPO_NAME}.repo \ 12 | && unset no_proxy NO_PROXY \ 13 | && yum install -y lustre-client-dkms lustre-client 14 | -------------------------------------------------------------------------------- /packaging/docker/native-lustre-client/Makefile: -------------------------------------------------------------------------------- 1 | 2 | REPO ?= $(notdir $(CURDIR)) 3 | 4 | BUILDER_URL ?= https://build.hpdd.intel.com 5 | LUSTRE_JOB ?= lustre-master 6 | LUSTRE_BUILD ?= lastSuccessfulBuild 7 | CLIENT_PACKAGE ?= lustre-client-dkms 8 | 9 | PACKAGE_URL = $(BUILDER_URL)/job/$(LUSTRE_JOB)/arch=x86_64,build_type=client,distro=el7,ib_stack=inkernel/$(LUSTRE_BUILD) 10 | CLIENT_VERSION = $(shell curl -sf $(PACKAGE_URL)/api/json | python -c 'import sys, json, re; pkg=[a for a in json.load(sys.stdin)["artifacts"] if "$(CLIENT_PACKAGE)" in a["fileName"]][0]["fileName"]; print re.sub(r"$(CLIENT_PACKAGE)-(.*)\.noarch\.rpm",r"\1",pkg)') 11 | IMAGE = $(shell latest=$$(docker images | awk "/$(REPO).*$(CLIENT_VERSION)/ {print \$$2}"); if [ "$$latest" == $(CLIENT_VERSION) ]; then true; else echo $(REPO)/$(CLIENT_VERSION); fi) 12 | 13 | $(CLIENT_VERSION): $(IMAGE) 14 | 15 | $(IMAGE): Dockerfile 16 | @echo "Building $(IMAGE) for $(CLIENT_VERSION)" 17 | docker build -t $(subst /,:,$(IMAGE)) -t $(REPO):latest --build-arg=package_url=$(PACKAGE_URL)/artifact/artifacts/ $(BUILD_VARS) . 18 | 19 | clean: 20 | docker rmi $(subst /,:,$(IMAGE)) $(REPO):latest lustre-client:latest 21 | 22 | .PHONY: $(CLIENT_VERSION) $(IMAGE) 23 | -------------------------------------------------------------------------------- /packaging/rpm/Makefile: -------------------------------------------------------------------------------- 1 | SOURCE := $(NAME)-$(VERSION).tar.gz 2 | TOPDIR ?= $(shell rpm --eval '%_topdir') 3 | WORKDIR := $(shell mktemp -d) 4 | SPECFILE := $(NAME).spec 5 | INITFILES := $(patsubst %,$(WORKDIR)/%,lhsmd.service lhsmd.conf) 6 | USER ?= $(shell whoami) 7 | YBLDDEP := $(shell if which yum-builddep 2>/dev/null; then true; else echo yum-builddep; fi) 8 | CREATEREPO := $(shell if which createrepo 2>/dev/null; then true; else echo createrepo; fi) 9 | 10 | rpm: $(WORKDIR)/$(SOURCE) $(INITFILES) $(WORKDIR)/$(SPECFILE) $(WORKDIR)/.deps 11 | cd $(WORKDIR) && \ 12 | rpmbuild --define '%_sourcedir $(WORKDIR)' \ 13 | --define '%_gitver $(VERSION)' \ 14 | --define '%_topdir $(TOPDIR)' \ 15 | --define '%PACKAGE_PREFIX $(NAME)' \ 16 | --define '%dist $(RELEASE)' \ 17 | -ba $(SPECFILE) && \ 18 | rm -fr $(WORKDIR) 19 | # clean up any cruft left by the go compiler 20 | rm -fr $$(rpm --eval '%{_builddir}')/$(NAME)-* 21 | 22 | repo: $(CREATEREPO) 23 | cd $(TOPDIR)/RPMS && \ 24 | createrepo -v -p . 25 | 26 | 27 | $(WORKDIR)/.deps: $(YBLDDEP) $(WORKDIR)/$(SPECFILE) 28 | sudo yum-builddep -y $(WORKDIR)/$(SPECFILE) && touch $(WORKDIR)/.deps 29 | 30 | $(YBLDDEP): 31 | sudo yum install -y yum-utils 32 | 33 | $(CREATEREPO): 34 | sudo yum install -y createrepo 35 | 36 | $(WORKDIR)/$(SPECFILE): 37 | cp $(SPECFILE) $(WORKDIR) && chown $(USER).$(USER) $(WORKDIR)/$(SPECFILE) 38 | 39 | $(WORKDIR)/$(SOURCE): 40 | cd ../../ && \ 41 | tar --owner=$(USER) --group=$(USER) \ 42 | --exclude=.git --exclude=*.swp --exclude=packaging/ci/* \ 43 | --transform 's|./|$(NAME)-$(VERSION)/|' -czf $@ ./ && \ 44 | chown $(USER).$(USER) $@ 45 | 46 | $(INITFILES): 47 | cp $(notdir $@) $@ 48 | -------------------------------------------------------------------------------- /packaging/rpm/lemur.spec: -------------------------------------------------------------------------------- 1 | %global debug_package %{nil} 2 | %define pkg_prefix %{?PACKAGE_PREFIX}%{!?PACKAGE_PREFIX:lemur} 3 | %define plugin_dir %{?PLUGIN_DIR}%{!?PLUGIN_DIR:%{_libexecdir}/lhsmd} 4 | 5 | Name: %{pkg_prefix}-azure-hsm-agent 6 | Version: %{?_gitver}%{!?_gitver:0.0.1} 7 | Release: %{?dist}%{!?dist:1} 8 | 9 | Vendor: Intel Corporation 10 | Source: %{pkg_prefix}-%{version}.tar.gz 11 | Source1: lhsmd.conf 12 | Source2: lhsmd.service 13 | License: GPLv2 14 | Summary: Lustre HSM Tools - Lustre HSM Agent 15 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root 16 | 17 | Requires: lustre-client >= %{?MIN_LUSTRE_VERSION}%{?!MIN_LUSTRE_VERSION:2.12.0} 18 | %{?systemd_requires} 19 | 20 | %description 21 | The Lustre HSM Agent provides a backend-agnostic HSM Agent for brokering 22 | communications between a Lustre filesystem's HSM coordinator and 23 | backend-specific data movers. 24 | 25 | %package -n %{pkg_prefix}-azure-data-movers 26 | Summary: Lustre HSM Tools - HSM Data Movers 27 | License: Apache 28 | Requires: %{pkg_prefix}-azure-hsm-agent = %{version} 29 | 30 | %description -n %{pkg_prefix}-azure-data-movers 31 | These data movers are designed to implement the Lustre HSM Agent's data 32 | movement protocol. When associated with an HSM archive number, a data 33 | mover fulfills data movement requests on behalf of the HSM Agent. 34 | 35 | %prep 36 | 37 | %setup -n %{pkg_prefix}-%{version} 38 | # ohhh myyyy... 39 | cd .. 40 | mkdir -p src/github.com/edwardsp 41 | mv %{pkg_prefix}-%{version} src/github.com/edwardsp/%{pkg_prefix} 42 | mkdir %{pkg_prefix}-%{version} 43 | mv src %{pkg_prefix}-%{version} 44 | 45 | %install 46 | export GOPATH=$PWD:$GOPATH 47 | cd src/github.com/edwardsp/%{pkg_prefix} 48 | %{__make} install PREFIX=%{buildroot}/%{_prefix} 49 | %{__make} install-example PREFIX=%{buildroot}/ 50 | 51 | # move datamover plugins to plugin dir 52 | install -d %{buildroot}%{plugin_dir} 53 | for plugin in %{buildroot}/%{_bindir}/lhsm-plugin-*; do 54 | mv $plugin %{buildroot}/%{plugin_dir}/$(basename $plugin) 55 | done 56 | 57 | # move lhsmd to /sbin 58 | install -d %{buildroot}%{_sbindir} 59 | mv %{buildroot}/%{_bindir}/lhsmd %{buildroot}/%{_sbindir} 60 | mv %{buildroot}/%{_bindir}/azure-import %{buildroot}/%{_sbindir} 61 | mv %{buildroot}/%{_bindir}/azure-list-versions %{buildroot}/%{_sbindir} 62 | mv %{buildroot}/%{_bindir}/changelog-reader %{buildroot}/%{_sbindir} 63 | 64 | %if 0%{?el6} 65 | install -m 700 -d %{buildroot}/%{_localstatedir}/run/lhsmd 66 | install -d %{buildroot}%{_sysconfdir}/init 67 | install -p -m 0644 %SOURCE1 %{buildroot}%{_sysconfdir}/init/lhsmd.conf 68 | %endif 69 | 70 | %if 0%{?el7} 71 | install -d %{buildroot}%{_unitdir} 72 | install -p -m 0644 %SOURCE2 %{buildroot}%{_unitdir}/lhsmd.service 73 | %endif 74 | 75 | %post 76 | %if 0%{?el7} 77 | %systemd_post lhsmd.service 78 | %endif 79 | 80 | %preun 81 | %if 0%{?el7} 82 | %systemd_preun lhsmd.service 83 | %endif 84 | 85 | %postun 86 | %if 0%{?el7} 87 | %systemd_postun_with_restart lhsmd.service 88 | %endif 89 | 90 | %files 91 | %defattr(-,root,root) 92 | %{_sbindir}/lhsmd 93 | %{_sbindir}/azure-import 94 | %{_sbindir}/azure-list-versions 95 | %{_sbindir}/changelog-reader 96 | %{_sysconfdir}/lhsmd/agent.example 97 | %if 0%{?el6} 98 | %config %{_sysconfdir}/init/lhsmd.conf 99 | %{_localstatedir}/run/lhsmd 100 | %endif 101 | %if 0%{?el7} 102 | %{_unitdir}/lhsmd.service 103 | %endif 104 | 105 | %files -n %{pkg_prefix}-azure-data-movers 106 | %defattr(-,root,root) 107 | %{plugin_dir}/lhsm-plugin-posix 108 | %{plugin_dir}/lhsm-plugin-az 109 | %{_sysconfdir}/lhsmd/lhsm-plugin-posix.example 110 | 111 | -------------------------------------------------------------------------------- /packaging/rpm/lhsmd.conf: -------------------------------------------------------------------------------- 1 | description "Lustre HSM Agent" 2 | 3 | start on runlevel [23] 4 | stop on runlevel [S016] 5 | 6 | respawn 7 | 8 | # Hmm. Do we need a --logfile arg? 9 | exec /usr/sbin/lhsmd > /var/log/lhsmd.log 2>&1 10 | -------------------------------------------------------------------------------- /packaging/rpm/lhsmd.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Lustre HSM Agent 3 | Requires=network.target 4 | 5 | [Service] 6 | ExecStart=/usr/sbin/lhsmd 7 | Restart=on-failure 8 | User=root 9 | RuntimeDirectory=lhsmd 10 | RuntimeDirectoryMode=0700 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /pdm/Makefile: -------------------------------------------------------------------------------- 1 | 2 | pdm.pb.go: pdm.proto 3 | protoc -I . ./pdm.proto --go_out=plugins=grpc:. 4 | -------------------------------------------------------------------------------- /pdm/pdm.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package pdm; 4 | 5 | 6 | 7 | // Interface exported by the server. 8 | service DataMover { 9 | rpc Register(Endpoint) returns (Handle); 10 | rpc GetActions(Handle) returns (stream ActionItem); 11 | rpc StatusStream(stream ActionStatus) returns (Empty); 12 | } 13 | 14 | message Endpoint { 15 | string fs_url = 2; 16 | uint32 archive = 1; 17 | } 18 | 19 | message Handle { 20 | uint64 id = 1; 21 | } 22 | 23 | enum Command { 24 | NONE = 0; 25 | ARCHIVE = 1; 26 | RESTORE = 2; 27 | REMOVE = 3; 28 | CANCEL = 4; 29 | } 30 | 31 | message ActionItem { 32 | uint64 id = 1; // Unique indentifier for this action, must be used in status messages 33 | Command op = 2; 34 | string primary_path = 3; // Path to primary file (for metadata or reading) 35 | string write_path = 4; // Path for writing data (for restore) 36 | int64 offset = 5; // Start IO at offset 37 | int64 length = 6; // Number of bytes to copy 38 | bytes deprecated1 = 7; // Archive ID of file (provided with Restore command) DEPRECATED 39 | bytes data = 8; // Arbitrary data passed to action. Data Mover specific. 40 | string uuid = 9; // trusted.lhsm_uuid if set 41 | bytes hash = 10; // trusted.lhsm_hash if set 42 | string url = 12; // trusted.lhsm_url if set 43 | } 44 | 45 | message ActionStatus { 46 | uint64 id = 1; // Unique identifier for action 47 | bool completed = 2; // True if this is last update 48 | int32 error = 3; // Non-zero indicates an error 49 | int64 offset = 4; 50 | int64 length = 5; 51 | Handle handle = 6; 52 | bytes depcreated1 = 7; // Included with completion of Archive DEPRECATED 53 | int32 flags = 8; // Additial flags (used for errors only?) 54 | string uuid = 9; // Included with completion of Archive 55 | bytes hash = 10; // Included with completion of Archive 56 | string url = 11; // Included with completion of Archive 57 | } 58 | 59 | 60 | message Empty { }; 61 | -------------------------------------------------------------------------------- /pkg/checksum/checksum.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package checksum 6 | 7 | import ( 8 | "crypto/sha1" 9 | "hash" 10 | "io" 11 | "os" 12 | 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | type ( 17 | // Writer wraps an io.WriterAt and updates the checksum 18 | // with every write. 19 | Writer interface { 20 | io.Writer 21 | Sum() []byte 22 | } 23 | 24 | // Sha1HashWriter implements Writer and uses the SHA1 25 | // algorithm to calculate the file checksum 26 | Sha1HashWriter struct { 27 | dest io.Writer 28 | cksum hash.Hash 29 | } 30 | 31 | // NoopHashWriter implements Writer but doesn't 32 | // actually calculate a checksum 33 | NoopHashWriter struct { 34 | dest io.Writer 35 | } 36 | ) 37 | 38 | // NewSha1HashWriter returns a new Sha1HashWriter 39 | func NewSha1HashWriter(dest io.Writer) Writer { 40 | return &Sha1HashWriter{ 41 | dest: dest, 42 | cksum: sha1.New(), 43 | } 44 | } 45 | 46 | // Write updates the checksum and writes the byte slice at offset 47 | func (hw *Sha1HashWriter) Write(b []byte) (int, error) { 48 | _, err := hw.cksum.Write(b) 49 | if err != nil { 50 | return 0, errors.Wrap(err, "updating checksum failed") 51 | } 52 | return hw.dest.Write(b) 53 | } 54 | 55 | // Sum returns the checksum 56 | func (hw *Sha1HashWriter) Sum() []byte { 57 | return hw.cksum.Sum(nil) 58 | } 59 | 60 | // NewNoopHashWriter returns a new NoopHashWriter 61 | func NewNoopHashWriter(dest io.Writer) Writer { 62 | return &NoopHashWriter{ 63 | dest: dest, 64 | } 65 | } 66 | 67 | // WriteAt writes the byte slice at offset 68 | func (hw *NoopHashWriter) Write(b []byte) (int, error) { 69 | return hw.dest.Write(b) 70 | } 71 | 72 | // Sum returns a dummy checksum 73 | func (hw *NoopHashWriter) Sum() []byte { 74 | return []byte{} 75 | } 76 | 77 | // FileSha1Sum returns the SHA1 checksum for the supplied file path 78 | func FileSha1Sum(filePath string) ([]byte, error) { 79 | file, err := os.Open(filePath) 80 | if err != nil { 81 | return nil, errors.Wrapf(err, "Failed to open %s for checksum", filePath) 82 | } 83 | defer file.Close() 84 | 85 | hash := sha1.New() 86 | _, err = io.Copy(hash, file) 87 | if err != nil { 88 | return nil, errors.Wrapf(err, "Failed to compute checksum for %s") 89 | } 90 | 91 | return hash.Sum(nil), nil 92 | } 93 | -------------------------------------------------------------------------------- /pkg/fsroot/client.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package fsroot 6 | 7 | import ( 8 | "github.com/edwardsp/go-lustre/fs" 9 | "github.com/edwardsp/go-lustre/pkg/mntent" 10 | ) 11 | 12 | type ( 13 | // FsID is a Lustre filesystem ID 14 | FsID struct { 15 | val [2]int32 16 | } 17 | 18 | // Client defines an interface for Lustre filesystem clients 19 | Client interface { 20 | FsName() string 21 | Path() string 22 | Root() fs.RootDir 23 | } 24 | 25 | // FsClient is an implementation of the Client interface 26 | fsClient struct { 27 | root fs.RootDir 28 | fsName string 29 | fsID *FsID 30 | } 31 | ) 32 | 33 | func getFsName(mountPath string) (string, error) { 34 | entry, err := mntent.GetEntryByDir(mountPath) 35 | if err != nil { 36 | return "", err 37 | } 38 | return entry.Fsname, nil 39 | } 40 | 41 | // New returns a new Client 42 | func New(path string) (Client, error) { 43 | root, err := fs.MountRoot(path) 44 | if err != nil { 45 | return nil, err 46 | } 47 | name, err := getFsName(root.Path()) 48 | if err != nil { 49 | return nil, err 50 | } 51 | id, err := getFsID(path) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return &fsClient{root: root, 56 | fsName: name, 57 | fsID: id, 58 | }, nil 59 | } 60 | 61 | // FsName returns the filesystem name 62 | func (c *fsClient) FsName() string { 63 | return c.fsName 64 | } 65 | 66 | // Path returns the filesystem root path 67 | func (c *fsClient) Path() string { 68 | return c.root.Path() 69 | } 70 | 71 | // Root returns the underlying fs.RootDir item 72 | func (c *fsClient) Root() fs.RootDir { 73 | return c.root 74 | } 75 | -------------------------------------------------------------------------------- /pkg/fsroot/fsid_darwin.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package fsroot 6 | 7 | import "golang.org/x/sys/unix" 8 | 9 | func getFsID(mountPath string) (*FsID, error) { 10 | statfs := &unix.Statfs_t{} 11 | 12 | if err := unix.Statfs(mountPath, statfs); err != nil { 13 | return nil, err 14 | } 15 | var id FsID 16 | id.val[0] = statfs.Fsid.Val[0] 17 | id.val[1] = statfs.Fsid.Val[1] 18 | return &id, nil 19 | } 20 | -------------------------------------------------------------------------------- /pkg/fsroot/fsid_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package fsroot 6 | 7 | import "golang.org/x/sys/unix" 8 | 9 | func getFsID(mountPath string) (*FsID, error) { 10 | statfs := &unix.Statfs_t{} 11 | 12 | if err := unix.Statfs(mountPath, statfs); err != nil { 13 | return nil, err 14 | } 15 | var id FsID 16 | id.val[0] = statfs.Fsid.Val[0] 17 | id.val[1] = statfs.Fsid.Val[1] 18 | return &id, nil 19 | } 20 | -------------------------------------------------------------------------------- /pkg/fsroot/testing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 DDN. All rights reserved. 2 | // Use of this source code is governed by a MIT-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package fsroot 6 | 7 | import "github.com/edwardsp/go-lustre/fs" 8 | 9 | // TestClient implements the client.Client interface 10 | type testClient struct { 11 | root string 12 | } 13 | 14 | // Test returns a test client. 15 | func Test(root string) Client { 16 | return &testClient{root: root} 17 | } 18 | 19 | // FsName returns a fake filesystem name 20 | func (c *testClient) FsName() string { 21 | return "test" 22 | } 23 | 24 | // Path returns a fake filesystem path 25 | func (c *testClient) Path() string { 26 | return c.root 27 | } 28 | 29 | // Root returns a fake fs.RootDir item 30 | func (c *testClient) Root() fs.RootDir { 31 | // Todo need a TestRootDir 32 | return fs.RootDir{} 33 | } 34 | -------------------------------------------------------------------------------- /pkg/zipcheck/analyze.go: -------------------------------------------------------------------------------- 1 | package zipcheck 2 | 3 | // Evaluate the "compressibilty" of a file by compressing 4 | // a small sample. 5 | 6 | import ( 7 | "compress/zlib" 8 | "io" 9 | "math" 10 | "os" 11 | "time" 12 | 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | // CompressResult returns resuls of comressibility check. 17 | type CompressResult struct { 18 | T time.Duration 19 | Samples int 20 | Size int64 21 | Bytes int64 22 | ZipBytes int64 23 | } 24 | 25 | // Null is a /dev/null Writer that counts how many bytes have been written to it. 26 | type Null struct { 27 | Bytes int64 28 | } 29 | 30 | func (n *Null) Write(b []byte) (int, error) { 31 | n.Bytes += int64(len(b)) 32 | return len(b), nil 33 | } 34 | 35 | // SampleFile reads count blocks of blockSize from fp, and copies them to w. 36 | func SampleFile(w io.Writer, fp io.ReaderAt, count int, blockSize int64, step int64) (int64, error) { 37 | var offset int64 38 | var copied int64 39 | for i := 0; i < count; i++ { 40 | r := io.NewSectionReader(fp, offset, blockSize) 41 | nb, err := io.Copy(w, r) 42 | if err != nil { 43 | return copied, errors.Wrap(err, "copy failed") 44 | } 45 | copied += nb 46 | offset += step 47 | 48 | } 49 | return copied, nil 50 | } 51 | 52 | func analyze(fname string, count int, block int64, zipper zipFunc) (*CompressResult, error) { 53 | var cr CompressResult 54 | 55 | f, err := os.Open(fname) 56 | if err != nil { 57 | return nil, errors.Wrap(err, "open failed") 58 | } 59 | defer f.Close() 60 | 61 | fi, err := f.Stat() 62 | if err != nil { 63 | return nil, errors.Wrap(err, "stat failed") 64 | } 65 | if count == 0 { 66 | // default is 2*log(size) smaples for a quick scan 67 | count = 2 * int(math.Log(float64(fi.Size()))) 68 | } 69 | null := &Null{} 70 | w, err := zipper(null) 71 | if err != nil { 72 | return nil, errors.Wrap(err, "create compressor failed") 73 | } 74 | 75 | // Compress entire file it is smaller than the total sample size 76 | if fi.Size() < int64(count)*block { 77 | block = fi.Size() 78 | count = 1 79 | } 80 | 81 | step := fi.Size() / int64(count) 82 | started := time.Now() 83 | cr.Bytes, err = SampleFile(w, f, count, block, step) 84 | w.Close() 85 | if err != nil { 86 | return nil, errors.Wrap(err, "sample failed") 87 | } 88 | 89 | cr.Samples = count 90 | cr.Size = block 91 | cr.T = time.Since(started) 92 | cr.ZipBytes = null.Bytes 93 | return &cr, nil 94 | } 95 | 96 | type zipFunc func(io.Writer) (io.WriteCloser, error) 97 | 98 | func gzip(level int) zipFunc { 99 | return func(w io.Writer) (io.WriteCloser, error) { 100 | zip, err := zlib.NewWriterLevel(w, level) 101 | if err != nil { 102 | return nil, errors.Wrap(err, "NewWriterLevel") 103 | } 104 | return zip, nil 105 | } 106 | } 107 | 108 | // AnalyzeFile will compress a sample of if the file and return estimated reduction percentage. 109 | // 0 means no reduction, 50% means file might be resuduced to half. 110 | func AnalyzeFile(fname string) (float64, error) { 111 | cr, err := analyze(fname, 0, 4096, gzip(1)) 112 | if err != nil { 113 | return 0, errors.Wrap(err, "analayze failed") 114 | } 115 | reduced := (1 - float64(cr.ZipBytes)/float64(cr.Bytes)) * 100 116 | return reduced, nil 117 | } 118 | -------------------------------------------------------------------------------- /toolset/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7.8.2003 2 | 3 | LABEL publisher="Paul Edwards" 4 | 5 | RUN yum -y install sudo gcc make wget rpm-build yum-utils openssl-devel libcurl-devel expat-devel tcl-devel gettext autoconf 6 | 7 | RUN wget https://github.com/git/git/archive/v2.32.0.tar.gz && \ 8 | tar -xvf v2.32.0.tar.gz && \ 9 | cd git-* && \ 10 | make configure && \ 11 | sudo ./configure --prefix=/usr && \ 12 | sudo make && \ 13 | sudo make install 14 | 15 | RUN cd /opt && wget -qO- https://dl.google.com/go/go1.13.15.linux-amd64.tar.gz | tar zxvf - 16 | 17 | RUN rpm -ivh --force --nodeps https://downloads.whamcloud.com/public/lustre/lustre-2.12.5/el7/server/RPMS/x86_64/lustre-2.12.5-1.el7.x86_64.rpm 18 | 19 | ENV PATH="/opt/go/bin:${PATH}" 20 | 21 | RUN cd /usr/local/bin && \ 22 | wget -q https://aka.ms/downloadazcopy-v10-linux -O - | tar zxf - --strip-components 1 --wildcards '*/azcopy' && \ 23 | chmod 755 azcopy 24 | -------------------------------------------------------------------------------- /toolset/LustrePack.repo: -------------------------------------------------------------------------------- 1 | [lustreclient] 2 | name=lustreclient 3 | baseurl=https://downloads.whamcloud.com/public/lustre/lustre-__LUSTRE_VERSION__/el7/client/ 4 | enabled=1 5 | gpgcheck=0 6 | -------------------------------------------------------------------------------- /toolset/README.md: -------------------------------------------------------------------------------- 1 | Lemur build container for vscode 2 | ================================ 3 | 4 | Instructions to build: 5 | 6 | ``` 7 | docker build -t paulmedwards/lustre-dev . 8 | ``` 9 | 10 | Once the container is built you can open the repo in vscode and start the container (click the "><" icon in the bottom left and choose "Remote-containers: Reopen in container"). Once the container has started you are able to start a terminal and run `make local-rpm`. 11 | 12 | --------------------------------------------------------------------------------