├── tests ├── __init__.py ├── cli │ ├── __init__.py │ └── test_cli.py ├── lib │ ├── __init__.py │ └── utils.py ├── commands │ ├── __init__.py │ └── test_validate_image.py ├── helper │ ├── __init__.py │ ├── test_manifest_reader.py │ ├── test_arg_parser.py │ ├── test_print_message.py │ └── test_docker.py └── validation_tool │ ├── __init__.py │ ├── validation_tests │ ├── __init__.py │ ├── test_check_local_job_run.py │ ├── test_check_manifest.py │ ├── test_check_files.py │ └── test_check_envs.py │ ├── test_check_inputs.py │ └── test_validation_helper.py ├── amazon_emr_serverless_image_cli ├── cli │ ├── __init__.py │ └── cli.py ├── commands │ ├── __init__.py │ ├── base_command.py │ └── validate_image.py ├── helper │ ├── __init__.py │ ├── docker_util │ │ ├── __init__.py │ │ ├── docker_helper.py │ │ └── docker_cl.py │ ├── manifest_reader.py │ ├── print_message.py │ ├── logging.py │ └── argument_parser.py ├── validation_tool │ ├── __init__.py │ ├── validation_models │ │ ├── __init__.py │ │ └── validation_models.py │ ├── validation_tests │ │ ├── __init__.py │ │ ├── base_check.py │ │ ├── check_local_job_run.py │ │ ├── check_envs.py │ │ ├── check_manifest.py │ │ └── check_files.py │ ├── check_inputs.py │ └── validation_helper.py ├── __init__.py ├── __main__.py └── assets │ └── image-manifest.yaml ├── setup.py ├── installer └── pyinstaller │ ├── requirements.txt │ ├── INSTRUCTION.md │ ├── custom-image-validation-tool-win.spec │ ├── custom-image-validation-tool.spec │ ├── build-win.bat │ ├── build-mac.sh │ └── build-linux.sh ├── NOTICE ├── .gitignore ├── CODE_OF_CONDUCT.md ├── setup.cfg ├── CONTRIBUTING.md ├── DEVELOPMENT_GUIDE.md ├── README.md └── LICENSE /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/helper/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/validation_tool/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | setup() -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/helper/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/validation_tool/validation_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/helper/docker_util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /installer/pyinstaller/requirements.txt: -------------------------------------------------------------------------------- 1 | setuptools 2 | pyinstaller -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.0.1" 2 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | .build_mac/ 3 | .idea 4 | __pycache__ 5 | build 6 | dist 7 | *.egg-info 8 | .coverage 9 | .DS_Store -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/__main__.py: -------------------------------------------------------------------------------- 1 | from amazon_emr_serverless_image_cli.cli.cli import run 2 | 3 | if __name__ == '__main__': 4 | run() 5 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/helper/docker_util/docker_helper.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | 3 | 4 | def verify_docker(): 5 | print("... Checking if docker cli is installed", flush=True) 6 | shutil.which("docker") 7 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_tests/base_check.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class BaseCheck(ABC): 5 | """ 6 | The BaseCheck interface declares a method for checking the tests. 7 | """ 8 | 9 | @abstractmethod 10 | def check(self): 11 | pass 12 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | 3 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 4 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 5 | opensource-codeofconduct@amazon.com with any additional questions or comments. 6 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/commands/base_command.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class BaseCommand(ABC): 5 | """ 6 | The BaseCommand interface declares a method for running a subcommand. 7 | """ 8 | 9 | @abstractmethod 10 | def initiate(self, args, log): 11 | pass 12 | 13 | @abstractmethod 14 | def run(self): 15 | pass 16 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/helper/manifest_reader.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | from amazon_emr_serverless_image_cli.validation_tool.validation_models.validation_models import \ 4 | ImageManifest 5 | 6 | yaml.add_constructor(ImageManifest.yaml_tag, ImageManifest.from_yaml) 7 | 8 | 9 | def load_yaml(file): 10 | with open(file, "r") as f: 11 | image_manifest = yaml.safe_load(f) 12 | return image_manifest 13 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/check_inputs.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def check_version(version, release_name, log): 5 | if not version: 6 | log.error("No matching image with %s \'%s\' : FAIL" % ('ReleaseName', release_name)) 7 | sys.exit(2) 8 | 9 | 10 | def check_image(image, image_type, log): 11 | if not image: 12 | log.error("No matching image with %s \'%s\' : FAIL" % ('Imagetype', image_type)) 13 | sys.exit(2) 14 | -------------------------------------------------------------------------------- /tests/lib/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define testing utils 3 | """ 4 | 5 | IMAGE = "895885662937.dkr.ecr.us-west-2.amazonaws.com/spark/emr-5.32.0-20210129:2.4.7-amzn-0-vanilla" 6 | RELEASE_NAME = "emr-5.32" 7 | IMAGE_TYPE = "spark" 8 | 9 | INSPECT = dict() 10 | INSPECT['Id'] = 'sha:asdf' 11 | INSPECT['Created'] = '2020/04/22' 12 | INSPECT['Config'] = dict() 13 | INSPECT['Config']['User'] = 'user' 14 | INSPECT['Config']['WorkingDir'] = 'workingdir' 15 | INSPECT['Config']['Entrypoint'] = ['entrypoint'] 16 | INSPECT['Config']['Env'] = ['env1=path1', "env2=path2"] 17 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = amazon_emr_serverless_image_cli 3 | version = attr: amazon_emr_serverless_image_cli.__version__ 4 | description_file = README.md 5 | 6 | [options] 7 | packages = find: 8 | install_requires = 9 | colorlog 10 | docker >= 5.0.0, < 7 11 | PyYAML >= 5.4.1, < 7 12 | include_package_data = True 13 | 14 | [options.packages.find] 15 | exclude = tests* 16 | 17 | [options.entry_points] 18 | console_scripts = 19 | amazon-emr-serverless-image = amazon_emr_serverless_image_cli.cli.cli:run 20 | 21 | [options.package_data] 22 | amazon_emr_serverless_image_cli = 23 | assets/* 24 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/cli/cli.py: -------------------------------------------------------------------------------- 1 | from amazon_emr_serverless_image_cli.commands.validate_image import ValidateImage 2 | from amazon_emr_serverless_image_cli.helper import argument_parser, print_message 3 | from amazon_emr_serverless_image_cli.helper.logging import Log 4 | 5 | 6 | def run(): 7 | args = argument_parser.parse_commandline_arguments() 8 | log = Log() 9 | print_message.print_pre_verification_text() 10 | 11 | if args.command is not None: 12 | commands = {"validate-image": ValidateImage()} 13 | 14 | commands[args.command].initiate(args, log) 15 | commands[args.command].run() 16 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/helper/print_message.py: -------------------------------------------------------------------------------- 1 | from .. import __version__ 2 | 3 | 4 | def print_validate_completion_message(validation_succeeded): 5 | print("-----------------------------------------------------------------") 6 | if validation_succeeded: 7 | print("Overall Custom Image Validation Succeeded.") 8 | else: 9 | print("Custom Image Validation Failed. Please see individual test results above for detailed information.") 10 | 11 | print("-----------------------------------------------------------------") 12 | 13 | 14 | def print_pre_verification_text(): 15 | print("Amazon EMR Serverless - Image CLI") 16 | print("Version: %s" % __version__) 17 | -------------------------------------------------------------------------------- /tests/helper/test_manifest_reader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | from unittest import mock 4 | 5 | import yaml 6 | 7 | from amazon_emr_serverless_image_cli.helper import manifest_reader 8 | 9 | 10 | class TestManifestReader(unittest.TestCase): 11 | def test_load_yaml(self): 12 | yaml.load = mock.MagicMock( 13 | return_value={"Test": {"ValidationTool": "AWSValidationTool"}}) 14 | with mock.patch("builtins.open", mock.mock_open(read_data="yaml_file")): 15 | actual = manifest_reader.load_yaml("yaml_file") 16 | expected = dict() 17 | expected['Test'] = dict() 18 | expected['Test']['ValidationTool'] = "AWSValidationTool" 19 | yaml.load.assert_called_once() 20 | self.assertEqual(actual, expected) 21 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/helper/logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import colorlog 4 | 5 | 6 | class Log: 7 | def __init__(self): 8 | self.log = logging.getLogger("logger") 9 | stream = logging.StreamHandler() 10 | formatter = colorlog.ColoredFormatter( 11 | "%(log_color)s[%(levelname)s]%(reset)s " 12 | "%(message)s", 13 | reset=True, 14 | log_colors={ 15 | 'DEBUG': 'cyan', 16 | 'INFO': 'green', 17 | 'WARNING': 'yellow', 18 | 'ERROR': 'red', 19 | 'CRITICAL': 'red', 20 | }) 21 | stream.setFormatter(formatter) 22 | self.log.addHandler(stream) 23 | self.log.setLevel(logging.INFO) 24 | 25 | def info(self, s): 26 | self.log.info(s) 27 | 28 | def error(self, s): 29 | self.log.error(s) 30 | 31 | def warn(self, s): 32 | self.log.warning(s) 33 | -------------------------------------------------------------------------------- /installer/pyinstaller/INSTRUCTION.md: -------------------------------------------------------------------------------- 1 | # Installer Use Guide 2 | 3 | ### Build Installer for Linux 4 | The build script for linux is tested on Amazon Linux 2. 5 | 6 | ``` 7 | cd 8 | ./installer/pyinstaller/build-linux.sh amazon-emr-serverless-image-cli-linux-x86_64.zip 9 | ``` 10 | 11 | ### Build Installer for macOS 12 | 13 | ``` 14 | cd 15 | ./installer/pyinstaller/build-mac.sh amazon-emr-serverless-image-cli-mac-x86_64.zip 3.10.5 16 | ``` 17 | 18 | ### Build Executable for windows 19 | 20 | #### Build the .exe file. 21 | 22 | If you have python3 pre-installed: 23 | ``` 24 | cd 25 | ./installer/pyinstaller/build-win.bat 26 | ``` 27 | 28 | If you don't have python3 pre-installed, you can either install it by yourself or enter python version as an 29 | input to automatically install. 30 | ``` 31 | cd 32 | ./installer/pyinstaller/build-win.bat 3.7.9 33 | ``` 34 | This may require to unblock the python-installation.exe file (The requirement differs in computers). 35 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/helper/docker_util/docker_cl.py: -------------------------------------------------------------------------------- 1 | import docker 2 | 3 | 4 | class DockerCommand: 5 | def __init__(self): 6 | self.client = docker.from_env() 7 | self.container = None 8 | 9 | def create_container(self, docker_image_uri): 10 | return self.client.containers.run(image=docker_image_uri, entrypoint="/bin/bash", detach=True, tty=True) 11 | 12 | def docker_list_files(self, docker_image_uri, path): 13 | command = ['bash', '-c', 'ls -al %s | awk \'{print $9}\'' % path] 14 | result = self.docker_run(docker_image_uri, command) 15 | return result.output 16 | 17 | def docker_inspect(self, docker_image_uri): 18 | return self.client.api.inspect_image(docker_image_uri) 19 | 20 | def docker_run(self, image_uri, command): 21 | if self.container is None: 22 | self.container = self.create_container(image_uri) 23 | return self.container.exec_run(command) 24 | 25 | def close_docker(self): 26 | if self.container is not None: 27 | self.container.stop(timeout=0) 28 | self.container.remove() 29 | self.client.close() 30 | -------------------------------------------------------------------------------- /installer/pyinstaller/custom-image-validation-tool-win.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python ; coding: utf-8 -*- 2 | 3 | 4 | block_cipher = None 5 | 6 | 7 | a = Analysis(['..\\..\\amazon_emr_serverless_image_cli\\__main__.py'], 8 | pathex=[''], 9 | binaries=[], 10 | datas=[('..\\..\\amazon_emr_serverless_image_cli\\assets\\image-manifest.yaml', 'assets')], 11 | hiddenimports=[], 12 | hookspath=[], 13 | runtime_hooks=[], 14 | excludes=[], 15 | win_no_prefer_redirects=False, 16 | win_private_assemblies=False, 17 | cipher=block_cipher, 18 | noarchive=False) 19 | pyz = PYZ(a.pure, a.zipped_data, 20 | cipher=block_cipher) 21 | exe = EXE(pyz, 22 | a.scripts, 23 | a.binaries, 24 | a.zipfiles, 25 | a.datas, 26 | [], 27 | name='amazon-emr-serverless-image', 28 | debug=False, 29 | bootloader_ignore_signals=False, 30 | strip=False, 31 | upx=True, 32 | upx_exclude=[], 33 | runtime_tmpdir=None, 34 | console=True ) 35 | -------------------------------------------------------------------------------- /installer/pyinstaller/custom-image-validation-tool.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python ; coding: utf-8 -*- 2 | 3 | 4 | block_cipher = None 5 | 6 | 7 | a = Analysis( 8 | ['../../amazon_emr_serverless_image_cli/__main__.py'], 9 | pathex=[], 10 | binaries=[], 11 | datas=[('../../amazon_emr_serverless_image_cli/assets/image-manifest.yaml', 'assets')], 12 | hiddenimports=[], 13 | hookspath=[], 14 | hooksconfig={}, 15 | runtime_hooks=[], 16 | excludes=[], 17 | win_no_prefer_redirects=False, 18 | win_private_assemblies=False, 19 | cipher=block_cipher, 20 | noarchive=False, 21 | ) 22 | pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) 23 | 24 | exe = EXE( 25 | pyz, 26 | a.scripts, 27 | a.binaries, 28 | a.zipfiles, 29 | a.datas, 30 | [], 31 | name='amazon-emr-serverless-image', 32 | debug=False, 33 | bootloader_ignore_signals=False, 34 | strip=False, 35 | upx=True, 36 | upx_exclude=[], 37 | runtime_tmpdir=None, 38 | console=True, 39 | disable_windowed_traceback=False, 40 | argv_emulation=False, 41 | target_arch=None, 42 | codesign_identity=None, 43 | entitlements_file=None, 44 | ) 45 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_local_job_run.py: -------------------------------------------------------------------------------- 1 | from docker.errors import ContainerError 2 | 3 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests import base_check 4 | 5 | 6 | class CheckLocalJobRun(base_check.BaseCheck): 7 | 8 | def __init__(self, image_uri, docker_cmd, log): 9 | self.image_uri = image_uri 10 | self.docker_cmd = docker_cmd 11 | self.log = log 12 | self.entry_point = 'local:///usr/lib/spark/examples/jars/spark-examples.jar' 13 | 14 | def check(self): 15 | try: 16 | print('... Start Running Sample Spark Job') 17 | command = ['bash', 'spark-submit ' 18 | '--deploy-mode client ' 19 | '--master local ' 20 | '--class org.apache.spark.examples.SparkPi ' + self.entry_point] 21 | self.docker_cmd.docker_run(self.image_uri, command) 22 | except ContainerError: 23 | self.log.error('Sample Spark Job Test with %s : FAIL' % self.entry_point) 24 | return False 25 | self.log.info('Sample Spark Job Test with %s : PASS' % self.entry_point) 26 | return True 27 | -------------------------------------------------------------------------------- /tests/helper/test_arg_parser.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from argparse import Namespace 3 | from unittest import mock 4 | 5 | from amazon_emr_serverless_image_cli.helper import argument_parser 6 | from tests.lib.utils import IMAGE, IMAGE_TYPE, RELEASE_NAME 7 | 8 | 9 | class TestArgParser(unittest.TestCase): 10 | @mock.patch("amazon_emr_serverless_image_cli.helper.argument_parser.ArgsParser") 11 | def test_arg_parser(self, argsparser): 12 | args = ["-i", IMAGE, "-r", RELEASE_NAME, "-t", IMAGE_TYPE] 13 | parser = argsparser.return_value 14 | parser.add_argument = mock.Mock() 15 | parser.parse_args = mock.Mock(return_value=Namespace(command='validate-image', local_image_uri=IMAGE, 16 | release_name=RELEASE_NAME, image_type=IMAGE_TYPE)) 17 | parser_args = argument_parser.parse_commandline_arguments(args) 18 | parser.add_argument.assert_called() 19 | parser.parse_args.assert_called_once() 20 | self.assertEqual(parser_args.local_image_uri, args[1]) 21 | self.assertEqual(parser_args.release_name, args[3]) 22 | self.assertEqual(parser_args.image_type, args[5]) 23 | self.assertEqual(parser_args.command, 'validate-image') 24 | -------------------------------------------------------------------------------- /installer/pyinstaller/build-win.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | set python_version=%1 3 | 4 | 5 | echo "Making Folders" 6 | mkdir .build_win\src 7 | mkdir .build_win\output\amazon-emr-serverless-image-cli 8 | 9 | echo "Copying Source" 10 | robocopy . .build_win\src /e /XD .build_win 11 | cd .build_win\src 12 | rmdir /s /q scripts 13 | del Makefile 14 | rmdir /s /q venv 15 | rmdir /s /q __pycache__ 16 | cd .. 17 | 18 | if not "%python_version%"=="" ( 19 | echo "Installing Python3" 20 | curl "https://www.python.org/ftp/python/%python_version%/python-%python_version%-amd64.exe" --output python_install.exe 21 | python_install.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0 22 | ) 23 | 24 | 25 | echo "Installing Python Libraries" 26 | py -3 -m venv venv 27 | venv\Scripts\pip3.exe install -r src/installer/pyinstaller/requirements.txt 28 | venv\Scripts\pip3.exe install src/ 29 | 30 | echo "Building Binary" 31 | cd src 32 | echo "custom-image-validation-tool.spec content is:" 33 | ..\venv\Scripts\python.exe -m PyInstaller --clean installer\pyinstaller\custom-image-validation-tool-win.spec 34 | 35 | mkdir pyinstaller-output 36 | mkdir pyinstaller-output\bin 37 | robocopy /e /move dist pyinstaller-output\bin 38 | cd .. 39 | robocopy /e src\pyinstaller-output output/amazon-emr-serverless-image-cli 40 | robocopy src output/amazon-emr-serverless-image-cli README.md -------------------------------------------------------------------------------- /tests/validation_tool/test_check_inputs.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from amazon_emr_serverless_image_cli.helper import logging 4 | from amazon_emr_serverless_image_cli.validation_tool import check_inputs 5 | from tests.lib.utils import IMAGE_TYPE, RELEASE_NAME 6 | 7 | 8 | class TestInputs(unittest.TestCase): 9 | def setUp(self) -> None: 10 | self.release_name = RELEASE_NAME 11 | self.type = IMAGE_TYPE 12 | self.log = logging.Log() 13 | 14 | def test_check_version(self): 15 | self.assertIsNone(check_inputs.check_version( 16 | RELEASE_NAME, RELEASE_NAME, self.log)) 17 | expected = "ERROR:logger:No matching image with ReleaseName '%s' : FAIL" % RELEASE_NAME 18 | with self.assertRaises(SystemExit) as t, self.assertLogs(self.log.log) as m: 19 | check_inputs.check_version(None, RELEASE_NAME, self.log) 20 | self.assertEqual(t.exception.code, 2) 21 | self.assertIn(expected, m.output) 22 | 23 | def test_check_image(self): 24 | self.assertIsNone(check_inputs.check_image( 25 | self.type, self.type, self.log)) 26 | expected = "ERROR:logger:No matching image with Imagetype '%s' : FAIL" % self.type 27 | with self.assertRaises(SystemExit) as t, self.assertLogs(self.log.log) as m: 28 | check_inputs.check_image(None, self.type, self.log) 29 | self.assertEqual(t.exception.code, 2) 30 | self.assertIn(expected, m.output) 31 | -------------------------------------------------------------------------------- /tests/cli/test_cli.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from argparse import Namespace 3 | from unittest import mock 4 | 5 | from amazon_emr_serverless_image_cli.cli import cli 6 | from tests.lib.utils import IMAGE, IMAGE_TYPE, RELEASE_NAME 7 | 8 | 9 | class TestCli(unittest.TestCase): 10 | 11 | @mock.patch("amazon_emr_serverless_image_cli.helper.print_message.print_pre_verification_text") 12 | @mock.patch("amazon_emr_serverless_image_cli.helper.argument_parser.parse_commandline_arguments") 13 | @mock.patch("amazon_emr_serverless_image_cli.commands.validate_image.ValidateImage.run") 14 | @mock.patch("amazon_emr_serverless_image_cli.commands.validate_image.ValidateImage.initiate") 15 | @mock.patch("amazon_emr_serverless_image_cli.commands.validate_image.ValidateImage.__init__") 16 | def test_run_validate_image(self, validate_image_constructor, validate_image_initiate, validate_image_run, 17 | parse_args, print_pre): 18 | validate_image_constructor.return_value = None 19 | validate_image_initiate.return_value = None 20 | validate_image_run.return_value = None 21 | 22 | parse_args.return_value = Namespace(local_image_uri=IMAGE, release_name=RELEASE_NAME, 23 | image_type=IMAGE_TYPE, command='validate-image') 24 | self.assertIsNone(cli.run()) 25 | 26 | print_pre.assert_called_once() 27 | validate_image_initiate.assert_called_once() 28 | validate_image_run.assert_called_once() 29 | parse_args.assert_called_once() 30 | -------------------------------------------------------------------------------- /tests/helper/test_print_message.py: -------------------------------------------------------------------------------- 1 | import io 2 | import unittest 3 | from unittest import mock 4 | 5 | from amazon_emr_serverless_image_cli import __version__ 6 | from amazon_emr_serverless_image_cli.helper import print_message 7 | 8 | 9 | class TestPrintMessage(unittest.TestCase): 10 | @mock.patch('sys.stdout', new_callable=io.StringIO) 11 | def test_print_pre(self, mock_stdout): 12 | expected = "Amazon EMR Serverless - Image CLI\nVersion: %s\n" % __version__ 13 | print_message.print_pre_verification_text() 14 | self.assertEqual(mock_stdout.getvalue(), expected) 15 | 16 | @mock.patch('sys.stdout', new_callable=io.StringIO) 17 | def test_validate_completion_msg_succeed(self, mock_stdout): 18 | expected = "-----------------------------------------------------------------\n" \ 19 | "Overall Custom Image Validation Succeeded.\n" \ 20 | "-----------------------------------------------------------------\n" 21 | print_message.print_validate_completion_message(True) 22 | self.assertEqual(mock_stdout.getvalue(), expected) 23 | 24 | @mock.patch('sys.stdout', new_callable=io.StringIO) 25 | def test_validate_completion_msg_fail(self, mock_stdout): 26 | expected = "-----------------------------------------------------------------\n" \ 27 | "Custom Image Validation Failed. Please see individual test results above for detailed information.\n" \ 28 | "-----------------------------------------------------------------\n" 29 | print_message.print_validate_completion_message(False) 30 | self.assertEqual(mock_stdout.getvalue(), expected) 31 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_envs.py: -------------------------------------------------------------------------------- 1 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests import base_check 2 | 3 | 4 | class CheckEnvs(base_check.BaseCheck): 5 | 6 | def __init__(self, env_path, env_list, env_vars, log): 7 | self.env_path = env_path 8 | self.env_list = env_list 9 | self.env_vars = env_vars 10 | self.log = log 11 | 12 | def check(self): 13 | env_variables = {} 14 | for env in self.env_path: 15 | var, val = env.split("=") 16 | env_variables[var] = val 17 | return self.match(env_variables) 18 | 19 | def match(self, env_variables): 20 | environment_test = True 21 | for key in self.env_list: 22 | env = [env_var for env_var in self.env_vars if env_var.key == key][0] 23 | if env.env_name in env_variables and env.env_value is not None and env.env_value != env_variables[env.env_name]: 24 | self.log.error("%s MUST set to %s : FAIL" % (env.env_name, env.env_value)) 25 | environment_test = False 26 | elif env.env_name not in env_variables or len(env_variables[env.env_name]) == 0: 27 | self.log.error("%s MUST be set : FAIL" % env.env_name) 28 | environment_test = False 29 | else: 30 | self.log.info("%s is set with value: %s : PASS" % (env.env_name, env_variables[env.env_name])) 31 | return environment_test 32 | 33 | def set_env_path(self, env_path): 34 | self.env_path = env_path 35 | 36 | def set_manifest_envs(self, env_list): 37 | self.env_list = env_list 38 | -------------------------------------------------------------------------------- /tests/validation_tool/validation_tests/test_check_local_job_run.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | 4 | from docker.errors import ContainerError 5 | 6 | from amazon_emr_serverless_image_cli.helper import logging 7 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_local_job_run import \ 8 | CheckLocalJobRun 9 | 10 | 11 | class TestCheckLocalSparkJob(unittest.TestCase): 12 | def setUp(self) -> None: 13 | self.log = logging.Log() 14 | 15 | @mock.patch('amazon_emr_serverless_image_cli.helper.docker_util.docker_cl.DockerCommand') 16 | def test_check(self, docker_constructor): 17 | docker_cmd = docker_constructor.return_value 18 | logger = self.log 19 | sanity_check_instance = CheckLocalJobRun( 20 | 'image_uri', docker_cmd, logger) 21 | 22 | docker_cmd.docker_run = mock.Mock(return_value=None) 23 | with self.assertLogs(logger.log) as t: 24 | result = sanity_check_instance.check() 25 | docker_cmd.docker_run.assert_called_once() 26 | self.assertEqual(result, 1) 27 | expected = 'INFO:logger:Sample Spark Job Test with ' \ 28 | 'local:///usr/lib/spark/examples/jars/spark-examples.jar : PASS' 29 | self.assertIn(expected, t.output) 30 | 31 | docker_cmd.docker_run.side_effect = ContainerError( 32 | None, '', '', '', None) 33 | with self.assertLogs(logger.log) as t: 34 | result = sanity_check_instance.check() 35 | self.assertEqual(result, 0) 36 | expected = 'ERROR:logger:Sample Spark Job Test with ' \ 37 | 'local:///usr/lib/spark/examples/jars/spark-examples.jar : FAIL' 38 | self.assertIn(expected, t.output) 39 | -------------------------------------------------------------------------------- /installer/pyinstaller/build-mac.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | binary_zip_filename=$1 3 | python_version=$2 4 | 5 | if [ "$python_version" = "" ]; then 6 | python_version="3.7.9"; 7 | fi 8 | 9 | 10 | set -eu 11 | 12 | 13 | echo "Making Folders" 14 | mkdir -p .build_mac/src 15 | mkdir -p .build_mac/output/amazon-emr-serverless-image-cli-src 16 | mkdir -p .build_mac/output/pyinstaller-output 17 | cd .build_mac 18 | 19 | echo "Copying Source" 20 | cp -r ../[!.]* ./src 21 | cd src 22 | rm -rf scripts 23 | rm -f Makefile 24 | rm -rf venv 25 | rm -rf __pycache__ 26 | cd .. 27 | cp -r ./src/* ./output/amazon-emr-serverless-image-cli-src 28 | 29 | echo "Installing Python3" 30 | curl "https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz" --output python.tgz 31 | tar -xzf python.tgz 32 | cd Python-$python_version 33 | ./configure --enable-shared 34 | make -j8 35 | make install 36 | cd .. 37 | 38 | echo "Installing Python Libraries" 39 | python3 -m venv venv 40 | ./venv/bin/pip3 install --upgrade pip 41 | ./venv/bin/pip3 install -r src/installer/pyinstaller/requirements.txt 42 | ./venv/bin/pip3 install src/ 43 | 44 | echo "Building Binary" 45 | cd src 46 | echo "custom-image-validation-tool.spec content is:" 47 | cat installer/pyinstaller/custom-image-validation-tool.spec 48 | ../venv/bin/python3 -m PyInstaller --clean installer/pyinstaller/custom-image-validation-tool.spec 49 | 50 | mkdir -p pyinstaller-output 51 | mkdir -p pyinstaller-output/dist 52 | mv dist/* pyinstaller-output/dist 53 | 54 | echo "Copying Binary" 55 | cd .. 56 | cp -r src/pyinstaller-output/* output/pyinstaller-output 57 | 58 | echo "Packaging Binary" 59 | cd output 60 | cd pyinstaller-output 61 | cd dist 62 | cd .. 63 | zip -r ../"$binary_zip_filename" ./* 64 | cd .. 65 | zip -r "$binary_zip_filename" amazon-emr-serverless-image-cli-src 66 | -------------------------------------------------------------------------------- /installer/pyinstaller/build-linux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | binary_zip_filename=$1 3 | python_version=$2 4 | 5 | if [ "$python_version" = "" ]; then 6 | python_version="3.7.9"; 7 | fi 8 | 9 | 10 | set -eu 11 | 12 | export LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib 13 | yum install libffi-devel 14 | yum install -y zlib-devel openssl-devel 15 | 16 | echo "Making Folders" 17 | mkdir -p .build_linux/src 18 | mkdir -p .build_linux/output/amazon-emr-serverless-image-cli-src 19 | mkdir -p .build_linux/output/pyinstaller-output 20 | cd .build_linux 21 | 22 | echo "Copying Source" 23 | cp -r ../[!.]* ./src 24 | cd src 25 | rm -f Makefile 26 | cd .. 27 | cp -r ./src/* ./output/amazon-emr-serverless-image-cli-src 28 | 29 | echo "Installing Python3" 30 | curl "https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz" --output python.tgz 31 | tar -xzf python.tgz 32 | cd Python-$python_version 33 | ./configure --enable-shared 34 | make -j8 35 | make install 36 | cd .. 37 | 38 | echo "Installing Python Libraries" 39 | python3 -m venv venv 40 | ./venv/bin/pip3 install --upgrade pip 41 | ./venv/bin/pip3 install -r src/installer/pyinstaller/requirements.txt 42 | ./venv/bin/pip3 install src/ 43 | 44 | echo "Building Binary" 45 | cd src 46 | echo "custom-image-validation-tool.spec content is:" 47 | cat installer/pyinstaller/custom-image-validation-tool.spec 48 | ../venv/bin/python3 -m PyInstaller -F --clean installer/pyinstaller/custom-image-validation-tool.spec 49 | 50 | mkdir -p pyinstaller-output 51 | mkdir -p pyinstaller-output/dist 52 | mv dist/* pyinstaller-output/dist 53 | 54 | echo "Copying Binary" 55 | cd .. 56 | cp -r src/pyinstaller-output/* output/pyinstaller-output 57 | 58 | echo "Packaging Binary" 59 | yum install zip 60 | cd output 61 | cd pyinstaller-output 62 | cd dist 63 | cd .. 64 | zip -r ../"$binary_zip_filename" ./* 65 | cd .. 66 | zip -r "$binary_zip_filename" amazon-emr-serverless-image-cli-src 67 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_helper.py: -------------------------------------------------------------------------------- 1 | from amazon_emr_serverless_image_cli.validation_tool import check_inputs 2 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests import (check_envs, check_files, check_local_job_run, 3 | check_manifest) 4 | 5 | 6 | def validate_all(inspect_result, 7 | docker_cmd, 8 | docker_image_uri, 9 | image_manifest, 10 | release_name, 11 | image_type, 12 | log): 13 | print("... Checking Image Manifest", flush=True) 14 | image, file_structure, env_vars = load_validation_info(image_manifest, release_name, image_type, log) 15 | 16 | # tests 17 | all_tests = [check_manifest.CheckManifest(inspect_result, image.manifest_config, log), 18 | check_envs.CheckEnvs(inspect_result['Config']['Env'], image.env_vars, env_vars, log), 19 | check_files.CheckFiles(image.file_structures, file_structure, docker_cmd, docker_image_uri, env_vars, 20 | inspect_result['Config']['Env'], log)] 21 | 22 | # Only spark has local job runs 23 | if image_type == 'spark': 24 | all_tests.append(check_local_job_run.CheckLocalJobRun(docker_image_uri, docker_cmd, log)) 25 | 26 | result = [test.check() for test in all_tests] 27 | return all(result) 28 | 29 | 30 | def load_validation_info(image_manifest, release_name, image_type, log): 31 | emr_releases = image_manifest.emr_releases 32 | file_structures = image_manifest.file_structures 33 | env_vars = image_manifest.env_vars 34 | 35 | # check user inputs 36 | release = None 37 | for emr_release in emr_releases: 38 | if release_name == emr_release.release_name: 39 | release = emr_release 40 | check_inputs.check_version(release, release_name, log) 41 | 42 | image = None 43 | for img in release.images: 44 | if image_type == img.image_type: 45 | image = img 46 | check_inputs.check_image(image, image_type, log) 47 | 48 | return image, file_structures, env_vars 49 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/helper/argument_parser.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | 4 | from amazon_emr_serverless_image_cli import __version__ 5 | 6 | 7 | # Parses command line arguments and assigns values to global variables 8 | class ArgsParser(argparse.ArgumentParser): 9 | def error(self, msg): 10 | sys.stderr.write('Error: %s \n' % msg) 11 | self.print_help() 12 | sys.exit(2) 13 | 14 | 15 | def parse_commandline_arguments(args=None): 16 | if args is None: 17 | args = sys.argv[1:] 18 | 19 | main_parser = ArgsParser(prog="amazon-emr-serverless-image", 20 | formatter_class=argparse.RawTextHelpFormatter) 21 | main_parser.add_argument('--version', action='version', 22 | version='Amazon EMR Serverless Image CLI ' 23 | '\nVersion: {version}'.format(version=__version__)) 24 | subparsers = main_parser.add_subparsers(dest="command") 25 | 26 | validate_image_parser = parse_validate_image(subparsers) 27 | main_parser_args = main_parser.parse_args(args) 28 | 29 | return main_parser_args 30 | 31 | 32 | def parse_validate_image(subparsers): 33 | validate_image_parser = subparsers.add_parser(name="validate-image", 34 | formatter_class=argparse.RawTextHelpFormatter) 35 | validate_image_parser.add_argument('--version', action='version', 36 | version='%(prog)s \nVersion: {version}'.format(version=__version__)) 37 | validate_image_parser.add_argument("-i", "--local-image-uri", 38 | help="specifies the name of image uri", 39 | required=True) 40 | validate_image_parser.add_argument("-r", "--release-name", 41 | help="specifies the release name of the image. e.g. emr-5.32.0", 42 | required=True) 43 | validate_image_parser.add_argument("-t", "--image-type", 44 | help="specifies the image runtime type. e.g. spark \ndefault runtime type is " 45 | "spark") 46 | return validate_image_parser 47 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_manifest.py: -------------------------------------------------------------------------------- 1 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests import base_check 2 | 3 | 4 | class CheckManifest(base_check.BaseCheck): 5 | 6 | def __init__(self, inspect_result, manifest_config, log): 7 | self.inspect_result = inspect_result 8 | self.manifest_config = manifest_config 9 | self.log = log 10 | 11 | def check(self): 12 | manifest_validation_test = True 13 | 14 | image_id = self.inspect_result['Id'].split(":")[1] 15 | 16 | self.log.info("Image ID: %s" % image_id) 17 | self.log.info("Created On: %s" % self.inspect_result['Created']) 18 | 19 | # test username 20 | username = self.inspect_result['Config']['User'] 21 | target_username = self.manifest_config.user 22 | if username == target_username: 23 | self.log.info("Default User Set to %s : PASS" % target_username) 24 | else: 25 | self.log.error("Default User MUST be %s. Set to %s : FAIL" % (target_username, username)) 26 | manifest_validation_test = False 27 | 28 | # test workingDir 29 | working_dir = self.inspect_result['Config']['WorkingDir'] 30 | target_working_dir = self.manifest_config.working_dir 31 | if working_dir == target_working_dir: 32 | self.log.info("Working Directory Set to %s : PASS" % target_working_dir) 33 | else: 34 | self.log.error("Working Directory MUST be %s. Set to %s : FAIL" % (target_working_dir, working_dir)) 35 | manifest_validation_test = False 36 | 37 | # test entrypoint 38 | entrypoint = self.inspect_result['Config']['Entrypoint'][0] 39 | target_entrypoint = self.manifest_config.entrypoint 40 | if entrypoint == target_entrypoint: 41 | self.log.info("Entrypoint Set to %s : PASS" % target_entrypoint) 42 | else: 43 | self.log.error("Entrypoint MUST be %s. Set to %s : FAIL" % (target_entrypoint, entrypoint)) 44 | manifest_validation_test = False 45 | 46 | return manifest_validation_test 47 | 48 | def set_inspect_result(self, inspect): 49 | self.inspect_result = inspect 50 | 51 | def set_manifest_config(self, manifest_config): 52 | self.manifest_config = manifest_config 53 | -------------------------------------------------------------------------------- /tests/validation_tool/validation_tests/test_check_manifest.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from amazon_emr_serverless_image_cli.helper import logging 4 | from amazon_emr_serverless_image_cli.validation_tool.validation_models.validation_models import \ 5 | ManifestConfig 6 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_manifest import \ 7 | CheckManifest 8 | from tests.lib.utils import INSPECT 9 | 10 | 11 | class TestManifest(unittest.TestCase): 12 | def setUp(self) -> None: 13 | self.log = logging.Log() 14 | self.inspect = INSPECT 15 | self.manifest_config = ManifestConfig( 16 | "entrypoint", "user", "workingdir") 17 | 18 | def test_check_manifest(self): 19 | with self.assertLogs(self.log.log) as l: 20 | manifest_check_instance = CheckManifest( 21 | self.inspect, self.manifest_config, self.log) 22 | result = manifest_check_instance.check() 23 | self.assertEqual(result, True) 24 | expected = ['INFO:logger:Image ID: %s' % self.inspect['Id'].split(":")[1], 25 | 'INFO:logger:Created On: %s' % self.inspect['Created'], 26 | 'INFO:logger:Default User Set to %s : PASS' % self.manifest_config.user, 27 | 'INFO:logger:Working Directory Set to %s : PASS' % self.manifest_config.working_dir, 28 | 'INFO:logger:Entrypoint Set to %s : PASS' % self.manifest_config.entrypoint] 29 | self.assertEqual(expected, l.output) 30 | 31 | self.inspect['Config']['User'] = 'other' 32 | self.inspect['Config']['WorkingDir'] = 'other' 33 | self.inspect['Config']['Entrypoint'][0] = 'other' 34 | with self.assertLogs(self.log.log) as l: 35 | manifest_check_instance.set_inspect_result(self.inspect) 36 | result = manifest_check_instance.check() 37 | self.assertEqual(result, False) 38 | 39 | expected = ['INFO:logger:Image ID: %s' % self.inspect['Id'].split(":")[1], 40 | 'INFO:logger:Created On: %s' % self.inspect['Created'], 41 | 'ERROR:logger:Default User MUST be %s. Set to %s : FAIL' 42 | % (self.manifest_config.user, self.inspect['Config']['User']), 43 | 'ERROR:logger:Working Directory MUST be %s. Set to %s : FAIL' 44 | % (self.manifest_config.working_dir, self.inspect['Config']['WorkingDir']), 45 | 'ERROR:logger:Entrypoint MUST be %s. Set to %s : FAIL' 46 | % (self.manifest_config.entrypoint, self.inspect['Config']['Entrypoint'][0])] 47 | self.assertEqual(expected, l.output) 48 | -------------------------------------------------------------------------------- /tests/validation_tool/validation_tests/test_check_files.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | 4 | from amazon_emr_serverless_image_cli.helper import logging 5 | from amazon_emr_serverless_image_cli.validation_tool.validation_models.validation_models import \ 6 | FileStructure 7 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_files import \ 8 | CheckFiles 9 | 10 | 11 | class TestCheckFileStructure(unittest.TestCase): 12 | def setUp(self) -> None: 13 | self.log = logging.Log() 14 | 15 | @mock.patch('amazon_emr_serverless_image_cli.helper.docker_util.docker_cl.DockerCommand') 16 | def test_match(self, docker_constructor): 17 | docker_cmd = docker_constructor.return_value 18 | logger = self.log 19 | docker_cmd.docker_list_files = mock.MagicMock( 20 | name="match", return_value=b'something\ntests\nhere\n') 21 | file_structure_list = ['Test1'] 22 | file_structure = [FileStructure( 23 | 'Test1', '/usr/bin', ['asdfasdf', 'sdfasdf'], None)] 24 | file_check_instance = CheckFiles( 25 | file_structure_list, file_structure, docker_cmd, 'image_uri', [], [], logger) 26 | 27 | file_prefixes = ['test'] 28 | with self.assertLogs(logger.log) as t: 29 | result = file_check_instance.match('test', 'path', file_prefixes) 30 | docker_cmd.docker_list_files.assert_called_once() 31 | self.assertEqual(result, 1) 32 | expected = 'INFO:logger:File Structure Test for test in path: PASS' 33 | self.assertIn(expected, t.output) 34 | 35 | file_prefixes = ['test2'] 36 | with self.assertLogs(logger.log) as t: 37 | expected = "ERROR:logger:test2 MUST be in path : FAIL" 38 | result = file_check_instance.match('test', 'path', file_prefixes) 39 | self.assertEqual(result, 0) 40 | self.assertIn(expected, t.output) 41 | 42 | @mock.patch('amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_files.CheckFiles.match') 43 | def test_check(self, match): 44 | logger = self.log 45 | match.return_value = 1 46 | file_structure_list = ['Test1'] 47 | file_structure = [FileStructure( 48 | 'Test1', '/usr/bin', ['asdfasdf', 'sdfasdf'], None)] 49 | file_check_instance = CheckFiles( 50 | file_structure_list, file_structure, "docker_cmd", 'image_uri', [], [], logger) 51 | 52 | actual = file_check_instance.check() 53 | match.assert_called_once() 54 | self.assertEqual(actual, 1) 55 | 56 | match.return_value = 0 57 | actual = file_check_instance.check() 58 | self.assertEqual(actual, 0) 59 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_files.py: -------------------------------------------------------------------------------- 1 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests import base_check 2 | 3 | 4 | class CheckFiles(base_check.BaseCheck): 5 | 6 | def __init__(self, file_structure_list, file_structure, docker_cmd, docker_image_uri, env_vars, image_env_vars, log): 7 | self.file_structure_list = file_structure_list 8 | self.file_structure = file_structure 9 | self.docker_cmd = docker_cmd 10 | self.docker_image_uri = docker_image_uri 11 | self.env_vars = env_vars 12 | self.image_env_vars = {} 13 | self.log = log 14 | 15 | for env in image_env_vars: 16 | var, val = env.split("=") 17 | self.image_env_vars[var] = val 18 | 19 | def check(self): 20 | file_system_test = True 21 | for file_structure_name in self.file_structure_list: 22 | file_structure = [structure for structure in self.file_structure 23 | if structure.name == file_structure_name][0] 24 | name = file_structure.name 25 | 26 | if file_structure.env_var is None: 27 | path = file_structure.relative_location 28 | else: 29 | env_var = [env for env in self.env_vars if env.key == file_structure.env_var][0] 30 | path = self.image_env_vars[env_var.env_name] + file_structure.relative_location 31 | 32 | file_prefixes = file_structure.file_prefixes 33 | if not self.match(name, path, file_prefixes): 34 | file_system_test = False 35 | return file_system_test 36 | 37 | def match(self, name, path, file_prefixes): 38 | local_test_pass = True 39 | docker_log = self.docker_cmd.docker_list_files(self.docker_image_uri, path) 40 | files = docker_log.decode().split('\n')[1:] 41 | for prefix in file_prefixes: 42 | is_match = False 43 | for file in files: 44 | if file.startswith(prefix): 45 | is_match = True 46 | break 47 | if not is_match: 48 | self.log.error("%s MUST be in %s : FAIL" % (prefix, path)) 49 | local_test_pass = False 50 | 51 | if local_test_pass: 52 | self.log.info("File Structure Test for %s in %s: PASS" % (name, path)) 53 | return local_test_pass 54 | 55 | def set_file_structure_list(self, file_structure_list): 56 | self.file_structure_list = file_structure_list 57 | 58 | def set_file_structure(self, file_structure): 59 | self.file_structure = file_structure 60 | 61 | def set_image_uri(self, image_uri): 62 | self.docker_image_uri = image_uri 63 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/commands/validate_image.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from amazon_emr_serverless_image_cli.commands import base_command 5 | from amazon_emr_serverless_image_cli.helper import manifest_reader 6 | from amazon_emr_serverless_image_cli.helper.docker_util import docker_cl, docker_helper 7 | from amazon_emr_serverless_image_cli.helper.print_message import \ 8 | print_validate_completion_message 9 | from amazon_emr_serverless_image_cli.validation_tool import validation_helper 10 | from pathlib import Path 11 | 12 | # Check if this is an executable. 13 | if getattr(sys, 'frozen', False): 14 | IMAGE_MANIFEST_YAML = sys._MEIPASS + "/assets/image-manifest.yaml" 15 | else: 16 | curr_dir = os.path.dirname(__file__) 17 | IMAGE_MANIFEST_YAML = os.path.join(curr_dir, "../assets/image-manifest.yaml") 18 | 19 | class ValidateImage(base_command.BaseCommand): 20 | 21 | def __init__(self): 22 | self.args = None 23 | self.log = None 24 | self.test_num = None 25 | self.image_manifest = None 26 | self.docker_cmd = None 27 | self.inspect_result = None 28 | 29 | def initiate(self, args, log, test_num=3): 30 | self.args = args 31 | self.log = log 32 | self.test_num = test_num 33 | # load image manifest 34 | try: 35 | self.image_manifest = manifest_reader.load_yaml(IMAGE_MANIFEST_YAML) 36 | except Exception as e: 37 | print(e) 38 | self.log.error("image-manifest.yaml doesn't exist and is required.") 39 | sys.exit(2) 40 | 41 | # initialize docker 42 | try: 43 | docker_helper.verify_docker() 44 | self.docker_cmd = docker_cl.DockerCommand() 45 | except Exception: 46 | self.log.error("docker cli doesn't exist but is required.") 47 | sys.exit(2) 48 | 49 | # inspect image 50 | try: 51 | self.inspect_result = self.docker_cmd.docker_inspect(args.local_image_uri) 52 | except Exception: 53 | self.log.error("No such image found.") 54 | sys.exit(2) 55 | 56 | # set default runtime image type to spark 57 | if self.args.image_type is None: 58 | self.args.image_type = 'spark' 59 | 60 | def run(self): 61 | validation_succeeded = validation_helper.validate_all(self.inspect_result, 62 | self.docker_cmd, 63 | self.args.local_image_uri, 64 | self.image_manifest, 65 | self.args.release_name, 66 | self.args.image_type, 67 | self.log) 68 | self.docker_cmd.close_docker() 69 | print_validate_completion_message(validation_succeeded) 70 | 71 | def set_args(self, args): 72 | self.args = args 73 | 74 | def set_docker_command(self, docker_command): 75 | self.docker_cmd = docker_command 76 | -------------------------------------------------------------------------------- /tests/validation_tool/validation_tests/test_check_envs.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | 4 | from amazon_emr_serverless_image_cli.helper import logging 5 | from amazon_emr_serverless_image_cli.validation_tool.validation_models.validation_models import \ 6 | EnvironmentVariable 7 | from amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_envs import \ 8 | CheckEnvs 9 | from tests.lib.utils import INSPECT 10 | 11 | 12 | class TestEnvs(unittest.TestCase): 13 | def setUp(self) -> None: 14 | self.log = logging.Log() 15 | self.inspect = INSPECT 16 | self.env_list = ["env1", "env2", "env3"] 17 | self.envs = [EnvironmentVariable("env1", "env1", "path1"), 18 | EnvironmentVariable("env2", "env2", "path2"), 19 | EnvironmentVariable("env3", "env3", None)] 20 | 21 | def test_match(self): 22 | env_vars = {'env1': 'path1', 'env2': 'path2', 'env3': 'path3'} 23 | env_path = self.inspect['Config']['Env'] 24 | env_check_instance = CheckEnvs( 25 | env_path, self.env_list, self.envs, self.log) 26 | 27 | with self.assertLogs(self.log.log) as t: 28 | result = env_check_instance.match(env_vars) 29 | self.assertEqual(result, 1) 30 | expected = 'INFO:logger:env1 is set with value: path1 : PASS' 31 | self.assertIn(expected, t.output) 32 | expected = 'INFO:logger:env2 is set with value: path2 : PASS' 33 | self.assertIn(expected, t.output) 34 | expected = 'INFO:logger:env3 is set with value: path3 : PASS' 35 | self.assertIn(expected, t.output) 36 | 37 | env_vars['env2'] = 'path3' 38 | with self.assertLogs(self.log.log) as t: 39 | result = env_check_instance.match(env_vars) 40 | self.assertEqual(result, 0) 41 | expected = 'INFO:logger:env1 is set with value: path1 : PASS' 42 | self.assertIn(expected, t.output) 43 | expected = 'ERROR:logger:env2 MUST set to path2 : FAIL' 44 | self.assertIn(expected, t.output) 45 | expected = 'INFO:logger:env3 is set with value: path3 : PASS' 46 | self.assertIn(expected, t.output) 47 | 48 | del env_vars['env3'] 49 | with self.assertLogs(self.log.log) as t: 50 | result = env_check_instance.match(env_vars) 51 | self.assertEqual(result, 0) 52 | expected = 'INFO:logger:env1 is set with value: path1 : PASS' 53 | self.assertIn(expected, t.output) 54 | expected = 'ERROR:logger:env2 MUST set to path2 : FAIL' 55 | self.assertIn(expected, t.output) 56 | expected = 'ERROR:logger:env3 MUST be set : FAIL' 57 | self.assertIn(expected, t.output) 58 | 59 | @mock.patch('amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_envs.CheckEnvs.match') 60 | def test_check(self, match): 61 | env_path = self.inspect['Config']['Env'] 62 | env_check_instance = CheckEnvs( 63 | env_path, self.env_list, self.envs, self.log) 64 | 65 | match.return_value = 1 66 | actual = env_check_instance.check() 67 | expected = {'env1': 'path1', 'env2': 'path2'} 68 | match.assert_called_once_with(expected) 69 | self.assertEqual(actual, 1) 70 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | ## Reporting Bugs/Feature Requests 10 | 11 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 12 | 13 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 14 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 15 | 16 | * A reproducible test case or series of steps 17 | * The version of our code being used 18 | * Any modifications you've made relevant to the bug 19 | * Anything unusual about your environment or deployment 20 | 21 | ## Contributing via Pull Requests 22 | 23 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 24 | 25 | 1. You are working against the latest source on the *main* branch. 26 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 27 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 28 | 29 | To send us a pull request, please: 30 | 31 | 1. Fork the repository. 32 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 33 | 3. Ensure local tests pass. 34 | 4. Commit to your fork using clear commit messages. 35 | 5. Send us a pull request, answering any default questions in the pull request interface. 36 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 37 | 38 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 39 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 40 | 41 | ## Finding contributions to work on 42 | 43 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 44 | 45 | ## Code of Conduct 46 | 47 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 48 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 49 | opensource-codeofconduct@amazon.com with any additional questions or comments. 50 | 51 | ## Security issue notifications 52 | 53 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 54 | 55 | ## Licensing 56 | 57 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 58 | -------------------------------------------------------------------------------- /tests/commands/test_validate_image.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from argparse import Namespace 3 | from unittest import mock 4 | 5 | from amazon_emr_serverless_image_cli.commands.validate_image import ValidateImage 6 | from tests.lib.utils import IMAGE, IMAGE_TYPE, RELEASE_NAME 7 | 8 | 9 | class testValidateImage(unittest.TestCase): 10 | 11 | @mock.patch("amazon_emr_serverless_image_cli.helper.docker_util.docker_cl.DockerCommand.close_docker") 12 | @mock.patch("amazon_emr_serverless_image_cli.helper.docker_util.docker_cl.DockerCommand.__init__") 13 | @mock.patch("amazon_emr_serverless_image_cli.commands.validate_image.ValidateImage.initiate") 14 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.validation_helper.validate_all") 15 | def test_run(self, validate_all, initiate, docker_cmd_constructor, close_docker): 16 | docker_cmd_constructor.return_value = None 17 | close_docker.return_value = None 18 | args = Namespace(local_image_uri=IMAGE, 19 | release_name=RELEASE_NAME, image_type=IMAGE_TYPE) 20 | 21 | validate_all.return_value = 3 22 | validate_image_instance = ValidateImage() 23 | initiate.return_value = None 24 | validate_image_instance.set_args(args) 25 | validate_image_instance.set_docker_command(docker_cmd_constructor) 26 | 27 | self.assertIsNone(validate_image_instance.run()) 28 | 29 | validate_all.assert_called_once() 30 | 31 | @mock.patch("amazon_emr_serverless_image_cli.helper.docker_util.docker_helper.verify_docker") 32 | @mock.patch("amazon_emr_serverless_image_cli.helper.docker_util.docker_cl.DockerCommand.docker_inspect") 33 | @mock.patch("amazon_emr_serverless_image_cli.helper.docker_util.docker_cl.DockerCommand.__init__") 34 | @mock.patch("amazon_emr_serverless_image_cli.helper.manifest_reader.load_yaml") 35 | @mock.patch("amazon_emr_serverless_image_cli.helper.logging.Log.info") 36 | @mock.patch("amazon_emr_serverless_image_cli.helper.logging.Log.error") 37 | @mock.patch("amazon_emr_serverless_image_cli.helper.logging.Log.__init__") 38 | def test_initialize(self, logger, log_error, log_info, 39 | load_yaml, docker_cmd_constructor, docker_inspect, verify_docker): 40 | validate_image_instance = ValidateImage() 41 | 42 | logger.return_value = None 43 | log_error.return_value = None 44 | log_info.return_value = None 45 | 46 | load_yaml.return_value = dict() 47 | docker_cmd_constructor.return_value = None 48 | args = Namespace(local_image_uri=IMAGE, 49 | release_name=RELEASE_NAME, image_type=IMAGE_TYPE) 50 | 51 | self.assertIsNone(validate_image_instance.initiate(args, logger)) 52 | load_yaml.assert_called_once() 53 | 54 | verify_docker.side_effect = Exception() 55 | with self.assertRaises(SystemExit) as t: 56 | validate_image_instance.initiate(args, logger) 57 | self.assertEqual(t.exception.code, 2) 58 | 59 | verify_docker.side_effect = None 60 | docker_inspect.side_effect = Exception() 61 | with self.assertRaises(SystemExit) as t: 62 | validate_image_instance.initiate(args, logger) 63 | self.assertEqual(t.exception.code, 2) 64 | 65 | load_yaml.side_effect = Exception() 66 | with self.assertRaises(SystemExit) as t: 67 | validate_image_instance.initiate(args, logger) 68 | self.assertEqual(t.exception.code, 2) 69 | -------------------------------------------------------------------------------- /tests/validation_tool/test_validation_helper.py: -------------------------------------------------------------------------------- 1 | import io 2 | import unittest 3 | from unittest import mock 4 | 5 | from amazon_emr_serverless_image_cli.validation_tool import validation_helper 6 | from amazon_emr_serverless_image_cli.validation_tool.validation_models.validation_models import (EmrRelease, ImageDetail, 7 | ImageManifest) 8 | from tests.lib.utils import INSPECT 9 | 10 | 11 | class TestValidationHelper(unittest.TestCase): 12 | def setUp(self) -> None: 13 | self.inspect = INSPECT 14 | self.manifest = ImageManifest( 15 | [EmrRelease("release_name", [ImageDetail("image_type", None, [], [])])], [], []) 16 | 17 | @mock.patch('sys.stdout', new_callable=io.StringIO) 18 | @mock.patch('amazon_emr_serverless_image_cli.validation_tool.validation_helper.load_validation_info') 19 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_local_job_run.CheckLocalJobRun.check") 20 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_manifest.CheckManifest.check") 21 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_manifest.CheckManifest.__init__") 22 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_files.CheckFiles.check") 23 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_files.CheckFiles.__init__") 24 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_envs.CheckEnvs.check") 25 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.validation_tests.check_envs.CheckEnvs.__init__") 26 | def test_validate_all(self, check_envs_constructor, check_envs, check_files_constructor, check_files, 27 | check_manifest_constructor, check_manifest, check_local_job_run, load_info, 28 | mock_stdout): 29 | check_envs_constructor.return_value = None 30 | check_envs.return_value = True 31 | check_files_constructor.return_value = None 32 | check_files.return_value = True 33 | check_manifest_constructor.return_value = None 34 | check_manifest.return_value = True 35 | check_local_job_run.return_value = True 36 | load_info.return_value = ImageDetail("image_type", None, [], []), [], [] 37 | 38 | actual = validation_helper.validate_all(self.inspect, "docker_cmd", "docker_image_uri", 39 | self.manifest, "release_name", "spark", "log") 40 | self.assertEqual(actual, True) 41 | 42 | check_manifest.assert_called_once() 43 | check_envs.assert_called_once() 44 | check_files.assert_called_once() 45 | check_local_job_run.assert_called_once() 46 | 47 | expected = "... Checking Image Manifest\n" 48 | self.assertEqual(expected, mock_stdout.getvalue()) 49 | 50 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.check_inputs.check_version") 51 | @mock.patch("amazon_emr_serverless_image_cli.validation_tool.check_inputs.check_image") 52 | def test_load_validation_info(self, check_image, check_version): 53 | check_version.return_value = None 54 | check_image.return_value = None 55 | 56 | actual_img, actual_file, actual_env = validation_helper.load_validation_info( 57 | self.manifest, "release_name", "image_type", "log") 58 | self.assertEqual(actual_img, self.manifest.emr_releases[0].images[0]) 59 | self.assertEqual(actual_file, []) 60 | self.assertEqual(actual_env, []) 61 | 62 | check_version.assert_called_once_with( 63 | self.manifest.emr_releases[0], "release_name", "log") 64 | check_image.assert_called_once_with( 65 | self.manifest.emr_releases[0].images[0], "image_type", "log") 66 | -------------------------------------------------------------------------------- /tests/helper/test_docker.py: -------------------------------------------------------------------------------- 1 | import io 2 | import shutil 3 | import unittest 4 | import warnings 5 | from unittest import mock 6 | 7 | import docker 8 | from docker.models.containers import Container 9 | 10 | from amazon_emr_serverless_image_cli.helper.docker_util.docker_cl import DockerCommand 11 | from amazon_emr_serverless_image_cli.helper.docker_util.docker_helper import verify_docker 12 | 13 | 14 | class TestDockers(unittest.TestCase): 15 | def setUp(self) -> None: 16 | warnings.filterwarnings( 17 | action="ignore", message="unclosed", category=ResourceWarning) 18 | docker.from_env = mock.Mock(return_value=docker.client.DockerClient()) 19 | self.docker_cmd = DockerCommand() 20 | docker.from_env.assert_called_once() 21 | 22 | @mock.patch('sys.stdout', new_callable=io.StringIO) 23 | def test_verify_docker(self, mock_stdout): 24 | shutil.which = mock.Mock() 25 | verify_docker() 26 | expected = "... Checking if docker cli is installed\n" 27 | self.assertEqual(expected, mock_stdout.getvalue()) 28 | shutil.which.assert_called_once_with("docker") 29 | 30 | def test_docker_inspect(self): 31 | self.docker_cmd.client.api.inspect_image = mock.Mock( 32 | return_value=dict()) 33 | actual = self.docker_cmd.docker_inspect("image") 34 | self.assertEqual(actual, {}) 35 | self.docker_cmd.client.api.inspect_image.assert_called() 36 | 37 | @mock.patch.object(docker.models.containers.ContainerCollection, "run") 38 | def test_docker_container(self, run): 39 | run.return_value = docker.models.containers.Container() 40 | actual = self.docker_cmd.create_container("image") 41 | self.assertIsInstance(actual, Container) 42 | self.docker_cmd.client.containers.run.assert_called_once() 43 | 44 | def test_docker_run(self): 45 | self.docker_cmd.create_container = mock.Mock( 46 | return_value=docker.models.containers.Container()) 47 | self.docker_cmd.container = self.docker_cmd.create_container() 48 | self.docker_cmd.container.exec_run = \ 49 | mock.Mock(return_value=docker.models.containers.ExecResult( 50 | exit_code=0, output=b'something')) 51 | actual = self.docker_cmd.docker_run("image", "command") 52 | self.assertEqual(actual, docker.models.containers.ExecResult( 53 | exit_code=0, output=b'something')) 54 | self.docker_cmd.container.exec_run.assert_called_once() 55 | 56 | def test_docker_list(self): 57 | self.docker_cmd.create_container = mock.Mock( 58 | return_value=docker.models.containers.Container()) 59 | self.docker_cmd.container = self.docker_cmd.create_container() 60 | self.docker_cmd.create_container.assert_called_once() 61 | 62 | self.docker_cmd.container.exec_run = \ 63 | mock.Mock(return_value=docker.models.containers.ExecResult( 64 | exit_code=0, output=b'something')) 65 | actual = self.docker_cmd.docker_list_files('somathing', '/usr/bin') 66 | self.assertIsInstance(actual, bytes) 67 | self.assertEqual(actual, b'something') 68 | self.docker_cmd.container.exec_run.assert_called_once() 69 | 70 | def test_docker_close(self): 71 | self.docker_cmd.create_container = mock.Mock( 72 | return_value=docker.models.containers.Container()) 73 | self.docker_cmd.container = self.docker_cmd.create_container() 74 | self.docker_cmd.create_container.assert_called_once() 75 | 76 | self.docker_cmd.container.stop = mock.Mock() 77 | self.docker_cmd.container.remove = mock.Mock() 78 | self.docker_cmd.client.close = mock.Mock() 79 | actual = self.docker_cmd.close_docker() 80 | self.assertIsNone(actual) 81 | self.docker_cmd.container.stop.assert_called_once() 82 | self.docker_cmd.container.remove.assert_called_once() 83 | self.docker_cmd.client.close.assert_called_once() 84 | -------------------------------------------------------------------------------- /DEVELOPMENT_GUIDE.md: -------------------------------------------------------------------------------- 1 | # Amazon EMR Serverless Image CLI Development Guide 2 | 3 | This guide will help you set up your development environment for testing and contributing to custom image validation tool. 4 | If you found something is missing or inaccurate, update this guide and send a Pull Request. 5 | 6 | ## Get Source Code 7 | 8 | Pull source code from GitHub repository. 9 | 10 | ## Environment Set Up 11 | 12 | ### Prerequisite 13 | 14 | Before running this tool, please make sure you have Docker CLI installed. 15 | 16 | ### Install Docker CLI (Optional). 17 | 18 | This tool utilizes [Docker CLI](https://docs.docker.com/docker-for-mac/install/) to help validate custom images. 19 | Please make sure you have Docker CLI installed prior to using the tool. 20 | 21 | ### Create Virtual Environment 22 | 23 | To avoid messing up with global python environment, create a virtual environment for this tool 24 | under current folder: 25 | 26 | `python3 -m venv ` 27 | 28 | *Note: You can change the path for you virtual env to whatever you want, but be careful of the slight difference of 29 | the path in Mac and Windows.* 30 | 31 | To activate/deactivate virtual environment, run following command: 32 | 33 | * For Mac/Unix Users, run `source /bin/activate` 34 | 35 | * For Windows Users, run `C:\> \Scripts\activate.bat` 36 | 37 | To deactivate the venv, type in the shell: `deactivate`. 38 | 39 | ### Install Required Dependencies. 40 | 41 | To ensure that all the required dependencies are successfully installed, run: 42 | 43 | ```bash 44 | python3 setup.py install 45 | ``` 46 | 47 | ## Validate Custom Image 48 | 49 | In the root directory, you can directly use python3 command to run the validation tool. 50 | 51 | Then run command: 52 | 53 | ``` 54 | python3 -m amazon_emr_serverless_image validate-image -i -r [-t ] 55 | ``` 56 | 57 | -i specifies the local image URI that needs to be validated, this can be the image URI or any name/tag you defined for your image. 58 | 59 | -r specifies the exact release version of the EMR base image used to generate the customized image. For example, if the custom image was developed using EMR base image with release version 5.32.0, then the parameter should specify emr-5.32.0. 60 | 61 | -t specifies the image type. If this is a spark image, just input spark. The default value is `spark` and the current version only supports spark runtime images. 62 | 63 | After successfully running the tool, the log info will show test results. If the image doesn't meet necessary configuration requirements, you will see error messages that inform the missing part. 64 | 65 | ##### Basic Test 66 | 67 | The [basic test](amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_manifest.py) ensures the image contains expected configuration. The following parameters are verified in this test: 68 | 69 | * `UserName` 70 | * `WorkingDir` 71 | * `EntryPoint` 72 | 73 | ##### Environment Test 74 | 75 | The [environment test](amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_envs.py) ensures the required environment variables are set to the expected paths. 76 | 77 | Examples: 78 | 79 | * `SPARK_HOME=/usr/lib/spark` 80 | * `JAVA_HOME=/etc/alternatives/jre` 81 | 82 | ##### File Structure Test 83 | 84 | The [file structure test](amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_files.py) ensures the required files exist in expected locations. For different 85 | types of images, the required dependencies are different. You should make sure those files are in the correct 86 | location. 87 | 88 | ##### Local Job Run Test 89 | 90 | The [local job run test](amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_local_job_run.py) ensures that the custom image is valid and can pass basic job run. We will run a sample local spark job with following configuration: 91 | 92 | ``` 93 | docker run -it --rm spark-submit 94 | --deploy-mode client 95 | --master local 96 | --class org.apache.spark.examples.SparkPi local:///usr/lib/spark/examples/jars/spark-examples.jar 97 | ``` 98 | 99 | ### Output Results 100 | 101 | Examples: 102 | 103 | ``` 104 | Amazon EMR Serverless Image CLI 105 | Version: X.X.X 106 | ... Checking if docker cli is installed 107 | ... Checking Image Manifest 108 | [INFO] Image ID: c0749c685b2a3cf50ff18c41510324585748a225bc4804a46d96a947db03a53e 109 | [INFO] Created On: 2021-05-17T20:50:07.986662904Z 110 | [INFO] Default User Set to hadoop:hadoop : PASS 111 | [INFO] Working Directory Set to /home/hadoop : PASS 112 | [INFO] Entrypoint Set to /usr/bin/entrypoint.sh : PASS 113 | [INFO] SPARK_HOME is set with value: /usr/lib/spark : PASS 114 | [INFO] JAVA_HOME is set with value: /etc/alternatives/jre : PASS 115 | [INFO] File Structure Test for spark-jars in /usr/lib/spark/jars: PASS 116 | [INFO] File Structure Test for hadoop-files in /usr/lib/hadoop: PASS 117 | [INFO] File Structure Test for hadoop-jars in /usr/lib/hadoop/lib: PASS 118 | [INFO] File Structure Test for bin-files in /usr/bin: PASS 119 | ... Start Running Sample Spark Job 120 | [INFO] Sample Spark Job Test with local:///usr/lib/spark/examples/jars/spark-examples.jar : PASS 121 | ----------------------------------------------------------------- 122 | Overall Custom Image Validation Succeeded. 123 | ----------------------------------------------------------------- 124 | ``` 125 | 126 | Error Message: 127 | 128 | ``` 129 | Amazon EMR Serverless Image CLI 130 | Version: X.X.X 131 | ... Checking if docker cli is installed 132 | ... Checking Image Manifest 133 | [INFO] Image ID: xxxx 134 | [INFO] Created On: 2021-04-20T22:12:05.523378Z 135 | [INFO] Default User Set to hadoop:hadoop : PASS 136 | [INFO] Working Directory Set to /home/hadoop : PASS 137 | [INFO] Entrypoint Set to /usr/bin/entrypoint.sh : PASS 138 | [INFO] SPARK_HOME is set with value: /usr/lib/spark : PASS 139 | [INFO] JAVA_HOME is set with value: /etc/alternatives/jre : PASS 140 | [ERROR] mockito-all MUST be in /usr/lib/hadoop/lib : FAIL 141 | [ERROR] servlet-api MUST be in /usr/lib/hadoop/lib : FAIL 142 | [ERROR] spotbugs-annotations MUST be in /usr/lib/hadoop/lib : FAIL 143 | [ERROR] stax-api MUST be in /usr/lib/hadoop/lib : FAIL 144 | [ERROR] xmlenc MUST be in /usr/lib/hadoop/lib : FAIL 145 | [INFO] File structure test for bin-files in /usr/bin: PASS 146 | ... Start Running Sample Spark Job 147 | [ERROR] Sample Spark Job Test with local:///usr/lib/spark/examples/jars/spark-examples.jar : FAIL 148 | ----------------------------------------------------------------- 149 | Custom Image Validation Failed. Please see individual test results above for detailed information. 150 | ----------------------------------------------------------------- 151 | ``` 152 | 153 | ## Unit Test 154 | 155 | To run unit tests for this tool, you can use command `python3 -m unittest discover`. 156 | 157 | ## Installation 158 | 159 | You can generate a whl file and install locally. 160 | 161 | ```bash 162 | python3 setup.py bdist 163 | pip3 install dist/amazon_emr_serverless_image_cli-1.3-py3-none-any.whl 164 | ``` 165 | 166 | You now have `emr-serverless-custom-image` as a binary. -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/validation_tool/validation_models/validation_models.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | 4 | class ImageManifest(yaml.YAMLObject): 5 | yaml_tag = u'!ImageManifest' 6 | yaml_loader = yaml.SafeLoader 7 | 8 | # emr_releases: list['EmrRelease'], files_structures: list['FileStructure'], env_vars: list['EnvironmentVariable'] 9 | def __init__(self, emr_releases, files_structures, env_vars): 10 | self.emr_releases = emr_releases 11 | self.file_structures = files_structures 12 | self.env_vars = env_vars 13 | 14 | def to_dict(self): 15 | return {'EmrReleases': self.emr_releases, 16 | 'FileStructures': self.file_structures, 17 | 'EnvironmentVariables': self.env_vars} 18 | 19 | @classmethod 20 | def to_yaml(cls, dumper, data): 21 | data.__dict__ = data.to_dict() 22 | return dumper.represent_yaml_object(cls.yaml_tag, data, cls) 23 | 24 | @classmethod 25 | def from_yaml(cls, loader, node): 26 | for data in loader.construct_yaml_map(node): 27 | pass 28 | emr_releases = data['EmrReleases'] 29 | file_structure = data['FileStructures'] 30 | env_vars = data['EnvironmentVariables'] 31 | return cls(emr_releases, file_structure, env_vars) 32 | 33 | 34 | class EmrRelease(yaml.YAMLObject): 35 | yaml_tag = u'!EmrRelease' 36 | yaml_loader = yaml.SafeLoader 37 | 38 | # release_name: str, images: list['ImageDetail'] 39 | def __init__(self, release_name, images): 40 | self.release_name = release_name 41 | self.images = images 42 | 43 | def to_dict(self): 44 | return {'ReleaseName': self.release_name, 45 | 'Images': self.images} 46 | 47 | @classmethod 48 | def to_yaml(cls, dumper, data): 49 | data.__dict__ = data.to_dict() 50 | return dumper.represent_yaml_object(cls.yaml_tag, data, cls) 51 | 52 | @classmethod 53 | def from_yaml(cls, loader, node): 54 | for data in loader.construct_yaml_map(node): 55 | pass 56 | release_name = data['ReleaseName'] 57 | images = data['Images'] 58 | return cls(release_name, images) 59 | 60 | 61 | class EnvironmentVariable(yaml.YAMLObject): 62 | yaml_tag = u'!EnvironmentVariable' 63 | yaml_loader = yaml.SafeLoader 64 | 65 | # key: str, env_name: str, env_value: str 66 | def __init__(self, key, env_name, env_value): 67 | self.key = key 68 | self.env_name = env_name 69 | self.env_value = env_value 70 | 71 | def to_dict(self): 72 | return {'Key': self.key, 73 | 'EnvName': self.env_name, 74 | 'EnvValue': self.env_value} 75 | 76 | @classmethod 77 | def to_yaml(cls, dumper, data): 78 | data.__dict__ = data.to_dict() 79 | return dumper.represent_yaml_object(cls.yaml_tag, data, cls) 80 | 81 | @classmethod 82 | def from_yaml(cls, loader, node): 83 | for data in loader.construct_yaml_map(node): 84 | pass 85 | key = data['Key'] 86 | env_name = data['EnvName'] 87 | 88 | if 'EnvValue' in data: 89 | env_value = data['EnvValue'] 90 | else: 91 | env_value = None 92 | 93 | return cls(key, env_name, env_value) 94 | 95 | 96 | class FileStructure(yaml.YAMLObject): 97 | yaml_tag = u'!FileStructure' 98 | yaml_loader = yaml.SafeLoader 99 | 100 | # name: str, relative_location: str, file_prefixes: list[str] 101 | def __init__(self, name, relative_location, file_prefixes, env_var): 102 | self.name = name 103 | self.relative_location = relative_location 104 | self.file_prefixes = file_prefixes 105 | self.env_var = env_var 106 | 107 | def to_dict(self): 108 | return {'Name': self.name, 109 | 'RelativeLocation': self.relative_location, 110 | 'FilePrefixes': self.file_prefixes} 111 | 112 | @classmethod 113 | def to_yaml(cls, dumper, data): 114 | data.__dict__ = data.to_dict() 115 | return dumper.represent_yaml_object(cls.yaml_tag, data, cls) 116 | 117 | @classmethod 118 | def from_yaml(cls, loader, node): 119 | for data in loader.construct_yaml_map(node): 120 | pass 121 | name = data['Name'] 122 | relative_location = data['RelativeLocation'] 123 | file_prefixes = data['FilePrefixes'] 124 | 125 | if 'EnvironmentVariableRootLocation' in data: 126 | env_var = data['EnvironmentVariableRootLocation'] 127 | else: 128 | env_var = None 129 | 130 | return cls(name, relative_location, file_prefixes, env_var) 131 | 132 | 133 | class ImageDetail(yaml.YAMLObject): 134 | yaml_tag = u'!ImageDetail' 135 | yaml_loader = yaml.SafeLoader 136 | 137 | # image_type: str, manifest_config: 'ManifestConfig', env_vars: list[str], file_structures: list[str] 138 | def __init__(self, image_type, manifest_config, env_vars, file_structures): 139 | self.image_type = image_type 140 | self.manifest_config = manifest_config 141 | self.env_vars = env_vars 142 | self.file_structures = file_structures 143 | 144 | def to_dict(self): 145 | return {'ImageType': self.image_type, 146 | 'ManifestConfig': self.manifest_config, 147 | 'EnvironmentVariable': self.env_vars, 148 | 'FileStructure': self.file_structures} 149 | 150 | @classmethod 151 | def to_yaml(cls, dumper, data): 152 | data.__dict__ = data.to_dict() 153 | return dumper.represent_yaml_object(cls.yaml_tag, data, cls) 154 | 155 | @classmethod 156 | def from_yaml(cls, loader, node): 157 | for data in loader.construct_yaml_map(node): 158 | pass 159 | image_type = data['ImageType'] 160 | manifest_config = data['ManifestConfig'] 161 | env_vars = data['EnvironmentVariable'] 162 | file_structures = data['FileStructure'] 163 | return cls(image_type, manifest_config, env_vars, file_structures) 164 | 165 | 166 | class ManifestConfig(yaml.YAMLObject): 167 | yaml_tag = u'!ManifestConfig' 168 | yaml_loader = yaml.SafeLoader 169 | 170 | # entrypoint: str, user: str, working_dir: str 171 | def __init__(self, entrypoint, user, working_dir): 172 | self.entrypoint = entrypoint 173 | self.user = user 174 | self.working_dir = working_dir 175 | 176 | def to_dict(self): 177 | return {'Entrypoint': self.entrypoint, 178 | 'User': self.user, 179 | 'WorkingDir': self.working_dir} 180 | 181 | @classmethod 182 | def to_yaml(cls, dumper, data): 183 | data.__dict__ = data.to_dict() 184 | return dumper.represent_yaml_object(cls.yaml_tag, data, cls) 185 | 186 | @classmethod 187 | def from_yaml(cls, loader, node): 188 | for data in loader.construct_yaml_map(node): 189 | pass 190 | entrypoint = data['Entrypoint'] 191 | user = data['User'] 192 | working_dir = data['WorkingDir'] 193 | return cls(entrypoint, user, working_dir) 194 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon EMR Serverless Image CLI 2 | 3 | ## Introduction 4 | 5 | [Amazon EMR Serverless](https://aws.amazon.com/emr/serverless/) provides support for 6 | Custom Images, a capability that enables you to customize the Docker container images used for running 7 | Apache Spark and Apache Hive applications on [Amazon EMR Serverless](https://aws.amazon.com/emr/serverless/). 8 | Custom images enables you to install and configure packages specific to your workload that are not available 9 | in the public distribution of EMR’s runtimes into a single immutable container. An immutable container 10 | promotes portability and simplifies dependency management for each workload and enables you to integrate 11 | developing applications for EMR Serverless with your own continuous integration (CI) pipeline. 12 | 13 | To test the compatibility of the modifications made to your EMR base image, we are providing a utility to validate 14 | the image’s file structure. The utility will examine basic required arguments and ensure that the modifications work as 15 | expected and prevent job failures due to common misconfigurations. This tool can be integrated into your Continuous 16 | Integration (CI) pipeline when you are building your image. 17 | 18 | ## For Developers 19 | 20 | Developers who wish to develop on or contribute to the source code, please refer to [Contribution Guide](CONTRIBUTING.md) and [Development Guide](DEVELOPMENT_GUIDE.md). 21 | 22 | ## Getting Started 23 | 24 | ### Prerequisite 25 | 26 | Before running this tool, please make sure you have Docker CLI installed. 27 | 28 | #### Install Docker CLI (Optional). 29 | 30 | This tool utilizes [Docker CLI](https://docs.docker.com/get-docker/) to help validate custom images. 31 | Please make sure you have Docker CLI installed prior to using the tool. 32 | 33 | ### Installation 34 | 35 | #### Option 1 - Python 36 | 37 | If you have python, you can get the wheel file from our Releases and install using Python3. 38 | 39 | ``` 40 | pip3 install 41 | ``` 42 | 43 | The command is available as `amazon-emr-serverless-image`. 44 | 45 | #### Option 2 - PyInstaller 46 | 47 | PyInstaller creates an executable. See instructions [here](installer/pyinstaller/INSTRUCTION.md). 48 | 49 | ### Usage 50 | 51 | #### Validate Custom Image 52 | 53 | Use command: 54 | 55 | ``` 56 | amazon-emr-serverless-image validate-image -i -r [-t ] 57 | ``` 58 | 59 | -i specifies the local image URI that needs to be validated, this can be the image URI or any name/tag you defined for your image. 60 | 61 | -r specifies the exact release version of the EMR base image used to generate the customized image. It supports `emr-6.9.0` and newer releases. 62 | 63 | -t specifies the image type. The default value is `spark`. It also accepts `hive`. 64 | 65 | After successfully running the tool, the log info will show test results. If the image doesn't meet necessary configuration requirements, you will see error messages that inform the missing part. 66 | 67 | ##### Basic Test 68 | 69 | The [basic test](amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_manifest.py) ensures the image contains expected configuration. The following parameters are verified in this test: 70 | 71 | * `UserName` 72 | * `WorkingDir` 73 | * `EntryPoint` 74 | 75 | ##### Environment Test 76 | 77 | The [environment test](amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_envs.py) ensures the required environment variables are set to the expected paths. 78 | 79 | Examples: 80 | 81 | * `SPARK_HOME=/usr/lib/spark` 82 | * `JAVA_HOME=/etc/alternatives/jre` 83 | 84 | ##### File Structure Test 85 | 86 | The [file structure test](amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_files.py) ensures the required files exist in expected locations. For different 87 | types of images, the required dependencies are different. You should make sure those files are in the correct 88 | location. 89 | 90 | ##### Local Job Run Test 91 | 92 | The [local job run test](amazon_emr_serverless_image_cli/validation_tool/validation_tests/check_local_job_run.py) ensures that the custom image is valid and can pass basic job run. We will run a sample local spark job with following configuration: 93 | 94 | ``` 95 | docker run -it --rm --entrypoint /bin/bash spark-submit \ 96 | --deploy-mode client \ 97 | --master local \ 98 | --class org.apache.spark.examples.SparkPi local:///usr/lib/spark/examples/jars/spark-examples.jar 99 | ``` 100 | 101 | ### Output Results 102 | 103 | Examples: 104 | 105 | ``` 106 | Amazon EMR Serverless - Image CLI 107 | Version: 0.0.1 108 | ... Checking if docker cli is installed 109 | ... Checking Image Manifest 110 | [INFO] Image ID: 4c43e76fe820dc0df0fefe31f5307bd0d4da25f9ab606fea8cbdf0cc3a01b9ae 111 | [INFO] Created On: 2022-12-02T07:39:13.100828697Z 112 | [INFO] Default User Set to hadoop:hadoop : PASS 113 | [INFO] Working Directory Set to /home/hadoop : PASS 114 | [INFO] Entrypoint Set to /usr/bin/entrypoint.sh : PASS 115 | [INFO] JAVA_HOME is set with value: /etc/alternatives/jre : PASS 116 | [INFO] SPARK_HOME is set with value: /usr/lib/spark : PASS 117 | [INFO] File Structure Test for bin-files in /usr/bin: PASS 118 | [INFO] File Structure Test for hadoop-files in /usr/lib/hadoop: PASS 119 | [INFO] File Structure Test for hadoop-jars in /usr/lib/hadoop/lib: PASS 120 | [INFO] File Structure Test for spark-jars in /usr/lib/spark/jars: PASS 121 | [INFO] File Structure Test for spark-bin in /usr/lib/spark/bin: PASS 122 | [INFO] File Structure Test for java-bin in /etc/alternatives/jre/bin: PASS 123 | ... Start Running Sample Spark Job 124 | [INFO] Sample Spark Job Test with local:///usr/lib/spark/examples/jars/spark-examples.jar : PASS 125 | ----------------------------------------------------------------- 126 | Overall Custom Image Validation Succeeded. 127 | ----------------------------------------------------------------- 128 | ``` 129 | 130 | Error Message: 131 | 132 | ``` 133 | Amazon EMR Serverless - Image CLI 134 | Version: 0.0.1 135 | ... Checking if docker cli is installed 136 | ... Checking Image Manifest 137 | [INFO] Image ID: 3f061067cacfb69cbf76f233fea73d01ab464a0de748fdba18addce0fe1cfd7b 138 | [INFO] Created On: 2022-12-08T22:45:38.767201727Z 139 | [INFO] Default User Set to hadoop:hadoop : PASS 140 | [INFO] Working Directory Set to /home/hadoop : PASS 141 | [INFO] Entrypoint Set to /usr/bin/entrypoint.sh : PASS 142 | [INFO] JAVA_HOME is set with value: /etc/alternatives/jre : PASS 143 | [INFO] SPARK_HOME is set with value: /usr/lib/spark : PASS 144 | [INFO] File Structure Test for bin-files in /usr/bin: PASS 145 | [INFO] File Structure Test for hadoop-files in /usr/lib/hadoop: PASS 146 | [ERROR] avro MUST be in /usr/lib/hadoop/lib : FAIL 147 | [INFO] File Structure Test for spark-jars in /usr/lib/spark/jars: PASS 148 | [INFO] File Structure Test for spark-bin in /usr/lib/spark/bin: PASS 149 | [INFO] File Structure Test for java-bin in /etc/alternatives/jre/bin: PASS 150 | ... Start Running Sample Spark Job 151 | [INFO] Sample Spark Job Test with local:///usr/lib/spark/examples/jars/spark-examples.jar : PASS 152 | ----------------------------------------------------------------- 153 | Custom Image Validation Failed. Please see individual test results above for detailed information. 154 | ----------------------------------------------------------------- 155 | ``` 156 | 157 | ## Support 158 | 159 | This tool is only supported for the following EMR Serverless releases: 160 | 161 | - `6.9.0` 162 | - `6.10.0` 163 | - `6.11.0` 164 | - `6.12.0` 165 | 166 | Future releases will be supported. Usage of the validation tool does not guarantee your image or job will run in EMR Serverless, but is meant to help validate common configuration issues. 167 | 168 | ## Security 169 | 170 | If you discover a potential security issue in this project, or think you may have discovered a security issue, we request you to notify AWS Security via our vulnerability [reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do not create a public GitHub issue. 171 | 172 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | -------------------------------------------------------------------------------- /amazon_emr_serverless_image_cli/assets/image-manifest.yaml: -------------------------------------------------------------------------------- 1 | !ImageManifest 2 | EmrReleases: 3 | - !EmrRelease 4 | ReleaseName: emr-6.12.0 5 | Images: 6 | - !ImageDetail 7 | ImageType: spark 8 | EnvironmentVariable: 9 | - java_home 10 | - spark_home 11 | FileStructure: 12 | - bin-files 13 | - hadoop-files 14 | - hadoop-jars-6.10 15 | - java-bin 16 | - spark-bin 17 | - spark-jars-6.12 18 | ManifestConfig: !ManifestConfig 19 | Entrypoint: /usr/bin/entrypoint.sh 20 | User: hadoop:hadoop 21 | WorkingDir: /home/hadoop 22 | - !ImageDetail 23 | ImageType: hive 24 | EnvironmentVariable: 25 | - hadoop_home 26 | - hadoop_libexec_dir 27 | - hadoop_user_home 28 | - hadoop_yarn_home 29 | - hive_home 30 | - java_home 31 | - tez_home 32 | - yarn_home 33 | FileStructure: 34 | - hadoop-files 35 | - hadoop-jars-6.10 36 | - hadoop-yarn-jars 37 | - hive-bin-files 38 | - hive-jars-6.12 39 | - java-bin 40 | - tez-jars 41 | ManifestConfig: !ManifestConfig 42 | Entrypoint: /usr/bin/entrypoint.sh 43 | User: hadoop:hadoop 44 | WorkingDir: "" 45 | - !EmrRelease 46 | ReleaseName: emr-6.11.0 47 | Images: 48 | - !ImageDetail 49 | ImageType: spark 50 | EnvironmentVariable: 51 | - java_home 52 | - spark_home 53 | FileStructure: 54 | - bin-files 55 | - hadoop-files 56 | - hadoop-jars-6.10 57 | - java-bin 58 | - spark-bin 59 | - spark-jars 60 | ManifestConfig: !ManifestConfig 61 | Entrypoint: /usr/bin/entrypoint.sh 62 | User: hadoop:hadoop 63 | WorkingDir: /home/hadoop 64 | - !ImageDetail 65 | ImageType: hive 66 | EnvironmentVariable: 67 | - hadoop_home 68 | - hadoop_libexec_dir 69 | - hadoop_user_home 70 | - hadoop_yarn_home 71 | - hive_home 72 | - java_home 73 | - tez_home 74 | - yarn_home 75 | FileStructure: 76 | - hadoop-files 77 | - hadoop-jars-6.10 78 | - hadoop-yarn-jars 79 | - hive-bin-files 80 | - hive-jars-6.10 81 | - java-bin 82 | - tez-jars 83 | ManifestConfig: !ManifestConfig 84 | Entrypoint: /usr/bin/entrypoint.sh 85 | User: hadoop:hadoop 86 | WorkingDir: "" 87 | - !EmrRelease 88 | ReleaseName: emr-6.10.0 89 | Images: 90 | - !ImageDetail 91 | ImageType: spark 92 | EnvironmentVariable: 93 | - java_home 94 | - spark_home 95 | FileStructure: 96 | - bin-files 97 | - hadoop-files 98 | - hadoop-jars-6.10 99 | - java-bin 100 | - spark-bin 101 | - spark-jars 102 | ManifestConfig: !ManifestConfig 103 | Entrypoint: /usr/bin/entrypoint.sh 104 | User: hadoop:hadoop 105 | WorkingDir: /home/hadoop 106 | - !ImageDetail 107 | ImageType: hive 108 | EnvironmentVariable: 109 | - hadoop_home 110 | - hadoop_libexec_dir 111 | - hadoop_user_home 112 | - hadoop_yarn_home 113 | - hive_home 114 | - java_home 115 | - tez_home 116 | - yarn_home 117 | FileStructure: 118 | - hadoop-files 119 | - hadoop-jars-6.10 120 | - hadoop-yarn-jars 121 | - hive-bin-files 122 | - hive-jars-6.10 123 | - java-bin 124 | - tez-jars 125 | ManifestConfig: !ManifestConfig 126 | Entrypoint: /usr/bin/entrypoint.sh 127 | User: hadoop:hadoop 128 | WorkingDir: "" 129 | - !EmrRelease 130 | ReleaseName: emr-6.9.0 131 | Images: 132 | - !ImageDetail 133 | ImageType: spark 134 | EnvironmentVariable: 135 | - java_home 136 | - spark_home 137 | FileStructure: 138 | - bin-files 139 | - hadoop-files 140 | - hadoop-jars-6.9 141 | - java-bin 142 | - spark-bin 143 | - spark-jars 144 | ManifestConfig: !ManifestConfig 145 | Entrypoint: /usr/bin/entrypoint.sh 146 | User: hadoop:hadoop 147 | WorkingDir: /home/hadoop 148 | - !ImageDetail 149 | ImageType: hive 150 | EnvironmentVariable: 151 | - hadoop_home 152 | - hadoop_libexec_dir 153 | - hadoop_user_home 154 | - hadoop_yarn_home 155 | - hive_home 156 | - java_home 157 | - tez_home 158 | - yarn_home 159 | FileStructure: 160 | - hadoop-files 161 | - hadoop-jars-6.9 162 | - hadoop-yarn-jars 163 | - hive-bin-files 164 | - hive-jars-6.9 165 | - java-bin 166 | - tez-jars 167 | ManifestConfig: !ManifestConfig 168 | Entrypoint: /usr/bin/entrypoint.sh 169 | User: hadoop:hadoop 170 | WorkingDir: "" 171 | EnvironmentVariables: 172 | - !EnvironmentVariable 173 | EnvName: SPARK_HOME 174 | Key: spark_home 175 | - !EnvironmentVariable 176 | EnvName: JAVA_HOME 177 | Key: java_home 178 | - !EnvironmentVariable 179 | EnvName: HADOOP_HOME 180 | EnvValue: /usr/lib/hadoop 181 | Key: hadoop_home 182 | - !EnvironmentVariable 183 | EnvName: YARN_HOME 184 | EnvValue: /usr/lib/hadoop-yarn 185 | Key: yarn_home 186 | - !EnvironmentVariable 187 | EnvName: HADOOP_USER_HOME 188 | EnvValue: /home/hadoop 189 | Key: hadoop_user_home 190 | - !EnvironmentVariable 191 | EnvName: HADOOP_LIBEXEC_DIR 192 | EnvValue: /usr/lib/hadoop/libexec 193 | Key: hadoop_libexec_dir 194 | - !EnvironmentVariable 195 | EnvName: HIVE_HOME 196 | EnvValue: /usr/lib/hive 197 | Key: hive_home 198 | - !EnvironmentVariable 199 | EnvName: HADOOP_YARN_HOME 200 | EnvValue: /usr/lib/hadoop-yarn 201 | Key: hadoop_yarn_home 202 | - !EnvironmentVariable 203 | EnvName: TEZ_HOME 204 | EnvValue: /usr/lib/tez 205 | Key: tez_home 206 | FileStructures: 207 | - !FileStructure 208 | Name: spark-jars-6.12 209 | EnvironmentVariableRootLocation: spark_home 210 | RelativeLocation: /jars 211 | FilePrefixes: 212 | - activation 213 | - aggdesigner-algorithm 214 | - aircompressor 215 | - algebra 216 | - annotations 217 | - antlr-runtime 218 | - antlr4-runtime 219 | - aopalliance-repackaged 220 | - arpack_combined_all 221 | - arpack 222 | - arrow-format 223 | - arrow-memory-core 224 | - arrow-memory-netty 225 | - arrow-vector 226 | - audience-annotations 227 | - avro 228 | - avro-ipc 229 | - avro-mapred 230 | - blas 231 | - bonecp 232 | - breeze 233 | - breeze-macros 234 | - cats-kernel 235 | - chill 236 | - chill-java 237 | - commons-cli 238 | - commons-codec 239 | - commons-collections 240 | - commons-collections4 241 | - commons-compiler 242 | - commons-compress 243 | - commons-crypto 244 | - commons-dbcp 245 | - commons-io 246 | - commons-lang 247 | - commons-lang3 248 | - commons-logging 249 | - commons-math3 250 | - commons-pool 251 | - commons-text 252 | - compress-lzf 253 | - curator-client 254 | - curator-framework 255 | - curator-recipes 256 | - datanucleus-api-jdo 257 | - datanucleus-core 258 | - datanucleus-rdbms 259 | - derby 260 | - disruptor 261 | - dropwizard-metrics-hadoop-metrics2 262 | - flatbuffers-java 263 | - gmetric4j 264 | - gson 265 | - guava 266 | - hadoop-client-api 267 | - hadoop-client-runtime 268 | - hadoop-shaded-guava 269 | - hadoop-yarn-server-web-proxy 270 | - HikariCP 271 | - hive-beeline 272 | - hive-cli 273 | - hive-common 274 | - hive-exec 275 | - hive-jdbc 276 | - hive-llap-common 277 | - hive-metastore 278 | - hive-serde 279 | - hive-service-rpc 280 | - hive-shims 281 | - hive-shims-common 282 | - hive-shims-scheduler 283 | - hive-storage-api 284 | - hk2-api 285 | - hk2-locator 286 | - hk2-utils 287 | - httpclient 288 | - httpcore 289 | - istack-commons-runtime 290 | - ivy 291 | - jackson-annotations 292 | - jackson-core 293 | - jackson-core-asl 294 | - jackson-databind 295 | - jackson-dataformat-yaml 296 | - jackson-datatype-jsr310 297 | - jackson-mapper-asl 298 | - jackson-module-scala 299 | - jakarta.annotation-api 300 | - jakarta.inject 301 | - jakarta.servlet-api 302 | - jakarta.validation-api 303 | - jakarta.ws.rs-api 304 | - jakarta.xml.bind-api 305 | - janino 306 | - javassist 307 | - javax.jdo 308 | - javax.servlet-api 309 | - javolution 310 | - jaxb-runtime 311 | - jcl-over-slf4j 312 | - jdo-api 313 | - jersey-client 314 | - jersey-common 315 | - jersey-container-servlet 316 | - jersey-container-servlet-core 317 | - jersey-hk2 318 | - jersey-server 319 | - jetty-rewrite 320 | - JLargeArrays 321 | - jline 322 | - joda-time 323 | - jodd-core 324 | - jpam 325 | - json 326 | - json4s-ast 327 | - json4s-core 328 | - json4s-jackson 329 | - json4s-scalap 330 | - jsr305 331 | - jta 332 | - JTransforms 333 | - jul-to-slf4j 334 | - kryo-shaded 335 | - lapack 336 | - leveldbjni-all 337 | - libfb303 338 | - libthrift 339 | - log4j-1.2-api 340 | - log4j-api 341 | - log4j-core 342 | - log4j-slf4j2-impl 343 | - logging-interceptor 344 | - lz4-java 345 | - metrics-core 346 | - metrics-graphite 347 | - metrics-jmx 348 | - metrics-json 349 | - metrics-jvm 350 | - minlog 351 | - netty-all 352 | - netty-buffer 353 | - netty-codec 354 | - netty-codec-http 355 | - netty-codec-http2 356 | - netty-codec-socks 357 | - netty-common 358 | - netty-handler 359 | - netty-handler-proxy 360 | - netty-resolver 361 | - netty-transport 362 | - netty-transport-classes-epoll 363 | - netty-transport-classes-kqueue 364 | - netty-transport-native-epoll 365 | - netty-transport-native-kqueue 366 | - netty-transport-native-unix-common 367 | - objenesis 368 | - okhttp 369 | - okio 370 | - opencsv 371 | - orc-core-1.8.3-shaded-protobuf 372 | - orc-mapreduce-1.8.3-shaded-protobuf 373 | - orc-shims 374 | - oro 375 | - osgi-resource-locator 376 | - paranamer 377 | - parquet-column 378 | - parquet-common 379 | - parquet-encoding 380 | - parquet-format-structures 381 | - parquet-hadoop 382 | - parquet-jackson 383 | - pickle 384 | - protobuf-java 385 | - py4j 386 | - remotetea-oncrpc 387 | - RoaringBitmap 388 | - rocksdbjni 389 | - scala-collection-compat 390 | - scala-compiler 391 | - scala-library 392 | - scala-parser-combinators 393 | - scala-reflect 394 | - scala-xml 395 | - shims 396 | - slf4j-api 397 | - snakeyaml 398 | - snappy-java 399 | - spark-catalyst 400 | - spark-core 401 | - spark-ganglia-lgpl 402 | - spark-graphx 403 | - spark-hive 404 | - spark-hive-thriftserver 405 | - spark-kvstore 406 | - spark-launcher 407 | - spark-mllib 408 | - spark-mllib-local 409 | - spark-network-common 410 | - spark-network-shuffle 411 | - spark-repl 412 | - spark-sketch 413 | - spark-sql 414 | - spark-streaming 415 | - spark-tags 416 | - spark-unsafe 417 | - spark-yarn 418 | - spire 419 | - spire-macros 420 | - spire-platform 421 | - spire-util 422 | - ST4 423 | - stax-api 424 | - stream 425 | - super-csv 426 | - threeten-extra 427 | - tink 428 | - transaction-api 429 | - univocity-parsers 430 | - volcano-client 431 | - volcano-model-v1beta1 432 | - xbean-asm9-shaded 433 | - xz 434 | - zjsonpatch 435 | - zookeeper 436 | - zookeeper-jute 437 | - zstd-jni 438 | - !FileStructure 439 | Name: spark-jars 440 | EnvironmentVariableRootLocation: spark_home 441 | RelativeLocation: /jars 442 | FilePrefixes: 443 | - activation 444 | - aggdesigner-algorithm 445 | - aircompressor 446 | - algebra 447 | - all 448 | - annotations 449 | - antlr-runtime 450 | - antlr4-runtime 451 | - aopalliance-repackaged 452 | - arpack_combined_all 453 | - arpack 454 | - arrow-format 455 | - arrow-memory-core 456 | - arrow-memory-netty 457 | - arrow-vector 458 | - audience-annotations 459 | - automaton 460 | - avro 461 | - avro-ipc 462 | - avro-mapred 463 | - blas 464 | - bonecp 465 | - breeze 466 | - breeze-macros 467 | - cats-kernel 468 | - chill 469 | - chill-java 470 | - commons-cli 471 | - commons-codec 472 | - commons-collections 473 | - commons-collections4 474 | - commons-compiler 475 | - commons-compress 476 | - commons-crypto 477 | - commons-dbcp 478 | - commons-io 479 | - commons-lang 480 | - commons-lang3 481 | - commons-logging 482 | - commons-math3 483 | - commons-pool 484 | - commons-text 485 | - compress-lzf 486 | - core 487 | - curator-client 488 | - curator-framework 489 | - curator-recipes 490 | - datanucleus-api-jdo 491 | - datanucleus-core 492 | - datanucleus-rdbms 493 | - derby 494 | - disruptor 495 | - dropwizard-metrics-hadoop-metrics2-reporter 496 | - emr-serverless-spark-goodies 497 | - emr-spark-goodies 498 | - flatbuffers-java 499 | - generex 500 | - gmetric4j 501 | - gson 502 | - guava 503 | - hadoop-client-api 504 | - hadoop-client-runtime 505 | - hadoop-shaded-guava 506 | - hadoop-yarn-server-web-proxy 507 | - HikariCP 508 | - hive-beeline 509 | - hive-cli 510 | - hive-common 511 | - hive-exec 512 | - hive-jdbc 513 | - hive-llap-common 514 | - hive-metastore 515 | - hive-serde 516 | - hive-service-rpc 517 | - hive-shims 518 | - hive-shims 519 | - hive-shims-common 520 | - hive-shims-scheduler 521 | - hive-storage-api 522 | - hive-vector-code-gen 523 | - hk2-api 524 | - hk2-locator 525 | - hk2-utils 526 | - httpclient 527 | - httpcore 528 | - istack-commons-runtime 529 | - ivy 530 | - jackson-annotations 531 | - jackson-core 532 | - jackson-core-asl 533 | - jackson-databind 534 | - jackson-dataformat-yaml 535 | - jackson-datatype-jsr310 536 | - jackson-mapper-asl 537 | - jackson-module-scala 538 | - jakarta.annotation-api 539 | - jakarta.inject 540 | - jakarta.servlet-api 541 | - jakarta.validation-api 542 | - jakarta.ws.rs-api 543 | - jakarta.xml.bind-api 544 | - janino 545 | - javassist 546 | - javax.jdo 547 | - javax.servlet-api 548 | - javolution 549 | - jaxb-runtime 550 | - jcl-over-slf4j 551 | - jdo-api 552 | - jersey-client 553 | - jersey-common 554 | - jersey-container-servlet 555 | - jersey-container-servlet-core 556 | - jersey-hk2 557 | - jersey-server 558 | - jetty-rewrite 559 | - JLargeArrays 560 | - jline 561 | - jniloader 562 | - joda-time 563 | - jodd-core 564 | - jpam 565 | - json 566 | - json4s-ast 567 | - json4s-core 568 | - json4s-jackson 569 | - json4s-scalap 570 | - jsr305 571 | - jta 572 | - JTransforms 573 | - jul-to-slf4j 574 | - kryo-shaded 575 | - lapack 576 | - leveldbjni-all 577 | - libfb303 578 | - libthrift 579 | - log4j-1.2-api 580 | - log4j-api 581 | - log4j-core 582 | - log4j-slf4j-impl 583 | - logging-interceptor 584 | - lz4-java 585 | - metrics-core 586 | - metrics-graphite 587 | - metrics-jmx 588 | - metrics-json 589 | - metrics-jvm 590 | - minlog 591 | - native_ref-java 592 | - native_system-java 593 | - netlib-native_ref-linux-armhf 594 | - netlib-native_ref-linux-i 595 | - netlib-native_ref-linux-x 596 | - netlib-native_ref-osx-x 597 | - netlib-native_ref-win-i 598 | - netlib-native_ref-win-x 599 | - netlib-native_system-linux-armhf 600 | - netlib-native_system-linux-i 601 | - netlib-native_system-linux-x 602 | - netlib-native_system-osx-x 603 | - netlib-native_system-win-i 604 | - netlib-native_system-win-x 605 | - netty-all 606 | - netty-buffer 607 | - netty-codec 608 | - netty-common 609 | - netty-handler 610 | - netty-resolver 611 | - netty-tcnative-classes 612 | - netty-transport 613 | - netty-transport-classes-epoll 614 | - netty-transport-classes-kqueue 615 | - netty-transport-native-epoll 616 | - netty-transport-native-kqueue 617 | - netty-transport-native-unix-common 618 | - objenesis 619 | - okhttp 620 | - okio 621 | - opencsv 622 | - orc-core 623 | - orc-mapreduce 624 | - orc-shims 625 | - oro 626 | - osgi-resource-locator 627 | - paranamer 628 | - parquet-column 629 | - parquet-common 630 | - parquet-encoding 631 | - parquet-format-structures 632 | - parquet-hadoop 633 | - parquet-jackson 634 | - pickle 635 | - protobuf-java 636 | - py4j 637 | - remotetea-oncrpc 638 | - RoaringBitmap 639 | - rocksdbjni 640 | - scala-collection-compat 641 | - scala-compiler 642 | - scala-library 643 | - scala-parser-combinators 644 | - scala-reflect 645 | - scala-xml 646 | - shapeless 647 | - shims 648 | - slf4j-api 649 | - snakeyaml 650 | - snappy-java 651 | - spark-catalyst 652 | - spark-core 653 | - spark-ganglia-lgpl 654 | - spark-graphx 655 | - spark-hive 656 | - spark-hive-thriftserver 657 | - spark-kvstore 658 | - spark-launcher 659 | - spark-mllib 660 | - spark-mllib-local 661 | - spark-network-common 662 | - spark-network-shuffle 663 | - spark-repl 664 | - spark-sketch 665 | - spark-sql 666 | - spark-streaming 667 | - spark-tags 668 | - spark-unsafe 669 | - spark-yarn 670 | - spire 671 | - spire-macros 672 | - spire-platform 673 | - spire-util 674 | - ST4 675 | - stax-api 676 | - stream 677 | - super-csv 678 | - threeten-extra 679 | - tink 680 | - transaction-api 681 | - univocity-parsers 682 | - velocity 683 | - xbean-asm9-shaded 684 | - xz 685 | - zjsonpatch 686 | - zookeeper 687 | - zookeeper-jute 688 | - zstd-jni 689 | - !FileStructure 690 | Name: hadoop-jars-6.9 691 | RelativeLocation: /usr/lib/hadoop/lib 692 | FilePrefixes: 693 | - accessors-smart 694 | - animal-sniffer-annotations 695 | - asm 696 | - audience-annotations 697 | - avro 698 | - checker-qual 699 | - commons-beanutils 700 | - commons-cli 701 | - commons-codec 702 | - commons-collections 703 | - commons-compress 704 | - commons-configuration2 705 | - commons-daemon 706 | - commons-io 707 | - commons-lang3 708 | - commons-logging 709 | - commons-math3 710 | - commons-net 711 | - commons-text 712 | - curator-client 713 | - curator-framework 714 | - curator-recipes 715 | - dnsjava 716 | - failureaccess 717 | - gson 718 | - guava 719 | - httpclient 720 | - httpcore 721 | - j2objc-annotations 722 | - jackson-annotations 723 | - jackson-core 724 | - jackson-core-asl 725 | - jackson-databind 726 | - jackson-jaxrs 727 | - jackson-mapper-asl 728 | - jackson-xc 729 | - jakarta.activation-api 730 | - javax.servlet-api 731 | - jaxb-api 732 | - jaxb-impl 733 | - jcip-annotations 734 | - jersey-core 735 | - jersey-json 736 | - jersey-server 737 | - jersey-servlet 738 | - jettison 739 | - jetty-http 740 | - jetty-io 741 | - jetty-security 742 | - jetty-server 743 | - jetty-servlet 744 | - jetty-util 745 | - jetty-util-ajax 746 | - jetty-webapp 747 | - jetty-xml 748 | - jsch 749 | - json-smart 750 | - jsp-api 751 | - jsr305 752 | - jsr311-api 753 | - jul-to-slf4j 754 | - kerb-admin 755 | - kerb-client 756 | - kerb-common 757 | - kerb-core 758 | - kerb-crypto 759 | - kerb-identity 760 | - kerb-server 761 | - kerb-simplekdc 762 | - kerb-util 763 | - kerby-asn1 764 | - kerby-config 765 | - kerby-pkix 766 | - kerby-util 767 | - kerby-xdr 768 | - listenablefuture 769 | - metrics-core 770 | - native 771 | - netty 772 | - netty-all 773 | - netty-buffer 774 | - netty-codec 775 | - netty-codec-dns 776 | - netty-codec-haproxy 777 | - netty-codec-http 778 | - netty-codec-http2 779 | - netty-codec-memcache 780 | - netty-codec-mqtt 781 | - netty-codec-redis 782 | - netty-codec-smtp 783 | - netty-codec-socks 784 | - netty-codec-stomp 785 | - netty-codec-xml 786 | - netty-common 787 | - netty-handler 788 | - netty-handler-proxy 789 | - netty-resolver 790 | - netty-resolver-dns 791 | - netty-resolver-dns-classes-macos 792 | - netty-resolver-dns-native-macos 793 | - netty-transport 794 | - netty-transport-classes-epoll 795 | - netty-transport-classes-kqueue 796 | - netty-transport-native-epoll 797 | - netty-transport-native-kqueue 798 | - netty-transport-native-unix-common 799 | - netty-transport-rxtx 800 | - netty-transport-sctp 801 | - netty-transport-udt 802 | - nimbus-jose-jwt 803 | - paranamer 804 | - protobuf-java 805 | - re2j 806 | - reload4j 807 | - slf4j-api 808 | - slf4j-reload4j 809 | - snappy-java 810 | - stax2-api 811 | - token-provider 812 | - woodstox-core 813 | - zookeeper 814 | - zookeeper-d 815 | - zookeeper-jute 816 | - !FileStructure 817 | Name: hadoop-jars-6.10 818 | RelativeLocation: /usr/lib/hadoop/lib 819 | FilePrefixes: 820 | - accessors-smart 821 | - animal-sniffer-annotations 822 | - asm 823 | - avro 824 | - checker-qual 825 | - commons-beanutils 826 | - commons-cli 827 | - commons-codec 828 | - commons-collections 829 | - commons-compress 830 | - commons-configuration2 831 | - commons-daemon 832 | - commons-io 833 | - commons-lang3 834 | - commons-logging 835 | - commons-math3 836 | - commons-net 837 | - commons-text 838 | - curator-client 839 | - curator-framework 840 | - curator-recipes 841 | - dnsjava 842 | - failureaccess 843 | - gson 844 | - guava 845 | - httpclient 846 | - httpcore 847 | - j2objc-annotations 848 | - jackson-annotations 849 | - jackson-core 850 | - jackson-core-asl 851 | - jackson-databind 852 | - jackson-jaxrs 853 | - jackson-mapper-asl 854 | - jackson-xc 855 | - jakarta.activation-api 856 | - javax.servlet-api 857 | - jaxb-api 858 | - jaxb-impl 859 | - jcip-annotations 860 | - jersey-core 861 | - jersey-json 862 | - jersey-server 863 | - jersey-servlet 864 | - jettison 865 | - jetty-http 866 | - jetty-io 867 | - jetty-security 868 | - jetty-server 869 | - jetty-servlet 870 | - jetty-util 871 | - jetty-util-ajax 872 | - jetty-webapp 873 | - jetty-xml 874 | - jsch 875 | - json-smart 876 | - jsp-api 877 | - jsr305 878 | - jsr311-api 879 | - jul-to-slf4j 880 | - kerb-admin 881 | - kerb-client 882 | - kerb-common 883 | - kerb-core 884 | - kerb-crypto 885 | - kerb-identity 886 | - kerb-server 887 | - kerb-simplekdc 888 | - kerb-util 889 | - kerby-asn1 890 | - kerby-config 891 | - kerby-pkix 892 | - kerby-util 893 | - kerby-xdr 894 | - listenablefuture 895 | - metrics-core 896 | - native 897 | - netty 898 | - netty-all 899 | - netty-buffer 900 | - netty-codec 901 | - netty-codec-dns 902 | - netty-codec-haproxy 903 | - netty-codec-http 904 | - netty-codec-http2 905 | - netty-codec-memcache 906 | - netty-codec-mqtt 907 | - netty-codec-redis 908 | - netty-codec-smtp 909 | - netty-codec-socks 910 | - netty-codec-stomp 911 | - netty-codec-xml 912 | - netty-common 913 | - netty-handler 914 | - netty-handler-proxy 915 | - netty-resolver 916 | - netty-resolver-dns 917 | - netty-resolver-dns-classes-macos 918 | - netty-resolver-dns-native-macos 919 | - netty-transport 920 | - netty-transport-classes-epoll 921 | - netty-transport-classes-kqueue 922 | - netty-transport-native-epoll 923 | - netty-transport-native-kqueue 924 | - netty-transport-native-unix-common 925 | - netty-transport-rxtx 926 | - netty-transport-sctp 927 | - netty-transport-udt 928 | - nimbus-jose-jwt 929 | - paranamer 930 | - protobuf-java 931 | - re2j 932 | - reload4j 933 | - slf4j-api 934 | - slf4j-reload4j 935 | - snappy-java 936 | - stax2-api 937 | - token-provider 938 | - woodstox-core 939 | - zookeeper 940 | - zookeeper-d 941 | - !FileStructure 942 | Name: hadoop-files 943 | RelativeLocation: /usr/lib/hadoop 944 | FilePrefixes: 945 | - bin 946 | - client 947 | - etc 948 | - hadoop-aliyun 949 | - hadoop-annotations 950 | - hadoop-archive-logs 951 | - hadoop-archives 952 | - hadoop-auth 953 | - hadoop-aws 954 | - hadoop-azure 955 | - hadoop-azure-datalake 956 | - hadoop-client 957 | - hadoop-common 958 | - hadoop-datajoin 959 | - hadoop-distcp 960 | - hadoop-dynamometer-blockgen 961 | - hadoop-dynamometer-infra 962 | - hadoop-dynamometer-workload 963 | - hadoop-extras 964 | - hadoop-fs2img 965 | - hadoop-gridmix 966 | - hadoop-kafka 967 | - hadoop-kms 968 | - hadoop-minicluster 969 | - hadoop-nfs 970 | - hadoop-openstack 971 | - hadoop-registry 972 | - hadoop-resourceestimator 973 | - hadoop-rumen 974 | - hadoop-shaded-guava 975 | - hadoop-shaded-protobuf_3_7 976 | - hadoop-sls 977 | - hadoop-streaming 978 | - lib 979 | - libexec 980 | - sbin 981 | - !FileStructure 982 | Name: bin-files 983 | RelativeLocation: /usr/bin 984 | FilePrefixes: 985 | - tini 986 | - !FileStructure 987 | Name: spark-bin 988 | EnvironmentVariableRootLocation: spark_home 989 | RelativeLocation: /bin 990 | FilePrefixes: 991 | - find-spark-home 992 | - pyspark 993 | - spark-class 994 | - spark-shell 995 | - spark-sql 996 | - spark-submit 997 | - !FileStructure 998 | Name: java-bin 999 | EnvironmentVariableRootLocation: java_home 1000 | RelativeLocation: /bin 1001 | FilePrefixes: 1002 | - java 1003 | - !FileStructure 1004 | Name: hive-bin-files 1005 | RelativeLocation: /usr/bin 1006 | FilePrefixes: 1007 | - hive 1008 | - !FileStructure 1009 | Name: tez-jars 1010 | RelativeLocation: /usr/lib/tez 1011 | FilePrefixes: 1012 | - hadoop-shim 1013 | - tez-api 1014 | - tez-aux-services 1015 | - tez-build-tools 1016 | - tez-common 1017 | - tez-dag 1018 | - tez-emr-serverless.jar 1019 | - tez-examples 1020 | - tez-history-parser 1021 | - tez-javadoc-tools 1022 | - tez-job-analyzer 1023 | - tez-mapreduce 1024 | - tez-protobuf-history-plugin 1025 | - tez-runtime-internals 1026 | - tez-runtime-library 1027 | - tez-tests 1028 | - tez-ui 1029 | - tez-yarn-timeline-cache-plugin 1030 | - tez-yarn-timeline-history 1031 | - tez-yarn-timeline-history-with-acls 1032 | - tez-yarn-timeline-history-with-fs 1033 | - !FileStructure 1034 | Name: hadoop-yarn-jars 1035 | RelativeLocation: /usr/lib/hadoop-yarn 1036 | FilePrefixes: 1037 | - hadoop-yarn-api 1038 | - hadoop-yarn-applications-distributedshell 1039 | - hadoop-yarn-applications-mawo-core 1040 | - hadoop-yarn-applications-unmanaged-am-launcher 1041 | - hadoop-yarn-client 1042 | - hadoop-yarn-common 1043 | - hadoop-yarn-registry 1044 | - hadoop-yarn-server-applicationhistoryservice 1045 | - hadoop-yarn-server-common 1046 | - hadoop-yarn-server-nodemanager 1047 | - hadoop-yarn-server-resourcemanager 1048 | - hadoop-yarn-server-router 1049 | - hadoop-yarn-server-sharedcachemanager 1050 | - hadoop-yarn-server-tests 1051 | - hadoop-yarn-server-timeline-pluginstorage 1052 | - hadoop-yarn-server-web-proxy 1053 | - hadoop-yarn-services-api 1054 | - hadoop-yarn-services-core 1055 | - !FileStructure 1056 | Name: hive-jars-6.9 1057 | RelativeLocation: /usr/lib/hive/lib 1058 | FilePrefixes: 1059 | - accessors-smart 1060 | - accumulo-core 1061 | - accumulo-fate 1062 | - accumulo-start 1063 | - accumulo-trace 1064 | - activation 1065 | - aircompressor 1066 | - all 1067 | - ant 1068 | - ant-launcher 1069 | - antlr-runtime 1070 | - antlr4-runtime 1071 | - aopalliance-repackaged 1072 | - apache-curator 1073 | - apache-jsp 1074 | - apache-jstl 1075 | - arpack_combined_all 1076 | - arrow-format 1077 | - arrow-memory 1078 | - arrow-vector 1079 | - asm 1080 | - asm-analysis 1081 | - asm-commons 1082 | - asm-tree 1083 | - audience-annotations 1084 | - avatica 1085 | - avro 1086 | - avro-ipc 1087 | - avro-mapred 1088 | - bonecp 1089 | - caffeine 1090 | - checker-qual 1091 | - chill 1092 | - chill-java 1093 | - commons-cli 1094 | - commons-codec 1095 | - commons-collections4 1096 | - commons-compiler 1097 | - commons-compress 1098 | - commons-crypto 1099 | - commons-dbcp 1100 | - commons-io 1101 | - commons-lang 1102 | - commons-lang3 1103 | - commons-logging 1104 | - commons-math3 1105 | - commons-pool 1106 | - commons-vfs2 1107 | - compress-lzf 1108 | - core 1109 | - curator-client 1110 | - curator-framework 1111 | - curator-recipes 1112 | - datanucleus-api-jdo 1113 | - datanucleus-core 1114 | - datanucleus-rdbms 1115 | - derby 1116 | - disruptor 1117 | - dropwizard-metrics-hadoop-metrics2-reporter 1118 | - druid-hdfs-storage 1119 | - error_prone_annotations 1120 | - flatbuffers 1121 | - flatbuffers-java 1122 | - groovy-all 1123 | - gson 1124 | - guava 1125 | - hadoop-shaded-guava 1126 | - hadoop-shaded-protobuf 1127 | - HikariCP 1128 | - hive-accumulo-handler 1129 | - hive-beeline 1130 | - hive-classification 1131 | - hive-cli 1132 | - hive-common 1133 | - hive-druid-handler 1134 | - hive-exec 1135 | - hive-hcatalog-core 1136 | - hive-hcatalog-server-extensions 1137 | - hive-hplsql 1138 | - hive-jdbc 1139 | - hive-jdbc-handler 1140 | - hive-kryo-registrator 1141 | - hive-llap-client 1142 | - hive-llap-common 1143 | - hive-llap-ext-client 1144 | - hive-llap-server 1145 | - hive-llap-tez 1146 | - hive-metastore 1147 | - hive-serde 1148 | - hive-service 1149 | - hive-service-rpc 1150 | - hive-shims 1151 | - hive-shims-common 1152 | - hive-shims-scheduler 1153 | - hive-spark-client 1154 | - hive-standalone-metastore 1155 | - hive-storage-api 1156 | - hive-streaming 1157 | - hive-testutils 1158 | - hive-upgrade-acid 1159 | - hive-vector-code-gen 1160 | - hk2-api 1161 | - hk2-locator 1162 | - hk2-utils 1163 | - hppc 1164 | - htrace-core 1165 | - htrace-core4 1166 | - httpclient 1167 | - httpcore 1168 | - ivy 1169 | - jackson-annotations 1170 | - jackson-bom 1171 | - jackson-core 1172 | - jackson-core-asl 1173 | - jackson-databind 1174 | - jackson-dataformat-smile 1175 | - jackson-mapper-asl 1176 | - jackson-module-scala 1177 | - jakarta.annotation-api 1178 | - jakarta.inject 1179 | - jakarta.servlet-api 1180 | - jakarta.validation-api 1181 | - jamon-runtime 1182 | - janino 1183 | - JavaEWAH 1184 | - javassist 1185 | - javax.activation-api 1186 | - javax.annotation-api 1187 | - javax.jdo 1188 | - javax.servlet-api 1189 | - javax.servlet.jsp 1190 | - javax.servlet.jsp-api 1191 | - javolution 1192 | - jaxb-api 1193 | - jcodings 1194 | - jcommander 1195 | - jdo-api 1196 | - jersey-hk2 1197 | - jettison 1198 | - jetty-annotations 1199 | - jetty-client 1200 | - jetty-http 1201 | - jetty-io 1202 | - jetty-jaas 1203 | - jetty-jndi 1204 | - jetty-plus 1205 | - jetty-rewrite 1206 | - jetty-runner 1207 | - jetty-schemas 1208 | - jetty-security 1209 | - jetty-server 1210 | - jetty-servlet 1211 | - jetty-util 1212 | - jetty-util-ajax 1213 | - jetty-webapp 1214 | - jetty-xml 1215 | - jline 1216 | - jniloader 1217 | - joda-time 1218 | - jodd-core 1219 | - joni 1220 | - jpam 1221 | - json 1222 | - json-path 1223 | - json-smart 1224 | - json4s-ast 1225 | - json4s-core 1226 | - json4s-jackson 1227 | - json4s-scalap 1228 | - jsr305 1229 | - jta 1230 | - kryo-shaded 1231 | - libfb303 1232 | - libthrift 1233 | - log4j-1.2-api 1234 | - log4j-api 1235 | - log4j-core 1236 | - log4j-slf4j-impl 1237 | - log4j-web 1238 | - lz4-java 1239 | - mariadb-connector-java 1240 | - metrics-core 1241 | - metrics-graphite 1242 | - metrics-jmx 1243 | - metrics-json 1244 | - metrics-jvm 1245 | - minlog 1246 | - mysql-metadata-storage 1247 | - native_ref-java 1248 | - native_system-java 1249 | - netlib-native_ref-linux-armhf 1250 | - netlib-native_ref-linux-i686 1251 | - netlib-native_ref-linux-x86_64 1252 | - netlib-native_ref-osx-x86_64 1253 | - netlib-native_ref-win-i686 1254 | - netlib-native_ref-win-x86_64 1255 | - netlib-native_system-linux-armhf 1256 | - netlib-native_system-linux-i686 1257 | - netlib-native_system-linux-x86_64 1258 | - netlib-native_system-osx-x86_64 1259 | - netlib-native_system-win-i686 1260 | - netlib-native_system-win-x86_64 1261 | - netty 1262 | - netty-all 1263 | - netty-buffer 1264 | - netty-codec 1265 | - netty-common 1266 | - netty-handler 1267 | - netty-resolver 1268 | - netty-transport 1269 | - netty-transport-classes-epoll 1270 | - netty-transport-native-epoll 1271 | - netty-transport-native-unix-common 1272 | - objenesis 1273 | - opencsv 1274 | - orc-core 1275 | - orc-shims 1276 | - orc-tools 1277 | - org.abego.treelayout.core 1278 | - paranamer 1279 | - parquet-hadoop-bundle 1280 | - php 1281 | - pickle 1282 | - postgresql 1283 | - postgresql-metadata-storage 1284 | - protobuf-java 1285 | - py 1286 | - py4j 1287 | - re2j 1288 | - RoaringBitmap 1289 | - rocksdbjni 1290 | - scala-library 1291 | - scala-reflect 1292 | - scala-xml 1293 | - shims 1294 | - snakeyaml 1295 | - snappy-java 1296 | - spark-core 1297 | - spark-kvstore 1298 | - spark-launcher 1299 | - spark-network-common 1300 | - spark-network-shuffle 1301 | - spark-tags 1302 | - spark-unsafe 1303 | - sqlline 1304 | - ST4 1305 | - stax-api 1306 | - stream 1307 | - super-csv 1308 | - taglibs-standard-impl 1309 | - taglibs-standard-spec 1310 | - tempus-fugit 1311 | - threetenbp 1312 | - tink 1313 | - transaction-api 1314 | - unused 1315 | - velocity 1316 | - websocket-api 1317 | - websocket-client 1318 | - websocket-common 1319 | - websocket-server 1320 | - websocket-servlet 1321 | - xbean-asm9-shaded 1322 | - xz 1323 | - zookeeper 1324 | - zstd-jni 1325 | - !FileStructure 1326 | Name: hive-jars-6.10 1327 | RelativeLocation: /usr/lib/hive/lib 1328 | FilePrefixes: 1329 | - accessors-smart 1330 | - accumulo-core 1331 | - accumulo-fate 1332 | - accumulo-start 1333 | - accumulo-trace 1334 | - activation 1335 | - aircompressor 1336 | - all 1337 | - annotations 1338 | - ant 1339 | - ant-launcher 1340 | - antlr-runtime 1341 | - antlr4-runtime 1342 | - aopalliance-repackaged 1343 | - apache-curator 1344 | - apache-jsp 1345 | - apache-jstl 1346 | - arpack_combined_all 1347 | - arrow-format 1348 | - arrow-memory 1349 | - arrow-vector 1350 | - asm 1351 | - asm-analysis 1352 | - asm-commons 1353 | - asm-tree 1354 | - audience-annotations 1355 | - auth 1356 | - avatica 1357 | - avro 1358 | - avro-ipc 1359 | - avro-mapred 1360 | - aws-core 1361 | - aws-java-sdk-emrwal 1362 | - aws-json-protocol 1363 | - bonecp 1364 | - caffeine 1365 | - checker-qual 1366 | - chill 1367 | - chill-java 1368 | - commons-cli 1369 | - commons-codec 1370 | - commons-collections4 1371 | - commons-compiler 1372 | - commons-compress 1373 | - commons-crypto 1374 | - commons-dbcp 1375 | - commons-io 1376 | - commons-lang 1377 | - commons-lang3 1378 | - commons-logging 1379 | - commons-math3 1380 | - commons-pool 1381 | - commons-vfs2 1382 | - compress-lzf 1383 | - core 1384 | - curator-client 1385 | - curator-framework 1386 | - curator-recipes 1387 | - datanucleus-api-jdo 1388 | - datanucleus-core 1389 | - datanucleus-rdbms 1390 | - derby 1391 | - disruptor 1392 | - dropwizard-metrics-hadoop-metrics2-reporter 1393 | - druid-hdfs-storage 1394 | - endpoints-spi 1395 | - error_prone_annotations 1396 | - eventstream 1397 | - flatbuffers 1398 | - flatbuffers-java 1399 | - groovy-all 1400 | - gson 1401 | - guava 1402 | - hadoop-shaded-guava 1403 | - hadoop-shaded-protobuf 1404 | - HikariCP 1405 | - hive-accumulo-handler 1406 | - hive-beeline 1407 | - hive-classification 1408 | - hive-cli 1409 | - hive-common 1410 | - hive-druid-handler 1411 | - hive-exec 1412 | - hive-hcatalog-core 1413 | - hive-hcatalog-server-extensions 1414 | - hive-hplsql 1415 | - hive-jdbc 1416 | - hive-jdbc-handler 1417 | - hive-kryo-registrator 1418 | - hive-llap-client 1419 | - hive-llap-common 1420 | - hive-llap-ext-client 1421 | - hive-llap-server 1422 | - hive-llap-tez 1423 | - hive-metastore 1424 | - hive-serde 1425 | - hive-service 1426 | - hive-service-rpc 1427 | - hive-shims 1428 | - hive-shims-common 1429 | - hive-shims-scheduler 1430 | - hive-spark-client 1431 | - hive-standalone-metastore 1432 | - hive-storage-api 1433 | - hive-streaming 1434 | - hive-testutils 1435 | - hive-upgrade-acid 1436 | - hive-vector-code-gen 1437 | - hk2-api 1438 | - hk2-locator 1439 | - hk2-utils 1440 | - hppc 1441 | - htrace-core 1442 | - htrace-core4 1443 | - http-client-spi 1444 | - httpclient 1445 | - httpcore 1446 | - ivy 1447 | - jackson-annotations 1448 | - jackson-bom 1449 | - jackson-core 1450 | - jackson-core-asl 1451 | - jackson-databind 1452 | - jackson-dataformat-smile 1453 | - jackson-mapper-asl 1454 | - jackson-module-scala 1455 | - jakarta.annotation-api 1456 | - jakarta.inject 1457 | - jakarta.servlet-api 1458 | - jakarta.validation-api 1459 | - jamon-runtime 1460 | - janino 1461 | - JavaEWAH 1462 | - javassist 1463 | - javax.activation-api 1464 | - javax.annotation-api 1465 | - javax.jdo 1466 | - javax.servlet-api 1467 | - javax.servlet.jsp 1468 | - javax.servlet.jsp-api 1469 | - javolution 1470 | - jaxb-api 1471 | - jcodings 1472 | - jcommander 1473 | - jdo-api 1474 | - jersey-hk2 1475 | - jettison 1476 | - jetty-annotations 1477 | - jetty-client 1478 | - jetty-http 1479 | - jetty-io 1480 | - jetty-jaas 1481 | - jetty-jndi 1482 | - jetty-plus 1483 | - jetty-rewrite 1484 | - jetty-runner 1485 | - jetty-schemas 1486 | - jetty-security 1487 | - jetty-server 1488 | - jetty-servlet 1489 | - jetty-util 1490 | - jetty-util-ajax 1491 | - jetty-webapp 1492 | - jetty-xml 1493 | - jline 1494 | - jniloader 1495 | - joda-time 1496 | - jodd-core 1497 | - joni 1498 | - jpam 1499 | - json 1500 | - json-path 1501 | - json-smart 1502 | - json-utils 1503 | - json4s-ast 1504 | - json4s-core 1505 | - json4s-jackson 1506 | - json4s-scalap 1507 | - jsr305 1508 | - jta 1509 | - kryo-shaded 1510 | - libfb303 1511 | - libthrift 1512 | - log4j-1.2-api 1513 | - log4j-api 1514 | - log4j-core 1515 | - log4j-slf4j-impl 1516 | - log4j-web 1517 | - lz4-java 1518 | - mariadb-connector-java 1519 | - metrics-core 1520 | - metrics-graphite 1521 | - metrics-jmx 1522 | - metrics-json 1523 | - metrics-jvm 1524 | - metrics-spi 1525 | - minlog 1526 | - mysql-metadata-storage 1527 | - native_ref-java 1528 | - native_system-java 1529 | - netlib-native_ref-linux-armhf 1530 | - netlib-native_ref-linux-i686 1531 | - netlib-native_ref-linux-x86_64 1532 | - netlib-native_ref-osx-x86_64 1533 | - netlib-native_ref-win-i686 1534 | - netlib-native_ref-win-x86_64 1535 | - netlib-native_system-linux-armhf 1536 | - netlib-native_system-linux-i686 1537 | - netlib-native_system-linux-x86_64 1538 | - netlib-native_system-osx-x86_64 1539 | - netlib-native_system-win-i686 1540 | - netlib-native_system-win-x86_64 1541 | - netty 1542 | - netty-all 1543 | - netty-buffer 1544 | - netty-codec 1545 | - netty-codec-http 1546 | - netty-codec-http2 1547 | - netty-common 1548 | - netty-handler 1549 | - netty-nio-client 1550 | - netty-resolver 1551 | - netty-transport 1552 | - netty-transport-classes-epoll 1553 | - netty-transport-native-unix-common 1554 | - objenesis 1555 | - opencsv 1556 | - orc-core 1557 | - orc-shims 1558 | - orc-tools 1559 | - org.abego.treelayout.core 1560 | - paranamer 1561 | - parquet-hadoop-bundle 1562 | - php 1563 | - pickle 1564 | - postgresql 1565 | - postgresql-metadata-storage 1566 | - profiles 1567 | - protobuf-java 1568 | - protocol-core 1569 | - py 1570 | - py4j 1571 | - re2j 1572 | - reactive-streams 1573 | - regions 1574 | - RoaringBitmap 1575 | - rocksdbjni 1576 | - scala-library 1577 | - scala-reflect 1578 | - scala-xml 1579 | - sdk-core 1580 | - shims 1581 | - snakeyaml 1582 | - snappy-java 1583 | - spark-core 1584 | - spark-kvstore 1585 | - spark-launcher 1586 | - spark-network-common 1587 | - spark-network-shuffle 1588 | - spark-tags 1589 | - spark-unsafe 1590 | - sqlline 1591 | - ST4 1592 | - stax-api 1593 | - stream 1594 | - super-csv 1595 | - taglibs-standard-impl 1596 | - taglibs-standard-spec 1597 | - tempus-fugit 1598 | - third-party-jackson-core 1599 | - threetenbp 1600 | - tink 1601 | - transaction-api 1602 | - unused 1603 | - utils 1604 | - velocity 1605 | - websocket-api 1606 | - websocket-client 1607 | - websocket-common 1608 | - websocket-server 1609 | - websocket-servlet 1610 | - xbean-asm9-shaded 1611 | - xz 1612 | - zookeeper 1613 | - zstd-jni 1614 | - !FileStructure 1615 | Name: hive-jars-6.12 1616 | RelativeLocation: /usr/lib/hive/lib 1617 | FilePrefixes: 1618 | - HikariCP 1619 | - JavaEWAH 1620 | - RoaringBitmap 1621 | - ST4 1622 | - accessors-smart 1623 | - accumulo-core 1624 | - accumulo-fate 1625 | - accumulo-start 1626 | - accumulo-trace 1627 | - activation 1628 | - aircompressor 1629 | - annotations 1630 | - ant 1631 | - ant-launcher 1632 | - antlr-runtime 1633 | - antlr4-runtime 1634 | - aopalliance-repackaged 1635 | - apache-curator 1636 | - apache-jsp 1637 | - apache-jstl 1638 | - apacheds-jdbm1 1639 | - arrow-format 1640 | - arrow-memory 1641 | - arrow-vector 1642 | - asm 1643 | - asm-analysis 1644 | - asm-commons 1645 | - asm-tree 1646 | - audience-annotations 1647 | - auth 1648 | - avatica 1649 | - avro 1650 | - avro-ipc 1651 | - avro-mapred 1652 | - aws-core 1653 | - aws-java-sdk-emrwal 1654 | - aws-json-protocol 1655 | - bonecp 1656 | - caffeine 1657 | - checker-qual 1658 | - chill-java 1659 | - chill_2.12 1660 | - commons-cli 1661 | - commons-codec 1662 | - commons-collections4 1663 | - commons-compiler 1664 | - commons-compress 1665 | - commons-crypto 1666 | - commons-dbcp 1667 | - commons-io 1668 | - commons-lang 1669 | - commons-lang3 1670 | - commons-logging 1671 | - commons-math3 1672 | - commons-pool 1673 | - commons-vfs2 1674 | - compress-lzf 1675 | - curator-client 1676 | - curator-framework 1677 | - curator-recipes 1678 | - datanucleus-api-jdo 1679 | - datanucleus-core 1680 | - datanucleus-rdbms 1681 | - derby 1682 | - disruptor 1683 | - dropwizard-metrics-hadoop-metrics2-reporter 1684 | - druid-hdfs-storage 1685 | - endpoints-spi 1686 | - error_prone_annotations 1687 | - eventstream 1688 | - flatbuffers-1.2.0-3f79e055.jar 1689 | - flatbuffers-java 1690 | - groovy-all 1691 | - gson 1692 | - guava 1693 | - hadoop-shaded-guava 1694 | - hadoop-shaded-protobuf 1695 | - hive-accumulo-handler 1696 | - hive-beeline 1697 | - hive-classification 1698 | - hive-cli 1699 | - hive-common 1700 | - hive-druid-handler 1701 | - hive-exec 1702 | - hive-hcatalog-core 1703 | - hive-hcatalog-server-extensions 1704 | - hive-hplsql 1705 | - hive-jdbc 1706 | - hive-jdbc-handler 1707 | - hive-kryo-registrator 1708 | - hive-llap-client 1709 | - hive-llap-common 1710 | - hive-llap-ext-client 1711 | - hive-llap-server 1712 | - hive-llap-tez 1713 | - hive-metastore 1714 | - hive-serde 1715 | - hive-service 1716 | - hive-service-rpc 1717 | - hive-shims 1718 | - hive-shims-common 1719 | - hive-shims-scheduler 1720 | - hive-spark-client 1721 | - hive-standalone-metastore 1722 | - hive-storage-api 1723 | - hive-streaming 1724 | - hive-testutils 1725 | - hive-upgrade-acid 1726 | - hive-vector-code-gen 1727 | - hk2-api 1728 | - hk2-locator 1729 | - hk2-utils 1730 | - hppc 1731 | - htrace-core 1732 | - htrace-core4 1733 | - http-client-spi 1734 | - httpclient 1735 | - httpcore 1736 | - ivy 1737 | - jackson-annotations 1738 | - jackson-bom 1739 | - jackson-core 1740 | - jackson-core-asl 1741 | - jackson-databind 1742 | - jackson-dataformat-smile 1743 | - jackson-mapper-asl 1744 | - jackson-module-scala 1745 | - jakarta.annotation-api 1746 | - jakarta.inject 1747 | - jakarta.servlet-api 1748 | - jakarta.validation-api 1749 | - jamon-runtime 1750 | - janino 1751 | - javassist 1752 | - javax.activation-api 1753 | - javax.annotation-api 1754 | - javax.jdo 1755 | - javax.servlet-api 1756 | - javax.servlet.jsp 1757 | - javax.servlet.jsp-api 1758 | - javolution 1759 | - jaxb-api 1760 | - jcodings 1761 | - jcommander 1762 | - jdo-api 1763 | - jersey-hk2 1764 | - jettison 1765 | - jetty-annotations 1766 | - jetty-client 1767 | - jetty-http 1768 | - jetty-io 1769 | - jetty-jaas 1770 | - jetty-jndi 1771 | - jetty-plus 1772 | - jetty-rewrite 1773 | - jetty-runner 1774 | - jetty-schemas 1775 | - jetty-security 1776 | - jetty-server 1777 | - jetty-servlet 1778 | - jetty-util 1779 | - jetty-util-ajax 1780 | - jetty-webapp 1781 | - jetty-xml 1782 | - jline 1783 | - joda-time 1784 | - jodd-core 1785 | - joni 1786 | - jpam 1787 | - json 1788 | - json-path 1789 | - json-smart 1790 | - json-utils 1791 | - json4s-ast 1792 | - json4s-core 1793 | - json4s-jackson 1794 | - json4s-scalap 1795 | - jsr305 1796 | - jta 1797 | - kryo-shaded 1798 | - libfb303 1799 | - libthrift 1800 | - log4j-1.2-api 1801 | - log4j-api 1802 | - log4j-core 1803 | - log4j-slf4j-impl 1804 | - log4j-web 1805 | - lz4-java 1806 | - mariadb-connector-java 1807 | - metrics-core 1808 | - metrics-graphite 1809 | - metrics-jmx 1810 | - metrics-json 1811 | - metrics-jvm 1812 | - metrics-spi 1813 | - minlog 1814 | - mysql-metadata-storage 1815 | - netty 1816 | - netty-all 1817 | - netty-buffer 1818 | - netty-codec 1819 | - netty-codec-http 1820 | - netty-codec-http2 1821 | - netty-common 1822 | - netty-handler 1823 | - netty-nio-client 1824 | - netty-resolver 1825 | - netty-transport 1826 | - netty-transport-classes-epoll 1827 | - netty-transport-classes-kqueue 1828 | - netty-transport-native-epoll 1829 | - netty-transport-native-kqueue 1830 | - netty-transport-native-unix-common 1831 | - objenesis 1832 | - opencsv 1833 | - orc-core 1834 | - orc-shims 1835 | - orc-tools 1836 | - org.abego.treelayout.core 1837 | - paranamer 1838 | - parquet-hadoop-bundle 1839 | - php 1840 | - pickle 1841 | - postgresql 1842 | - postgresql-metadata-storage 1843 | - profiles 1844 | - protobuf-java 1845 | - protocol-core 1846 | - py 1847 | - py4j 1848 | - re2j 1849 | - reactive-streams 1850 | - regions 1851 | - scala-library 1852 | - scala-reflect 1853 | - scala-xml 1854 | - sdk-core 1855 | - shims 1856 | - snappy-java 1857 | - spark-core 1858 | - spark-kvstore 1859 | - spark-launcher 1860 | - spark-network-common 1861 | - spark-network-shuffle 1862 | - spark-tags 1863 | - spark-unsafe 1864 | - sqlline 1865 | - stax-api 1866 | - stream 1867 | - super-csv 1868 | - taglibs-standard-impl 1869 | - taglibs-standard-spec 1870 | - tempus-fugit 1871 | - third-party-jackson-core 1872 | - threetenbp 1873 | - tink 1874 | - transaction-api 1875 | - utils 1876 | - velocity 1877 | - websocket-api 1878 | - websocket-client 1879 | - websocket-common 1880 | - websocket-server 1881 | - websocket-servlet 1882 | - xbean-asm9-shaded 1883 | - xz 1884 | - zookeeper 1885 | - zstd-jni --------------------------------------------------------------------------------