item);
115 | }
116 | }),
117 | input.onDidChangeSelection((items) => resolve(items[0])),
118 | input.onDidHide(() => {
119 | (async () => {
120 | reject(
121 | shouldResume && (await shouldResume())
122 | ? InputFlowAction.resume
123 | : InputFlowAction.cancel
124 | );
125 | })().catch(reject);
126 | })
127 | );
128 | if (this.current) {
129 | this.current.dispose();
130 | }
131 | this.current = input;
132 | this.current.show();
133 | });
134 | } finally {
135 | disposables.forEach((d) => d.dispose());
136 | }
137 | }
138 |
139 | async showInputBox({
140 | title,
141 | step,
142 | totalSteps,
143 | value,
144 | prompt,
145 | validate,
146 | buttons,
147 | shouldResume,
148 | ignoreFocusOut,
149 | }: P) {
150 | const disposables: Disposable[] = [];
151 | try {
152 | return await new Promise<
153 | string | (P extends { buttons: (infer I)[] } ? I : never)
154 | >((resolve, reject) => {
155 | const input = window.createInputBox();
156 | input.title = title;
157 | input.step = step;
158 | input.totalSteps = totalSteps;
159 | input.value = value || "";
160 | input.prompt = prompt;
161 | input.buttons = [
162 | ...(this.steps.length > 1 ? [QuickInputButtons.Back] : []),
163 | ...(buttons || []),
164 | ];
165 | input.ignoreFocusOut = ignoreFocusOut ? ignoreFocusOut : false;
166 | let validating = validate("");
167 | disposables.push(
168 | input.onDidTriggerButton((item) => {
169 | if (item === QuickInputButtons.Back) {
170 | reject(InputFlowAction.back);
171 | } else {
172 | resolve(item);
173 | }
174 | }),
175 | input.onDidAccept(async () => {
176 | const value = input.value;
177 | input.enabled = false;
178 | input.busy = true;
179 | if (!(await validate(value))) {
180 | resolve(value);
181 | }
182 | input.enabled = true;
183 | input.busy = false;
184 | }),
185 | input.onDidChangeValue(async (text) => {
186 | const current = validate(text);
187 | validating = current;
188 | const validationMessage = await current;
189 | if (current === validating) {
190 | input.validationMessage = validationMessage;
191 | }
192 | }),
193 | input.onDidHide(() => {
194 | (async () => {
195 | reject(
196 | shouldResume && (await shouldResume())
197 | ? InputFlowAction.resume
198 | : InputFlowAction.cancel
199 | );
200 | })().catch(reject);
201 | })
202 | );
203 | if (this.current) {
204 | this.current.dispose();
205 | }
206 | this.current = input;
207 | this.current.show();
208 | });
209 | } finally {
210 | disposables.forEach((d) => d.dispose());
211 | }
212 | }
213 | }
214 |
--------------------------------------------------------------------------------
/src/extension.ts:
--------------------------------------------------------------------------------
1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 | // SPDX-License-Identifier: Apache-2.0
3 |
4 | // The module 'vscode' contains the VS Code extensibility API
5 | // Import the module and reference it with the alias vscode in your code below
6 | import path = require("path");
7 | import * as vscode from "vscode";
8 | import { AwsContextCommands } from "./aws_context";
9 | import { DefaultEMRClient } from "./clients/emrClient";
10 | import { DefaultEMRContainersClient } from "./clients/emrContainersClient";
11 | import { DefaultEMRServerlessClient } from "./clients/emrServerlessClient";
12 | import { DefaultGlueClient } from "./clients/glueClient";
13 | import { DefaultS3Client } from "./clients/s3Client";
14 | import { EMREC2Deploy } from "./commands/deploy/emrEC2Deploy";
15 | import { EMRServerlessDeploy } from "./commands/emrDeploy";
16 | import { EMREC2Filter } from "./emr_explorer";
17 | import { EMRLocalEnvironment } from "./emr_local";
18 | import { copyIdCommand } from "./explorer/commands";
19 | import { EMRContainersNode } from "./explorer/emrContainers";
20 | import { EMRNode } from "./explorer/emrEC2";
21 | import { EMRServerlessNode } from "./explorer/emrServerless";
22 | import { GlueCatalogNode } from "./explorer/glueCatalog";
23 | import { getWebviewContent } from "./panels/glueTablePanel";
24 |
25 |
26 | // Workaround for https://github.com/aws/aws-sdk-js-v3/issues/3807
27 | declare global {
28 | interface ReadableStream {}
29 | }
30 |
31 | // We create a global namespace for common variables
32 | export interface Globals {
33 | context: vscode.ExtensionContext;
34 | outputChannel: vscode.OutputChannel;
35 | awsContext: AwsContextCommands;
36 | selectedRegion: string;
37 | selectedProfile: string;
38 | }
39 | const globals = {} as Globals;
40 | export { globals };
41 |
42 | // this method is called when your extension is activated
43 | // your extension is activated the very first time the command is executed
44 | export function activate(context: vscode.ExtensionContext) {
45 | const logger = vscode.window.createOutputChannel("Amazon EMR");
46 | globals.outputChannel = logger;
47 |
48 | // Allow users to set profile and region
49 | const awsContext = new AwsContextCommands();
50 | globals.awsContext = awsContext;
51 |
52 | // Allow other modules to access vscode context
53 | globals.context = context;
54 |
55 | context.subscriptions.push(
56 | vscode.commands.registerCommand("emr-tools-v2.selectProfile", async () => {
57 | await awsContext.onCommandSetProfile();
58 | })
59 | );
60 |
61 | context.subscriptions.push(
62 | vscode.commands.registerCommand("emr-tools-v2.selectRegion", async () => {
63 | await awsContext.onCommandSetRegion();
64 | })
65 | );
66 |
67 | const treeFilter = new EMREC2Filter();
68 | context.subscriptions.push(
69 | vscode.commands.registerCommand("emr-tools-v2.filterClusters", async () => {
70 | await treeFilter.run();
71 | })
72 | );
73 |
74 | // EMR on EC2 support
75 | const emrEC2Client = new DefaultEMRClient(globals);
76 | const emrExplorer = new EMRNode(emrEC2Client, treeFilter);
77 | vscode.window.registerTreeDataProvider("emrExplorer", emrExplorer);
78 | vscode.commands.registerCommand("emr-tools-v2.refreshEntry", () =>
79 | emrExplorer.refresh()
80 | );
81 |
82 | // Tree data providers
83 | // const emrTools = new EMREC2Provider(
84 | // vscode.workspace.rootPath + "",
85 | // treeFilter,
86 | // logger
87 | // );
88 | // vscode.window.registerTreeDataProvider("emrExplorer", emrTools);
89 | // vscode.commands.registerCommand("emr-tools-v2.refreshEntry", () =>
90 | // emrTools.refresh()
91 | // );
92 | // vscode.commands.registerCommand(
93 | // "emr-tools-v2.connectToCluster",
94 | // async (cluster: EMRCluster) => {
95 | // await connectToClusterCommand(cluster);
96 | // }
97 | // );
98 | context.subscriptions.push(
99 | vscode.commands.registerCommand(
100 | "emr-tools-v2.copyId",
101 | async (node: vscode.TreeItem) => await copyIdCommand(node)
102 | )
103 | );
104 |
105 | context.subscriptions.push(
106 | vscode.commands.registerCommand(
107 | "emr-tools-v2.viewGlueTable",
108 | async (node: vscode.TreeItem) => {
109 | const panel = vscode.window.createWebviewPanel(
110 | "glue-table", node.id!.split(" ").reverse().join("."),
111 | vscode.ViewColumn.One,
112 | {
113 | enableScripts: true,
114 | enableFindWidget: true
115 | });
116 |
117 | panel.webview.html = await getWebviewContent(node, new DefaultGlueClient(globals), context.extensionUri, panel.webview);}
118 | )
119 | );
120 |
121 | // EMR on EKS support
122 | // const emrContainerTools = new EMRContainersProvider(globals);
123 | const emrContainerExplorer = new EMRContainersNode(
124 | new DefaultEMRContainersClient(globals)
125 | );
126 | vscode.window.registerTreeDataProvider(
127 | "emrContainersExplorer",
128 | emrContainerExplorer
129 | );
130 | vscode.commands.registerCommand("emr-tools-v2.refreshContainerEntry", () =>
131 | emrContainerExplorer.refresh()
132 | );
133 |
134 | // Glue support
135 | const glueCatalogExplorer = new GlueCatalogNode(
136 | new DefaultGlueClient(globals)
137 | );
138 | vscode.window.registerTreeDataProvider(
139 | "glueCatalogExplorer",
140 | glueCatalogExplorer
141 | );
142 |
143 | vscode.commands.registerCommand("emr-tools-v2.refreshGlueCatalog", () =>
144 | glueCatalogExplorer.refresh()
145 | );
146 |
147 | // EMR Serverless support
148 | // const emrServerlessTools = new EMRServerlessProvider();
149 | const emrServerlessClient = new DefaultEMRServerlessClient(globals);
150 | const emrServerlessTools = new EMRServerlessNode(emrServerlessClient);
151 | vscode.window.registerTreeDataProvider(
152 | "emrServerlessExplorer",
153 | emrServerlessTools
154 | );
155 | vscode.commands.registerCommand("emr-tools-v2.refreshServerlessEntry", () =>
156 | emrServerlessTools.refresh()
157 | );
158 |
159 | // When the region changes, refresh all our explorers
160 | globals.awsContext.onDidConfigChange(() => {
161 | emrExplorer.refresh();
162 | emrContainerExplorer.refresh();
163 | emrServerlessTools.refresh();
164 | glueCatalogExplorer.refresh();
165 | });
166 |
167 | const s3Client = new DefaultS3Client(globals);
168 | const emrServerlessDeployer = new EMRServerlessDeploy(context, emrServerlessClient, s3Client);
169 | context.subscriptions.push(
170 | vscode.commands.registerCommand(
171 | "emr-tools-v2.deployEMRServerless", async() => {
172 | await emrServerlessDeployer.run();
173 | }
174 | )
175 | );
176 |
177 | const emrEC2Deployer = new EMREC2Deploy(context, emrEC2Client, s3Client);
178 | context.subscriptions.push(
179 | vscode.commands.registerCommand(
180 | "emr-tools-v2.deployEMREC2",async() => {
181 | await emrEC2Deployer.run();
182 | }
183 | )
184 | );
185 |
186 | // Deployment support for all our available options
187 | // Removing until future release :)
188 | // context.subscriptions.push(
189 | // vscode.commands.registerCommand(
190 | // "emr-tools-v2.deploy", async () => {
191 | // await new EMRDeployer(emrTools, emrContainerTools, emrServerlessTools).run();
192 | // }
193 | // )
194 | // );
195 |
196 | // Local environment support
197 | const emrLocalCreator = new EMRLocalEnvironment(context);
198 | context.subscriptions.push(
199 | vscode.commands.registerCommand(
200 | "emr-tools-v2.localEnvironmentMagic",
201 | async () => {
202 | await emrLocalCreator.run();
203 | }
204 | )
205 | );
206 |
207 | // Use the console to output diagnostic information (console.log) and errors (console.error)
208 | // This line of code will only be executed once when your extension is activated
209 | console.log('Congratulations, your extension "emr-tools-v2" is now active!');
210 | }
211 | // this method is called when your extension is deactivated
212 | export function deactivate() {}
213 |
--------------------------------------------------------------------------------
/src/commands/deploy/emrEC2Deploy.ts:
--------------------------------------------------------------------------------
1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 | // SPDX-License-Identifier: Apache-2.0
3 |
4 | // We want folks to be able to developer EMR jobs locally.
5 | // We give them an option to create an EMR environment
6 | // They select:
7 | // - Type of job (pyspark, scala, SQL)
8 | // - EMR Release (only those supported by EMR on EKS)
9 | // - Region (used to build the local Image URI)
10 |
11 | import { QuickPickItem } from "vscode";
12 | import { MultiStepInput } from "./../../helpers";
13 | import * as fs from "fs";
14 | import * as vscode from "vscode";
15 | import {
16 | DefaultEMRClient,
17 | ClusterStep,
18 | } from "../../clients/emrClient";
19 | import { DefaultS3Client } from "../../clients/s3Client";
20 | import { pickFile } from "../../utils/quickPickItem";
21 | import { basename } from "path";
22 |
23 | // Step 1, add EMR Deploy option for EMR on EC2
24 | // Command: "EMR on EC2: Deploy and start step"
25 | // Process:
26 | // - Ask for (and save):
27 | // - S3 bucket/prefix for code location
28 | // - IAM Job Role ARN (todo)
29 | // - Copy main entry script to S3
30 | // - call StartJobRunCommand
31 |
32 |
33 | interface State {
34 | title: string;
35 | step: number;
36 | totalSteps: number;
37 | resourceGroup: QuickPickItem | string;
38 |
39 | s3TargetURI: string;
40 | clusterID: string;
41 | jobRoleARN: string;
42 | s3LogTargetURI: string;
43 | srcScriptURI: string;
44 | }
45 |
46 | const TOTAL_STEPS = 3;
47 |
48 | export class EMREC2Deploy {
49 | context: vscode.ExtensionContext;
50 | title: string;
51 | previousClusterID: string | undefined;
52 | previousS3TargetURI: string | undefined;
53 | previousS3LogTargetURI: string | undefined;
54 | previousJobRoleARN: string | undefined;
55 |
56 |
57 |
58 | constructor(
59 | context: vscode.ExtensionContext,
60 | private readonly emr: DefaultEMRClient,
61 | private readonly s3: DefaultS3Client
62 | ) {
63 | this.context = context;
64 | this.title = "Deploy to EMR on EC2";
65 |
66 | this.previousClusterID = undefined;
67 | this.previousS3TargetURI = undefined;
68 | this.previousS3LogTargetURI = undefined;
69 | this.previousJobRoleARN = undefined;
70 | }
71 |
72 |
73 |
74 | async collectInputs() {
75 | const state = {} as Partial;
76 | await MultiStepInput.run((input) => this.insertS3TargetURI(input, state));
77 | return state as State;
78 | }
79 |
80 |
81 |
82 | async insertS3TargetURI(
83 | input: MultiStepInput,
84 | state: Partial
85 | ) {
86 | let defaultTarget = "s3://bucket-name/prefix/";
87 | if (this.previousS3TargetURI) {
88 | defaultTarget = this.previousS3TargetURI;
89 | }
90 | const pick = await input.showInputBox({
91 | title: this.title,
92 | step: 1,
93 | totalSteps: TOTAL_STEPS,
94 | value: defaultTarget,
95 | prompt: "Provide an S3 URI where you want to upload your code.",
96 | validate: this.validateBucketURI,
97 | shouldResume: this.shouldResume,
98 | ignoreFocusOut: true,
99 | });
100 |
101 | state.s3TargetURI = pick.valueOf();
102 | this.previousS3TargetURI = state.s3TargetURI;
103 | return (input: MultiStepInput) => this.selectClusterID(input, state);
104 | }
105 |
106 |
107 | async insertS3LogTargetURI(
108 | input: MultiStepInput,
109 | state: Partial
110 | ) {
111 | let defaultTarget = "s3://bucket-name/logs/";
112 | if (this.previousS3LogTargetURI !== undefined) {
113 | defaultTarget = this.previousS3LogTargetURI;
114 | } else if (state.s3TargetURI) {
115 | let codeBucket = this.extractBucketName(state.s3TargetURI!);
116 | defaultTarget = `s3://${codeBucket}/logs/`;
117 | }
118 | const pick = await input.showInputBox({
119 | title: this.title,
120 | step: 2,
121 | totalSteps: TOTAL_STEPS,
122 | value: defaultTarget,
123 | prompt: "Provide an S3 URI for Spark logs (leave blank to disable).",
124 | validate: this.validateOptionalBucketURI.bind(this),
125 | shouldResume: this.shouldResume,
126 | ignoreFocusOut: true,
127 | });
128 |
129 | state.s3LogTargetURI = pick.valueOf();
130 | this.previousS3LogTargetURI = state.s3LogTargetURI;
131 | return (input: MultiStepInput) => this.selectClusterID(input, state);
132 | }
133 |
134 | async insertJobRoleARN(
135 | input: MultiStepInput,
136 | state: Partial
137 | ) {
138 | let defaultJobRole = this.previousJobRoleARN ? this.previousJobRoleARN : "arn:aws:iam::xxx:role/job-role";
139 | const pick = await input.showInputBox({
140 | title: this.title,
141 | step: 3,
142 | totalSteps: TOTAL_STEPS,
143 | value: defaultJobRole,
144 | prompt:
145 | "Provide an IAM Role that has access to the resources for your job.",
146 | validate: this.validateJobRole,
147 | shouldResume: this.shouldResume,
148 | ignoreFocusOut: true,
149 | });
150 |
151 | state.jobRoleARN = pick.valueOf();
152 | this.previousJobRoleARN = state.jobRoleARN;
153 | return (input: MultiStepInput) => this.selectClusterID(input, state);
154 | }
155 |
156 | async selectClusterID(
157 | input: MultiStepInput,
158 | state: Partial
159 | ) {
160 | let defaultClusterId = this.previousClusterID ? this.previousClusterID : "j-AABBCCDD00112";
161 | // TODO: Populate the list of cluster IDs automatically
162 | const pick = await input.showInputBox({
163 | title: this.title,
164 | step: 2,
165 | totalSteps: TOTAL_STEPS,
166 | value: defaultClusterId,
167 | prompt: "Provide the EMR Cluster ID.",
168 | validate: this.validateClusterID,
169 | shouldResume: this.shouldResume,
170 | ignoreFocusOut: true,
171 | });
172 |
173 | state.clusterID = pick.valueOf();
174 | this.previousClusterID = state.clusterID;
175 | return (input: MultiStepInput) => this.selectSourceFile(input, state);
176 | }
177 |
178 | async selectSourceFile(
179 | input: MultiStepInput,
180 | state: Partial
181 | ) {
182 | const uri = await pickFile("Type the filename with your source code.");
183 | if (uri) {
184 | state.srcScriptURI = uri.fsPath;
185 | }
186 | }
187 |
188 | async validateOptionalBucketURI(uri: string): Promise {
189 | if (uri === "" || uri === undefined) {
190 | return undefined;
191 | }
192 |
193 | return this.validateBucketURI(uri);
194 | }
195 |
196 | async validateBucketURI(uri: string): Promise {
197 | if (!uri.startsWith("s3://")) {
198 | return "S3 location must start with s3://";
199 | }
200 | return undefined;
201 | }
202 |
203 | extractBucketName(uri: string): string {
204 | return uri.split("/")[2];
205 | }
206 |
207 | async validateJobRole(uri: string): Promise {
208 | if (!uri.startsWith("arn:aws:iam::")) {
209 | return "Job role must be a full ARN: arn:aws:iam:::role/";
210 | }
211 | return undefined;
212 | }
213 |
214 | async validateClusterID(
215 | clusterId: string
216 | ): Promise {
217 | if (!clusterId.startsWith("j-")) {
218 | return "Cluster must begin with 'j-'";
219 | }
220 | if (clusterId.length !== 15) {
221 | return "Provide the Cluster ID, like j-AABBCCDD00112";
222 | }
223 | return undefined;
224 | }
225 |
226 | shouldResume() {
227 | // Could show a notification with the option to resume.
228 | return new Promise((resolve, reject) => {
229 | // noop
230 | });
231 | }
232 |
233 | public async run() {
234 | const state = await this.collectInputs();
235 |
236 | const detail = `Entry point: ${state.s3TargetURI}${basename(
237 | state.srcScriptURI
238 | )}\Cluster ID: ${state.clusterID}`;
239 |
240 | const confirmDeployment = await vscode.window
241 | .showInformationMessage(
242 | "Confirm EMR on EC2 deployment",
243 | { modal: true, detail },
244 | "Yes"
245 | )
246 | .then((answer) => {
247 | return answer === "Yes";
248 | });
249 |
250 | if (confirmDeployment) {
251 | await this.deploy(
252 | state.clusterID,
253 | state.jobRoleARN,
254 | state.srcScriptURI,
255 | state.s3TargetURI,
256 | state.s3LogTargetURI,
257 | );
258 | }
259 | }
260 |
261 | private async deploy(
262 | clusterID: string,
263 | executionRoleARN: string,
264 | sourceFile: string,
265 | s3TargetURI: string,
266 | s3LogTargetURI: string,
267 | ) {
268 | const data = fs.readFileSync(sourceFile);
269 | const bucketName = s3TargetURI.split("/")[2];
270 | const key = s3TargetURI.split("/").slice(3).join("/");
271 | const fullS3Key = `${key.replace(/\/$/, '')}/${basename(sourceFile)}`;
272 | const fullS3Path = `s3://${bucketName}/${fullS3Key}`;
273 |
274 | await this.s3.uploadFile(bucketName, fullS3Key, data);
275 |
276 | this.emr.startJobRun(clusterID, fullS3Path);
277 |
278 | vscode.window.showInformationMessage("Your job has been submitted, refresh the EMR view to keep an eye on it.");
279 | }
280 | }
281 |
--------------------------------------------------------------------------------
/src/commands/emrDeploy.ts:
--------------------------------------------------------------------------------
1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 | // SPDX-License-Identifier: Apache-2.0
3 |
4 | // We want folks to be able to developer EMR jobs locally.
5 | // We give them an option to create an EMR environment
6 | // They select:
7 | // - Type of job (pyspark, scala, SQL)
8 | // - EMR Release (only those supported by EMR on EKS)
9 | // - Region (used to build the local Image URI)
10 |
11 | import { QuickPickItem, window } from "vscode";
12 | import { MultiStepInput } from "./../helpers";
13 | import * as fs from "fs";
14 | import * as vscode from "vscode";
15 | import {
16 | DefaultEMRServerlessClient,
17 | JobRun,
18 | } from "../clients/emrServerlessClient";
19 | import { DefaultS3Client } from "../clients/s3Client";
20 | import { pickFile } from "../utils/quickPickItem";
21 | import { basename } from "path";
22 |
23 | // Step 1, add EMR Deploy option for EMR Serverless
24 | // Command: "EMR Serverless: Deploy and start job"
25 | // Process:
26 | // - Ask for (and save):
27 | // - S3 bucket/prefix for code location
28 | // - IAM Job Role ARN
29 | // - S3 log bucket (optional)
30 | // - Copy main entry script to S3
31 | // - Copy any additional .py files to s3
32 | // - call StartJobRunCommand
33 |
34 | // - open explorer view ;)
35 |
36 |
37 | interface State {
38 | title: string;
39 | step: number;
40 | totalSteps: number;
41 | resourceGroup: QuickPickItem | string;
42 |
43 | s3TargetURI: string;
44 | applicationID: string;
45 | jobRoleARN: string;
46 | s3LogTargetURI: string;
47 | srcScriptURI: string;
48 | }
49 |
50 | const TOTAL_STEPS = 5;
51 |
52 | export class EMRServerlessDeploy {
53 | context: vscode.ExtensionContext;
54 | title: string;
55 | previousAppID: string | undefined;
56 | previousS3TargetURI: string | undefined;
57 | previousS3LogTargetURI: string | undefined;
58 | previousJobRoleARN: string | undefined;
59 |
60 |
61 |
62 | constructor(
63 | context: vscode.ExtensionContext,
64 | private readonly emr: DefaultEMRServerlessClient,
65 | private readonly s3: DefaultS3Client
66 | ) {
67 | this.context = context;
68 | this.title = "Deploy to EMR Serverless";
69 |
70 | this.previousAppID = undefined;
71 | this.previousS3TargetURI = undefined;
72 | this.previousS3LogTargetURI = undefined;
73 | this.previousJobRoleARN = undefined;
74 | }
75 |
76 |
77 |
78 | async collectInputs() {
79 | const state = {} as Partial;
80 | await MultiStepInput.run((input) => this.insertS3TargetURI(input, state));
81 | return state as State;
82 | }
83 |
84 |
85 |
86 | async insertS3TargetURI(
87 | input: MultiStepInput,
88 | state: Partial
89 | ) {
90 | let defaultTarget = "s3://bucket-name/prefix/";
91 | if (this.previousS3TargetURI) {
92 | defaultTarget = this.previousS3TargetURI;
93 | }
94 | const pick = await input.showInputBox({
95 | title: this.title,
96 | step: 1,
97 | totalSteps: TOTAL_STEPS,
98 | value: defaultTarget,
99 | prompt: "Provide an S3 URI where you want to upload your code.",
100 | validate: this.validateBucketURI,
101 | shouldResume: this.shouldResume,
102 | ignoreFocusOut: true,
103 | });
104 |
105 | state.s3TargetURI = pick.valueOf();
106 | this.previousS3TargetURI = state.s3TargetURI;
107 | return (input: MultiStepInput) => this.insertS3LogTargetURI(input, state);
108 | }
109 |
110 |
111 | async insertS3LogTargetURI(
112 | input: MultiStepInput,
113 | state: Partial
114 | ) {
115 | let defaultTarget = "s3://bucket-name/logs/";
116 | if (this.previousS3LogTargetURI !== undefined) {
117 | defaultTarget = this.previousS3LogTargetURI;
118 | } else if (state.s3TargetURI) {
119 | let codeBucket = this.extractBucketName(state.s3TargetURI!);
120 | defaultTarget = `s3://${codeBucket}/logs/`;
121 | }
122 | const pick = await input.showInputBox({
123 | title: this.title,
124 | step: 2,
125 | totalSteps: TOTAL_STEPS,
126 | value: defaultTarget,
127 | prompt: "Provide an S3 URI for Spark logs (leave blank to disable).",
128 | validate: this.validateOptionalBucketURI.bind(this),
129 | shouldResume: this.shouldResume,
130 | ignoreFocusOut: true,
131 | });
132 |
133 | state.s3LogTargetURI = pick.valueOf();
134 | this.previousS3LogTargetURI = state.s3LogTargetURI;
135 | return (input: MultiStepInput) => this.insertJobRoleARN(input, state);
136 | }
137 |
138 | async insertJobRoleARN(
139 | input: MultiStepInput,
140 | state: Partial
141 | ) {
142 | let defaultJobRole = this.previousJobRoleARN ? this.previousJobRoleARN : "arn:aws:iam::xxx:role/job-role";
143 | const pick = await input.showInputBox({
144 | title: this.title,
145 | step: 3,
146 | totalSteps: TOTAL_STEPS,
147 | value: defaultJobRole,
148 | prompt:
149 | "Provide an IAM Role that has access to the resources for your job.",
150 | validate: this.validateJobRole,
151 | shouldResume: this.shouldResume,
152 | ignoreFocusOut: true,
153 | });
154 |
155 | state.jobRoleARN = pick.valueOf();
156 | this.previousJobRoleARN = state.jobRoleARN;
157 | return (input: MultiStepInput) => this.selectApplicationID(input, state);
158 | }
159 |
160 | async selectApplicationID(
161 | input: MultiStepInput,
162 | state: Partial
163 | ) {
164 | let defaultAppId = this.previousAppID ? this.previousAppID : "00f3aabbccdd1234";
165 | // TODO: Populate the list of application IDs automatically
166 | const pick = await input.showInputBox({
167 | title: this.title,
168 | step: 4,
169 | totalSteps: TOTAL_STEPS,
170 | value: defaultAppId,
171 | prompt: "Provide the EMR Serverless Application ID.",
172 | validate: this.validateApplicationID,
173 | shouldResume: this.shouldResume,
174 | ignoreFocusOut: true,
175 | });
176 |
177 | state.applicationID = pick.valueOf();
178 | this.previousAppID = state.applicationID;
179 | return (input: MultiStepInput) => this.selectSourceFile(input, state);
180 | }
181 |
182 | async selectSourceFile(
183 | input: MultiStepInput,
184 | state: Partial
185 | ) {
186 | const uri = await pickFile("Type the filename with your source code.");
187 | if (uri) {
188 | state.srcScriptURI = uri.fsPath;
189 | }
190 | }
191 |
192 | async validateOptionalBucketURI(uri: string): Promise {
193 | if (uri === "" || uri === undefined) {
194 | return undefined;
195 | }
196 |
197 | return this.validateBucketURI(uri);
198 | }
199 |
200 | async validateBucketURI(uri: string): Promise {
201 | if (!uri.startsWith("s3://")) {
202 | return "S3 location must start with s3://";
203 | }
204 | return undefined;
205 | }
206 |
207 | extractBucketName(uri: string): string {
208 | return uri.split("/")[2];
209 | }
210 |
211 | async validateJobRole(uri: string): Promise {
212 | if (!uri.startsWith("arn:aws:iam::")) {
213 | return "Job role must be a full ARN: arn:aws:iam:::role/";
214 | }
215 | return undefined;
216 | }
217 |
218 | async validateApplicationID(
219 | appId: string
220 | ): Promise {
221 | if (appId.length !== 16) {
222 | return "Provide just the Application ID, like 00f3ranvrvchl625";
223 | }
224 | return undefined;
225 | }
226 |
227 | shouldResume() {
228 | // Could show a notification with the option to resume.
229 | return new Promise((resolve, reject) => {
230 | // noop
231 | });
232 | }
233 |
234 | public async run() {
235 | const state = await this.collectInputs();
236 |
237 | const detail = `Entry point: ${state.s3TargetURI}${basename(
238 | state.srcScriptURI
239 | )}\nApplication ID: ${state.applicationID}\nJob Role: ${state.jobRoleARN}`;
240 |
241 | const confirmDeployment = await vscode.window
242 | .showInformationMessage(
243 | "Confirm EMR Serverless deployment",
244 | { modal: true, detail },
245 | "Yes"
246 | )
247 | .then((answer) => {
248 | return answer === "Yes";
249 | });
250 |
251 | if (confirmDeployment) {
252 | await this.deploy(
253 | state.applicationID,
254 | state.jobRoleARN,
255 | state.srcScriptURI,
256 | state.s3TargetURI,
257 | state.s3LogTargetURI,
258 | );
259 | }
260 | // Do I do a "deploy" and "run"
261 | }
262 |
263 | private async deploy(
264 | applicationID: string,
265 | executionRoleARN: string,
266 | sourceFile: string,
267 | s3TargetURI: string,
268 | s3LogTargetURI: string,
269 | ) {
270 | const data = fs.readFileSync(sourceFile);
271 | const bucketName = s3TargetURI.split("/")[2];
272 | const key = s3TargetURI.split("/").slice(3).join("/");
273 | const fullS3Key = `${key.replace(/\/$/, '')}/${basename(sourceFile)}`;
274 | const fullS3Path = `s3://${bucketName}/${fullS3Key}`;
275 |
276 | await this.s3.uploadFile(bucketName, fullS3Key, data);
277 |
278 | this.emr.startJobRun(applicationID, executionRoleARN,fullS3Path, s3LogTargetURI);
279 |
280 | vscode.window.showInformationMessage("Your job has been submitted, refresh the EMR Serverless view to keep an eye on it.");
281 | }
282 | }
283 |
--------------------------------------------------------------------------------
/src/emr_explorer.ts:
--------------------------------------------------------------------------------
1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 | // SPDX-License-Identifier: Apache-2.0
3 |
4 | import * as vscode from "vscode";
5 | import {
6 | EMR,
7 | ListClustersCommand,
8 | ClusterState,
9 | DescribeClusterCommand,
10 | Application,
11 | ClusterSummary,
12 | InstanceCollectionType,
13 | ListInstanceGroupsCommand,
14 | ListInstanceFleetsCommand,
15 | InstanceFleetStatus,
16 | Cluster,
17 | ListInstancesCommand,
18 | InstanceTypeSpecification,
19 | InstanceGroupType,
20 | InstanceStateChangeReason,
21 | InstanceState,
22 | InstanceFleetType,
23 | } from "@aws-sdk/client-emr";
24 | import { window } from "vscode";
25 |
26 |
27 | export class EMREC2Provider
28 | implements vscode.TreeDataProvider
29 | {
30 | emrClient: EMR;
31 | private _onDidChangeTreeData: vscode.EventEmitter<
32 | EMRCluster | undefined | null | void
33 | > = new vscode.EventEmitter();
34 | readonly onDidChangeTreeData: vscode.Event<
35 | EMRCluster | undefined | null | void
36 | > = this._onDidChangeTreeData.event;
37 |
38 | refresh(): void {
39 | this._onDidChangeTreeData.fire();
40 | }
41 |
42 | constructor(private workspaceRoot: string, private stateFilter: EMREC2Filter, private logger: vscode.OutputChannel) {
43 | this.emrClient = new EMR({ region: "us-west-2" });
44 | this.stateFilter = stateFilter;
45 | this.logger = logger;
46 | }
47 |
48 | getTreeItem(element: EMRCluster): vscode.TreeItem {
49 | return element;
50 | }
51 |
52 | getChildren(element?: EMRCluster): Thenable {
53 | if (element) {
54 | return Promise.resolve(element.getChildren());
55 | } else {
56 | return Promise.resolve(this.listEMRClusters(this.emrClient));
57 | }
58 | }
59 |
60 | private async listEMRClusters(client: EMR): Promise {
61 | // Currently only show running or waiting clusters
62 | const showStates = this.stateFilter.getStates();
63 | const params = {
64 | // eslint-disable-next-line @typescript-eslint/naming-convention
65 | // ClusterStates: [ClusterState.RUNNING, ClusterState.WAITING, ClusterState.TERMINATED, ClusterState.TERMINATING],
66 | ClusterStates: showStates,
67 | };
68 | this.logger.appendLine("Fetching clusters in state: " + [...showStates].join(", "));
69 | vscode.window.showInformationMessage(
70 | "Fetching clusters in state: " + [...showStates].join(", ")
71 | );
72 | try {
73 | const result = await client.send(new ListClustersCommand(params));
74 | return (result.Clusters || []).map((cluster) => {
75 | return new EMRCluster(
76 | cluster,
77 | this.emrClient,
78 | vscode.TreeItemCollapsibleState.Collapsed
79 | );
80 | });
81 | } catch (e) {
82 | vscode.window.showErrorMessage("Bummer!" + e);
83 | console.log("There was an error fetching clusters", e);
84 | return [];
85 | }
86 | }
87 | }
88 |
89 | export class EMRCluster extends vscode.TreeItem {
90 | constructor(
91 | private readonly details: ClusterSummary,
92 | private readonly emr: EMR,
93 | public readonly collapsibleState: vscode.TreeItemCollapsibleState
94 | ) {
95 | super(details.Name || "No name", collapsibleState);
96 | this.tooltip = `${details.Name} (${details.Id})`;
97 | this.description = details.Id;
98 | this.contextValue = 'EMRCluster';
99 | }
100 |
101 | public async getChildren(element?: EMRCluster): Promise {
102 | const response = await this.emr.send(
103 | // eslint-disable-next-line @typescript-eslint/naming-convention
104 | new DescribeClusterCommand({ ClusterId: this.details.Id })
105 | );
106 | // TODO (2022-04-13): ERROR CHECKING!
107 | return [
108 | new EMRClusterApps(
109 | this.emr,
110 | response.Cluster ? response.Cluster.Applications : undefined
111 | ),
112 | new EMRClusterInstances(
113 | this.emr, response.Cluster!,
114 | ),
115 | ];
116 | }
117 | }
118 |
119 | class EMRClusterApps extends vscode.TreeItem {
120 | constructor(
121 | private readonly emr: EMR,
122 | private readonly apps: Application[] | undefined
123 | ) {
124 | super("Apps", vscode.TreeItemCollapsibleState.Collapsed);
125 | }
126 |
127 | getTreeItem(element: EMRClusterApps): vscode.TreeItem {
128 | return element;
129 | }
130 |
131 | getChildren(): vscode.TreeItem[] {
132 | return (this.apps || []).map((item) => new EMRAppNode(item));
133 | }
134 | }
135 |
136 | class EMRClusterInstances extends vscode.TreeItem {
137 | constructor(
138 | private readonly emr: EMR,
139 | private readonly cluster: Cluster,
140 | ) {
141 | super("Instances", vscode.TreeItemCollapsibleState.Collapsed);
142 |
143 | this.cluster = cluster;
144 | }
145 |
146 | getTreeItem(element: EMRClusterInstances): vscode.TreeItem {
147 | return element;
148 | }
149 |
150 | public async getChildren(element?: EMRClusterInstances|undefined): Promise {
151 | // TODO (2022-04-13): Pagination
152 | let instanceCollectionMapping: Map = new Map();
153 |
154 | if (this.cluster.InstanceCollectionType === InstanceCollectionType.INSTANCE_GROUP) {
155 | const response = await this.emr.send(
156 | new ListInstanceGroupsCommand({ ClusterId: this.cluster.Id })
157 | );
158 | instanceCollectionMapping.set("master", response.InstanceGroups?.filter(item => item.InstanceGroupType === InstanceGroupType.MASTER).map(item => item.Id as string) || []);
159 | instanceCollectionMapping.set("core", response.InstanceGroups?.filter(item => item.InstanceGroupType === InstanceGroupType.CORE).map(item => item.Id as string) || []);
160 | instanceCollectionMapping.set("task", response.InstanceGroups?.filter(item => item.InstanceGroupType === InstanceGroupType.TASK).map(item => item.Id as string) || []);
161 |
162 | } else if (
163 | this.cluster.InstanceCollectionType === InstanceCollectionType.INSTANCE_FLEET
164 | ) {
165 | const response = await this.emr.send(
166 | new ListInstanceFleetsCommand({ ClusterId: this.cluster.Id })
167 | );
168 | instanceCollectionMapping.set("master", response.InstanceFleets?.filter(item => item.InstanceFleetType === InstanceFleetType.MASTER).map(item => item.Id as string) || []);
169 | instanceCollectionMapping.set("core", response.InstanceFleets?.filter(item => item.InstanceFleetType === InstanceGroupType.CORE).map(item => item.Id as string) || []);
170 | instanceCollectionMapping.set("task", response.InstanceFleets?.filter(item => item.InstanceFleetType === InstanceGroupType.TASK).map(item => item.Id as string) || []);
171 | }
172 |
173 | const instances = await this.emr.send(
174 | new ListInstancesCommand({ClusterId: this.cluster.Id, InstanceStates: [InstanceState.RUNNING, InstanceState.BOOTSTRAPPING, InstanceState.PROVISIONING]})
175 | );
176 |
177 | const instanceTypeMapping = {
178 | master: instances.Instances?.filter(item => instanceCollectionMapping.get("master")?.includes(item.InstanceGroupId!) ),
179 | core: instances.Instances?.filter(item => instanceCollectionMapping.get("core")?.includes(item.InstanceGroupId!)),
180 | task: instances.Instances?.filter(item => instanceCollectionMapping.get("task")?.includes(item.InstanceGroupId!)),
181 | };
182 |
183 | return [
184 | new InstanceNodeTree(
185 | "Primary",
186 | instanceTypeMapping.master?.map(item => new InstanceNodeTree(item.Ec2InstanceId!, undefined, item.InstanceType)),
187 | ),
188 | new InstanceNodeTree(
189 | "Core",
190 | instanceTypeMapping.core?.map(item => new InstanceNodeTree(item.Ec2InstanceId!, undefined, item.InstanceType)),
191 | ),
192 | new InstanceNodeTree(
193 | "Task",
194 | instanceTypeMapping.task?.map(item => new InstanceNodeTree(item.Ec2InstanceId!, undefined, item.InstanceType)),
195 | ),
196 | ];
197 | }
198 | }
199 |
200 | class EMRAppNode extends vscode.TreeItem {
201 | constructor(private readonly app: Application) {
202 | super(app.Name || "Unknown");
203 | this.description = app.Version;
204 | }
205 | }
206 |
207 | class InstanceNodeTree extends vscode.TreeItem {
208 | children: InstanceNodeTree[]|undefined;
209 |
210 | constructor(label: string, children?: InstanceNodeTree[], description?: string) {
211 | super(label, children === undefined ? vscode.TreeItemCollapsibleState.None : vscode.TreeItemCollapsibleState.Collapsed);
212 | this.children = children;
213 | if (description) { this.description = description;}
214 | }
215 |
216 | getChildren(element?: InstanceNodeTree): InstanceNodeTree[] {
217 | return this.children || [];
218 | }
219 |
220 | }
221 |
222 | export class EMREC2Filter {
223 | static defaultStates = [ClusterState.RUNNING, ClusterState.WAITING];
224 | private _showStates: Set;
225 | private _onDidChange = new vscode.EventEmitter();
226 |
227 | constructor() {
228 | // Default set of states
229 | this._showStates = new Set(EMREC2Filter.defaultStates);
230 | }
231 |
232 | public get onDidChange(): vscode.Event {
233 | return this._onDidChange.event;
234 | }
235 |
236 | public async run() {
237 | // TODO (2022-06-13): Refactor this to All / Active / Terminated / Failed
238 | const allStates = [
239 | {
240 | name: "Running",
241 | state: ClusterState.RUNNING,
242 | },
243 | {
244 | name: "Waiting",
245 | state: ClusterState.WAITING,
246 | },
247 | {
248 | name: "Terminated",
249 | state: ClusterState.TERMINATED,
250 | },
251 | {
252 | name: "Terminating",
253 | state: ClusterState.TERMINATING,
254 | },
255 | {
256 | name: "Failed",
257 | state: ClusterState.TERMINATED_WITH_ERRORS,
258 | }
259 | ];
260 |
261 | const items = [];
262 | for (const s of allStates) {
263 | items.push({
264 | label: s.name,
265 | picked: this._showStates ? this._showStates.has(s.state) : false,
266 | state: s.state,
267 | });
268 | }
269 |
270 | const result = await window.showQuickPick(items, {
271 | placeHolder: "Show or hide cluster states",
272 | canPickMany: true,
273 | });
274 |
275 | if (!result) { return false; }
276 |
277 | this._showStates = new Set(result.map(res => res.state!));
278 | this._onDidChange.fire("yolo");
279 |
280 | return true;
281 | }
282 |
283 | public getStates() {
284 | return [...this._showStates];
285 | }
286 | }
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
--------------------------------------------------------------------------------
/src/emr_local.ts:
--------------------------------------------------------------------------------
1 | // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 | // SPDX-License-Identifier: Apache-2.0
3 |
4 | // We want folks to be able to developer EMR jobs locally.
5 | // We give them an option to create an EMR environment
6 | // They select:
7 | // - Type of job (pyspark, scala, SQL)
8 | // - EMR Release (only those supported by EMR on EKS)
9 | // - Region (used to build the local Image URI)
10 |
11 | import { QuickPickItem, window } from "vscode";
12 | import { MultiStepInput } from "./helpers";
13 | import * as fs from "fs";
14 | import * as vscode from "vscode";
15 | import path = require("path");
16 |
17 | function welcomeText(region: string, accountId: string, authType: string) {
18 | const envUpdate = (authType === "ENV_FILE") ? "- Update .devcontainer/aws.env with your AWS credentials.\n": "";
19 | return `# EMR Local Container
20 |
21 | ## Getting Started
22 |
23 | Thanks for installing your local EMR environment. To get started, there are a few steps.
24 |
25 | ${envUpdate}- Login to ECR with the following command:
26 |
27 | aws ecr get-login-password --region ${region} \\
28 | | docker login \\
29 | --username AWS \\
30 | --password-stdin \\
31 | ${accountId}.dkr.ecr.${region}.amazonaws.com
32 |
33 | - Use the \`Remote-Containers: Reopen in Container\` command to build your new environment.
34 |
35 | ## Usage tips
36 |
37 | - You can start a new shell with the \`pyspark\` command in a terminal.
38 | - If you've configured your AWS credentials in \`.env\`, you should have access to everything you need.
39 | - A sample PySpark script has been created for you in the \`emr_tools_demo.py\` file.
40 |
41 | `;
42 | }
43 |
44 | function getEmrMajorVersion(emrRelease: string): number | null {
45 |
46 | // Regular expression to match emr-5.x[.y] or emr-6.x[.y] or emr-7.x[.y] format
47 | const emrPattern = /^emr-(5|6|7)\.\d+(\.\d+)?$/;
48 | const match = emrRelease.match(emrPattern);
49 |
50 | if (match) {
51 | return parseInt(match[1]); // Returns 5, 6, or 7
52 | }
53 | return null;
54 | }
55 |
56 |
57 | export class EMRLocalEnvironment {
58 | context: vscode.ExtensionContext;
59 |
60 | constructor(context: vscode.ExtensionContext) {
61 | this.context = context;
62 | }
63 |
64 | public async run() {
65 | const title = "Create Local Environment";
66 |
67 | interface State {
68 | title: string;
69 | step: number;
70 | totalSteps: number;
71 | resourceGroup: QuickPickItem | string;
72 |
73 | jobType: string;
74 | emrRelease: string;
75 | region: string;
76 | accountId: string;
77 | authType: string;
78 | }
79 |
80 | interface RegionMapping {
81 | label: string;
82 | accountId: string;
83 | }
84 |
85 | interface AuthOption {
86 | label: string;
87 | code: string;
88 | }
89 |
90 | interface EMRContainerEntry {
91 | label: string;
92 | releaseVersion: string;
93 | }
94 |
95 | const emrReleases = [
96 | { label: "EMR 7.7.0", releaseVersion: "emr-7.7.0" },
97 | { label: "EMR 7.6.0", releaseVersion: "emr-7.6.0" },
98 | { label: "EMR 7.5.0", releaseVersion: "emr-7.5.0" },
99 | { label: "EMR 7.4.0", releaseVersion: "emr-7.4.0" },
100 | { label: "EMR 7.3.0", releaseVersion: "emr-7.3.0" },
101 | { label: "EMR 7.2.0", releaseVersion: "emr-7.2.0" },
102 | { label: "EMR 7.1.0", releaseVersion: "emr-7.1.0" },
103 | { label: "EMR 7.0.0", releaseVersion: "emr-7.0.0" },
104 | { label: "EMR 6.15.0", releaseVersion: "emr-6.15.0" },
105 | { label: "EMR 6.14.0", releaseVersion: "emr-6.14.0" },
106 | { label: "EMR 6.13.0", releaseVersion: "emr-6.13.0" },
107 | { label: "EMR 6.12.0", releaseVersion: "emr-6.12.0" },
108 | { label: "EMR 6.11.0", releaseVersion: "emr-6.11.0" },
109 | { label: "EMR 6.10.0", releaseVersion: "emr-6.10.0" },
110 | { label: "EMR 6.9.0", releaseVersion: "emr-6.9.0" },
111 | { label: "EMR 6.8.0", releaseVersion: "emr-6.8.0" },
112 | { label: "EMR 6.7.0", releaseVersion: "emr-6.7.0" },
113 | { label: "EMR 6.6.0", releaseVersion: "emr-6.6.0" },
114 | { label: "EMR 6.5.0", releaseVersion: "emr-6.5.0" },
115 | { label: "EMR 6.4.0", releaseVersion: "emr-6.4.0" },
116 | { label: "EMR 6.3.0", releaseVersion: "emr-6.3.0" },
117 | { label: "EMR 6.2.0", releaseVersion: "emr-6.2.0" },
118 | { label: "EMR 5.35.0", releaseVersion: "emr-5.35.0" },
119 | { label: "EMR 5.34.0", releaseVersion: "emr-5.34.0" },
120 | { label: "EMR 5.33.0", releaseVersion: "emr-5.33.0" },
121 | { label: "EMR 5.32.0", releaseVersion: "emr-5.32.0" },
122 | ];
123 |
124 | async function collectInputs() {
125 | const state = {} as Partial;
126 | await MultiStepInput.run((input) => pickJobType(input, state));
127 | return state as State;
128 | }
129 |
130 | async function pickJobType(input: MultiStepInput, state: Partial) {
131 | const pick = await input.showQuickPick({
132 | title,
133 | step: 1,
134 | totalSteps: 4,
135 | placeholder: "Pick a sample job type",
136 | items: [{ label: "PySpark" }],
137 | activeItem:
138 | typeof state.resourceGroup !== "string"
139 | ? state.resourceGroup
140 | : undefined,
141 | shouldResume: shouldResume,
142 | });
143 |
144 | state.jobType = pick.label;
145 | return (input: MultiStepInput) => pickEMRRelease(input, state);
146 | }
147 |
148 | async function pickEMRRelease(
149 | input: MultiStepInput,
150 | state: Partial
151 | ) {
152 | const pick = await input.showQuickPick({
153 | title,
154 | step: 2,
155 | totalSteps: 4,
156 | placeholder: "Pick an EMR release version",
157 | items: emrReleases,
158 | shouldResume: shouldResume,
159 | });
160 |
161 | state.emrRelease = (pick as EMRContainerEntry).releaseVersion;
162 | return (input: MultiStepInput) => pickImageRegion(input, state);
163 | }
164 |
165 | async function pickImageRegion(
166 | input: MultiStepInput,
167 | state: Partial
168 | ) {
169 | const regionMapping = [
170 | { label: "ap-east-1", accountId: "736135916053" },
171 | { label: "ap-northeast-1", accountId: "059004520145" },
172 | { label: "ap-northeast-2", accountId: "996579266876" },
173 | { label: "ap-northeast-3", accountId: "705689932349" },
174 | { label: "ap-southeast-3", accountId: "946962994502" },
175 | { label: "ap-south-1", accountId: "235914868574" },
176 | { label: "ap-south-2", accountId: "691480105545" },
177 | { label: "ap-southeast-1", accountId: "671219180197" },
178 | { label: "ap-southeast-2", accountId: "038297999601" },
179 | { label: "ca-central-1", accountId: "351826393999" },
180 | { label: "eu-central-1", accountId: "107292555468" },
181 | { label: "eu-central-2", accountId: "314408114945" },
182 | { label: "eu-north-1", accountId: "830386416364" },
183 | { label: "eu-west-1", accountId: "483788554619" },
184 | { label: "eu-west-2", accountId: "118780647275" },
185 | { label: "eu-west-3", accountId: "307523725174" },
186 | { label: "eu-south-1", accountId: "238014973495" },
187 | { label: "eu-south-2", accountId: "350796622945" },
188 | { label: "il-central-1", accountId: "395734710648" },
189 | { label: "me-south-1", accountId: "008085056818" },
190 | { label: "me-central-1", accountId: "818935616732" },
191 | { label: "sa-east-1", accountId: "052806832358" },
192 | { label: "us-east-1", accountId: "755674844232" },
193 | { label: "us-east-2", accountId: "711395599931" },
194 | { label: "us-west-1", accountId: "608033475327" },
195 | { label: "us-west-2", accountId: "895885662937" },
196 | { label: "af-south-1", accountId: "358491847878" },
197 | ];
198 | const pick = await input.showQuickPick({
199 | title,
200 | step: 3,
201 | totalSteps: 4,
202 | placeholder: "Pick a region to pull the container image from",
203 | items: regionMapping,
204 | shouldResume: shouldResume,
205 | });
206 |
207 | state.region = pick.label;
208 | state.accountId = (pick as RegionMapping).accountId;
209 |
210 | return (input: MultiStepInput) => pickAuthenticationType(input, state);
211 | }
212 |
213 | async function pickAuthenticationType(
214 | input: MultiStepInput,
215 | state: Partial
216 | ) {
217 | const areEnvVarsSet =
218 | process.env.AWS_ACCESS_KEY_ID !== undefined &&
219 | process.env.AWS_SECRET_ACCESS_KEY !== undefined;
220 | const authOptions = [
221 | {
222 | label: "Use existing ~/.aws config",
223 | code: "AWS_CONFIG",
224 | description: "Mount your ~/.aws directory to the container.",
225 | },
226 | {
227 | label: "Environment Variables",
228 | code: "ENV_VAR",
229 | description: `If you already have AWS_* environment variables defined.`,
230 | },
231 | {
232 | label: ".env file",
233 | code: "ENV_FILE",
234 | description: "A sample file will be created for you.",
235 | },
236 | {
237 | label: "None",
238 | code: "NONE",
239 | description:
240 | "Requires you to define credentials yourself in the container",
241 | },
242 | ];
243 |
244 | const pick = await input.showQuickPick({
245 | title,
246 | step: 4,
247 | totalSteps: 4,
248 | placeholder: "Select an authentication mechanism for your container",
249 | items: authOptions,
250 | shouldResume: shouldResume,
251 | });
252 |
253 | state.authType = (pick as AuthOption).code;
254 | }
255 |
256 | function shouldResume() {
257 | // Could show a notification with the option to resume.
258 | return new Promise((resolve, reject) => {
259 | // noop
260 | });
261 | }
262 |
263 | const state = await collectInputs();
264 |
265 | // We made it here, now we can create the local environment for the user
266 | await this.createDevContainer(
267 | state.emrRelease,
268 | state.region,
269 | state.accountId,
270 | state.authType
271 | );
272 | }
273 |
274 | private async createDevContainer(
275 | release: string,
276 | region: string,
277 | account: string,
278 | authType: string
279 | ) {
280 | const stripJSONComments = (data: string) => {
281 | var re = new RegExp("//(.*)", "g");
282 | return data.replace(re, "");
283 | };
284 |
285 | // selectWorkspace will be useful
286 | // https://github.com/cantonios/vscode-project-templates/blob/b8e7f65c82fd4fe210c1c188f96eeabdd2b3b317/src/projectTemplatesPlugin.ts#L45
287 | if (vscode.workspace.workspaceFolders === undefined) {
288 | vscode.window.showErrorMessage(
289 | "Amazon EMR: Working folder not found, open a folder and try again."
290 | );
291 | return;
292 | }
293 |
294 | const wsPath = vscode.workspace.workspaceFolders[0].uri.fsPath;
295 | if (!fs.existsSync(wsPath + "/.devcontainer")) {
296 | fs.mkdirSync(wsPath + "/.devcontainer");
297 | }
298 | const targetDcPath = vscode.Uri.file(wsPath + "/.devcontainer");
299 |
300 | const demoFileName = "emr_tools_demo.py";
301 | const samplePyspark = this.context.asAbsolutePath(
302 | path.join("templates", demoFileName)
303 | );
304 |
305 | const dcPath = this.context.asAbsolutePath(
306 | path.join("templates", "devcontainer.json")
307 | );
308 | const envPath = this.context.asAbsolutePath(
309 | path.join("templates", "aws.env")
310 | );
311 | // TODO: Don't implement this yet - we wouldn't want to overwrite a .gitignore
312 | const gitIgnorePath = this.context.asAbsolutePath(
313 | path.join("templates", "_.gitignore")
314 | );
315 | const devContainerConfig = JSON.parse(
316 | stripJSONComments(fs.readFileSync(dcPath).toString())
317 | );
318 | // Update the devcontainer with the requisite release and Image URI details
319 | devContainerConfig["build"]["args"]["RELEASE"] = release;
320 | devContainerConfig["build"]["args"]["REGION"] = region;
321 | devContainerConfig["build"]["args"]["EMR_ACCOUNT_ID"] = account;
322 |
323 | // This is useful to prevent EC2 Metadata errors as well as allows pyspark in Jupyter to work
324 | devContainerConfig["containerEnv"]["AWS_REGION"] = region;
325 |
326 | // Depending on auth type, set the corresponding section in the devcontainer
327 | if (authType === "AWS_CONFIG") {
328 | devContainerConfig["mounts"] = [
329 | "source=${localEnv:HOME}${localEnv:USERPROFILE}/.aws,target=/home/hadoop/.aws,type=bind"
330 | ];
331 | } else if (authType === "ENV_VAR") {
332 | devContainerConfig['containerEnv'] = {
333 | ...devContainerConfig['containerEnv'],
334 | ...{
335 | /* eslint-disable @typescript-eslint/naming-convention */
336 | "AWS_ACCESS_KEY_ID": "${localEnv:AWS_ACCESS_KEY_ID}",
337 | "AWS_SECRET_ACCESS_KEY": "${localEnv:AWS_SECRET_ACCESS_KEY}",
338 | "AWS_SESSION_TOKEN": "${localEnv:AWS_SESSION_TOKEN}",
339 | /* eslint-enable @typescript-eslint/naming-convention */
340 | }
341 | };
342 | } else if (authType === "ENV_FILE") {
343 | devContainerConfig['runArgs'] = [
344 | "--env-file", "${localWorkspaceFolder}/.devcontainer/aws.env"
345 | ];
346 | fs.copyFileSync(envPath, targetDcPath.fsPath + "/aws.env");
347 | }
348 |
349 | // TODO (2022-07-22): Optionally, add mounts of ~/.aws exists
350 | // "source=${env:HOME}${env:USERPROFILE}/.aws,target=/home/hadoop/.aws,type=bind"
351 | // Also make adding environment credentials optional...they could get exposed in logs
352 |
353 | let dockerfilePath;
354 |
355 | if (getEmrMajorVersion(release) === 5 || getEmrMajorVersion(release) === 6) {
356 | dockerfilePath = this.context.asAbsolutePath(
357 | path.join("templates", "pyspark-emr-6_x.dockerfile")
358 | );
359 | } else if (getEmrMajorVersion(release) === 7) {
360 | dockerfilePath = this.context.asAbsolutePath(
361 | path.join("templates", "pyspark-emr-7_x.dockerfile")
362 | );
363 | } else {
364 | throw new Error(`EMR version ${release} not supported`);
365 | }
366 |
367 |
368 | let dockerfile;
369 |
370 | if (dockerfilePath) {
371 | dockerfile = fs.readFileSync(dockerfilePath).toString();
372 | }
373 |
374 |
375 | fs.writeFileSync(
376 | targetDcPath.fsPath + "/devcontainer.json",
377 | JSON.stringify(devContainerConfig, null, " ")
378 | );
379 | fs.writeFileSync(targetDcPath.fsPath + "/Dockerfile", dockerfile!);
380 | fs.copyFileSync(samplePyspark, wsPath + `/${demoFileName}`);
381 |
382 | const howtoPath = vscode.Uri.file(wsPath).fsPath + "/emr-local.md";
383 | fs.writeFileSync(howtoPath, welcomeText(region, account, authType));
384 | vscode.workspace
385 | .openTextDocument(howtoPath)
386 | .then((a: vscode.TextDocument) => {
387 | vscode.window.showTextDocument(a, 1, false);
388 | });
389 |
390 | // var setting: vscode.Uri = vscode.Uri.parse("untitled:" + "emr-local.md");
391 | // vscode.workspace
392 | // .openTextDocument(setting)
393 | // .then((a: vscode.TextDocument) => {
394 | // vscode.window.showTextDocument(a, 1, false).then((e) => {
395 | // e.edit((edit) => {
396 | // edit.insert(
397 | // new vscode.Position(0, 0),
398 | // welcomeText(region, account)
399 | // );
400 | // });
401 | // });
402 | // });
403 | }
404 | }
405 |
--------------------------------------------------------------------------------