├── grails-app ├── i18n │ └── messages.properties ├── conf │ ├── UrlMappings.groovy │ ├── TrackingFilters.groovy │ ├── DataSource.groovy │ ├── Config.groovy │ └── BuildConfig.groovy └── views │ ├── dashboard │ ├── editappgroup.gsp │ ├── summary.gsp │ └── breakdown.gsp │ └── layouts │ └── main.gsp ├── screenshots ├── ss_detail.png ├── ss_summary.png ├── ss_breakdown_appgroup.png ├── ss_reservation_bytype.png └── ss_reservation_byreservation.png ├── web-app ├── images │ ├── spinner.gif │ ├── downarrowdark.gif │ ├── skin │ │ └── information.png │ └── tango │ │ ├── 16 │ │ ├── places │ │ │ └── user-trash.png │ │ ├── tools │ │ │ └── draw-freehand.png │ │ ├── actions │ │ │ └── document-save.png │ │ └── apps │ │ │ └── utilities-system-monitor.png │ │ ├── 24 │ │ ├── actions │ │ │ ├── edit-undo.png │ │ │ └── document-save.png │ │ └── places │ │ │ └── user-trash.png │ │ └── tango-icon-theme │ │ └── 32x32 │ │ └── actions │ │ ├── go-next.png │ │ └── go-previous.png ├── css │ └── ui-lightness │ │ └── images │ │ ├── animated-overlay.gif │ │ ├── ui-icons_222222_256x240.png │ │ ├── ui-icons_228ef1_256x240.png │ │ ├── ui-icons_ef8c08_256x240.png │ │ ├── ui-icons_ffd27a_256x240.png │ │ ├── ui-icons_ffffff_256x240.png │ │ ├── ui-bg_flat_10_000000_40x100.png │ │ ├── ui-bg_glass_65_ffffff_1x400.png │ │ ├── ui-bg_glass_100_f6f6f6_1x400.png │ │ ├── ui-bg_glass_100_fdf5ce_1x400.png │ │ ├── ui-bg_gloss-wave_35_f6a828_500x100.png │ │ ├── ui-bg_diagonals-thick_18_b81900_40x40.png │ │ ├── ui-bg_diagonals-thick_20_666666_40x40.png │ │ ├── ui-bg_highlight-soft_100_eeeeee_1x100.png │ │ └── ui-bg_highlight-soft_75_ffe45c_1x100.png └── WEB-INF │ ├── sitemesh.xml │ ├── applicationContext.xml │ └── grails.xml ├── wrapper ├── springloaded-core-1.1.1.jar ├── grails-wrapper.properties └── grails-wrapper-runtime-2.2.1.jar ├── src ├── java │ ├── log4j.properties │ ├── com │ │ └── netflix │ │ │ └── ice │ │ │ ├── reader │ │ │ ├── AggregateType.java │ │ │ ├── ThroughputMetricService.java │ │ │ ├── DataManager.java │ │ │ ├── ApplicationGroupService.java │ │ │ ├── Managers.java │ │ │ ├── ReadOnlyData.java │ │ │ ├── ApplicationGroup.java │ │ │ ├── TagGroupManager.java │ │ │ └── ReaderConfig.java │ │ │ ├── tag │ │ │ ├── ApplicationGroup.java │ │ │ ├── TagType.java │ │ │ ├── Account.java │ │ │ ├── Tag.java │ │ │ ├── ResourceGroup.java │ │ │ ├── InstanceOs.java │ │ │ ├── Product.java │ │ │ ├── Operation.java │ │ │ ├── UsageType.java │ │ │ ├── Region.java │ │ │ └── Zone.java │ │ │ ├── common │ │ │ ├── ConsolidateType.java │ │ │ ├── Randomizer.java │ │ │ ├── ProductService.java │ │ │ ├── AccountService.java │ │ │ ├── Config.java │ │ │ ├── ResourceService.java │ │ │ ├── Poller.java │ │ │ └── IceOptions.java │ │ │ ├── processor │ │ │ ├── LineItemProcessor.java │ │ │ ├── DataWriter.java │ │ │ ├── TagGroupWriter.java │ │ │ ├── ReservationService.java │ │ │ ├── ReadWriteData.java │ │ │ └── ProcessorConfig.java │ │ │ └── basic │ │ │ ├── BasicResourceService.java │ │ │ ├── BasicAccountService.java │ │ │ ├── SampleMapDbResourceService.java │ │ │ ├── MapDb.java │ │ │ ├── BasicS3ApplicationGroupService.java │ │ │ ├── BasicProductService.java │ │ │ ├── EddaResourceService.java │ │ │ ├── BasicManagers.java │ │ │ └── BasicThroughputMetricService.java │ └── sample.properties ├── groovy │ └── com │ │ └── netflix │ │ └── ice │ │ └── JSONConverter.groovy └── test │ └── com │ └── netflix │ └── ice │ └── basic │ └── EddaResourceServiceTest.java ├── .gitignore ├── application.properties ├── install.sh ├── conf └── system │ ├── setenv.sh │ └── server.xml └── grailsw.bat /grails-app/i18n/messages.properties: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /screenshots/ss_detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/screenshots/ss_detail.png -------------------------------------------------------------------------------- /screenshots/ss_summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/screenshots/ss_summary.png -------------------------------------------------------------------------------- /web-app/images/spinner.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/spinner.gif -------------------------------------------------------------------------------- /web-app/images/downarrowdark.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/downarrowdark.gif -------------------------------------------------------------------------------- /web-app/images/skin/information.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/skin/information.png -------------------------------------------------------------------------------- /wrapper/springloaded-core-1.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/wrapper/springloaded-core-1.1.1.jar -------------------------------------------------------------------------------- /screenshots/ss_breakdown_appgroup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/screenshots/ss_breakdown_appgroup.png -------------------------------------------------------------------------------- /screenshots/ss_reservation_bytype.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/screenshots/ss_reservation_bytype.png -------------------------------------------------------------------------------- /wrapper/grails-wrapper.properties: -------------------------------------------------------------------------------- 1 | wrapper.dist.url=http://dist.springframework.org.s3.amazonaws.com/release/GRAILS/ 2 | -------------------------------------------------------------------------------- /wrapper/grails-wrapper-runtime-2.2.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/wrapper/grails-wrapper-runtime-2.2.1.jar -------------------------------------------------------------------------------- /screenshots/ss_reservation_byreservation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/screenshots/ss_reservation_byreservation.png -------------------------------------------------------------------------------- /web-app/images/tango/16/places/user-trash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/16/places/user-trash.png -------------------------------------------------------------------------------- /web-app/images/tango/24/actions/edit-undo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/24/actions/edit-undo.png -------------------------------------------------------------------------------- /web-app/images/tango/24/places/user-trash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/24/places/user-trash.png -------------------------------------------------------------------------------- /web-app/images/tango/16/tools/draw-freehand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/16/tools/draw-freehand.png -------------------------------------------------------------------------------- /web-app/images/tango/16/actions/document-save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/16/actions/document-save.png -------------------------------------------------------------------------------- /web-app/images/tango/24/actions/document-save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/24/actions/document-save.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/animated-overlay.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/animated-overlay.gif -------------------------------------------------------------------------------- /web-app/images/tango/16/apps/utilities-system-monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/16/apps/utilities-system-monitor.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_222222_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-icons_222222_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_228ef1_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-icons_228ef1_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_ef8c08_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-icons_ef8c08_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_ffd27a_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-icons_ffd27a_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_ffffff_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-icons_ffffff_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_flat_10_000000_40x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_flat_10_000000_40x100.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_glass_65_ffffff_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_glass_65_ffffff_1x400.png -------------------------------------------------------------------------------- /web-app/images/tango/tango-icon-theme/32x32/actions/go-next.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/tango-icon-theme/32x32/actions/go-next.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_glass_100_f6f6f6_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_glass_100_f6f6f6_1x400.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_glass_100_fdf5ce_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_glass_100_fdf5ce_1x400.png -------------------------------------------------------------------------------- /web-app/images/tango/tango-icon-theme/32x32/actions/go-previous.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/images/tango/tango-icon-theme/32x32/actions/go-previous.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_gloss-wave_35_f6a828_500x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_gloss-wave_35_f6a828_500x100.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_diagonals-thick_18_b81900_40x40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_diagonals-thick_18_b81900_40x40.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_diagonals-thick_20_666666_40x40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_diagonals-thick_20_666666_40x40.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_highlight-soft_100_eeeeee_1x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_highlight-soft_100_eeeeee_1x100.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_highlight-soft_75_ffe45c_1x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mdsol/ice/master/web-app/css/ui-lightness/images/ui-bg_highlight-soft_75_ffe45c_1x100.png -------------------------------------------------------------------------------- /src/java/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | # stdout 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} - %-5p %C{1} - [%F:%L] %m%n -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iws 2 | *Db.properties 3 | *Db.script 4 | .settings 5 | stacktrace.log 6 | *.zip 7 | /plugin.xml 8 | *.log 9 | *DB.* 10 | cobertura.ser 11 | .DS_Store 12 | target/ 13 | out/ 14 | work/ 15 | work/** 16 | web-app/plugins 17 | web-app/WEB-INF/classes 18 | .link_to_grails_plugins/ 19 | target-eclipse/ 20 | 21 | .idea 22 | *.iml 23 | -------------------------------------------------------------------------------- /web-app/WEB-INF/sitemesh.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 7 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /application.properties: -------------------------------------------------------------------------------- 1 | #Grails Metadata file 2 | #Tue Sep 24 10:12:14 EDT 2013 3 | *=limitations under the License. 4 | */= 5 | /*= 6 | app.grails.version=2.2.3 7 | app.name=ice 8 | app.servlet.version=2.4 9 | app.version=1.1.0 10 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 11 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 12 | log4j.appender.stdout.layout.ConversionPattern=%d %5p [%c] %m%n 13 | log4j.category.com.amazonaws.request=WARN 14 | log4j.category.com.netflix.cloud=DEBUG 15 | log4j.category.com.netflix.logging=WARN 16 | log4j.category.com.netflix.monitoring=WARN 17 | log4j.category.hibernate=ERROR 18 | log4j.category.httpclient=ERROR 19 | log4j.category.org.codehaus.groovy=WARN 20 | log4j.category.sun.reflect=WARN 21 | log4j.rootLogger=INFO, stdout 22 | plugins.hibernate=2.2.3 23 | plugins.tomcat=2.2.3 24 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/AggregateType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | public enum AggregateType { 21 | none, 22 | stats, 23 | data, 24 | both 25 | } 26 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/ApplicationGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public class ApplicationGroup extends Tag { 21 | 22 | public ApplicationGroup(String name) { 23 | super(name); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/TagType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public enum TagType { 21 | Account, 22 | Region, 23 | Zone, 24 | Product, 25 | Operation, 26 | UsageType, 27 | ResourceGroup, 28 | ApplicationGroup; 29 | } 30 | -------------------------------------------------------------------------------- /grails-app/conf/UrlMappings.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | class UrlMappings { 18 | 19 | static mappings = { 20 | "/$controller/$action?/$id?" { 21 | constraints { 22 | // apply constraints here 23 | } 24 | } 25 | 26 | "/" { controller = "dashboard"} 27 | "500"(view: '/error') 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Account.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public class Account extends Tag { 21 | public final String id; 22 | 23 | public Account(String accountId, String accountName) { 24 | super(accountName); 25 | this.id = accountId; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/ConsolidateType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | public enum ConsolidateType { 21 | hourly(3600000L), 22 | daily(24*3600000L), 23 | weekly(24*7*3600000L), 24 | monthly(-1); 25 | 26 | public final long millis; 27 | 28 | private ConsolidateType(long millis) { 29 | this.millis = millis; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ThroughputMetricService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.netflix.ice.common.ConsolidateType; 21 | import org.joda.time.Interval; 22 | 23 | /** 24 | * Interface to provide throughput metrics. 25 | */ 26 | public interface ThroughputMetricService { 27 | void init(); 28 | String getMetricName(); 29 | String getMetricUnitName(); 30 | String getFactoredCostCurrencySign(); 31 | double getFactoredCostMultiply(); 32 | double[] getData(Interval interval, ConsolidateType consolidateType) throws Exception; 33 | } 34 | -------------------------------------------------------------------------------- /grails-app/conf/TrackingFilters.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import java.util.regex.Pattern 17 | import org.joda.time.DateTime 18 | 19 | class TrackingFilters { 20 | 21 | static final Pattern nonBrowserUserAgents = Pattern.compile( 22 | '.*(libcurl|Python-urllib|Wget|HttpClient|lwp-request|Java).*') 23 | 24 | static filters = { 25 | allExceptError(controller: 'error', invert: true) { 26 | before = { 27 | 28 | String userAgent = request.getHeader('user-agent') 29 | if (userAgent?.contains('MSIE') && !userAgent.contains('chromeframe')) { 30 | request['ieWithoutChromeFrame'] = true 31 | } 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/Randomizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import java.util.Map; 21 | 22 | /** 23 | * Used to generate random/fake data 24 | */ 25 | public interface Randomizer { 26 | 27 | /** 28 | * Get random/fake usage for given tag group. 29 | * @param time 30 | * @param tagGroup 31 | * @param usage 32 | * @return random/fake usage 33 | */ 34 | double randomizeUsage(long time, TagGroup tagGroup, double usage); 35 | 36 | /** 37 | * Get random/fake cost for given tag group. 38 | * @param tagGroup 39 | * @return random/fake cost 40 | */ 41 | double randomizeCost(TagGroup tagGroup); 42 | 43 | /** 44 | * Get map of resource distribution. 45 | * @return map of resource distribution 46 | */ 47 | Map getDistribution(TagGroup tagGroup); 48 | } 49 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/DataManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.netflix.ice.tag.Tag; 21 | import com.netflix.ice.tag.TagType; 22 | import org.joda.time.DateTime; 23 | import org.joda.time.Interval; 24 | 25 | import java.util.Map; 26 | 27 | /** 28 | * Interface to feed data to UI. 29 | */ 30 | public interface DataManager { 31 | 32 | /** 33 | * Get map of data. 34 | * @param interval 35 | * @param tagLists 36 | * @param groupBy 37 | * @param aggregate 38 | * @param forReservation 39 | * @return 40 | */ 41 | Map getData(Interval interval, TagLists tagLists, TagType groupBy, AggregateType aggregate, boolean forReservation); 42 | 43 | /** 44 | * Get data length. 45 | * @param start 46 | * @return 47 | */ 48 | int getDataLength(DateTime start); 49 | } 50 | -------------------------------------------------------------------------------- /src/groovy/com/netflix/ice/JSONConverter.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice 19 | 20 | import grails.converters.JSON 21 | import com.netflix.ice.tag.Tag 22 | import com.netflix.ice.reader.ApplicationGroup 23 | 24 | class JSONConverter { 25 | 26 | static void register() { 27 | JSON.registerObjectMarshaller(Tag) { Tag it -> 28 | return [name: it.name] 29 | } 30 | JSON.registerObjectMarshaller(ApplicationGroup) { ApplicationGroup appgroup -> 31 | def result = [name: appgroup.name, owner: appgroup.owner, data: [:]] 32 | for (String key: appgroup.data.keySet()) { 33 | result.data[key] = [] 34 | for (String v: appgroup.data.get(key)) { 35 | result.data[key].add([name: v]) 36 | } 37 | } 38 | return result 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /grails-app/conf/DataSource.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /*dataSource { 17 | pooled = true 18 | driverClassName = "org.hsqldb.jdbcDriver" 19 | username = "sa" 20 | password = "" 21 | }*/ 22 | hibernate { 23 | cache.use_second_level_cache = true 24 | cache.use_query_cache = true 25 | cache.provider_class = 'net.sf.ehcache.hibernate.EhCacheProvider' 26 | } 27 | 28 | // environment specific settings 29 | environments { 30 | development { 31 | dataSource { 32 | dbCreate = "create-drop" // one of 'create', 'create-drop','update' 33 | //url = "jdbc:hsqldb:mem:devDB" 34 | } 35 | } 36 | test { 37 | dataSource { 38 | dbCreate = "update" 39 | //url = "jdbc:hsqldb:mem:testDb" 40 | } 41 | } 42 | production { 43 | dataSource { 44 | dbCreate = "update" 45 | //url = "jdbc:hsqldb:file:prodDb;shutdown=true" 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /web-app/WEB-INF/applicationContext.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | Grails application factory bean 9 | 10 | 11 | 12 | 13 | 14 | A bean that manages Grails plugins 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | utf-8 31 | 32 | 33 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/ProductService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import com.netflix.ice.tag.Product; 21 | 22 | import java.util.Collection; 23 | import java.util.List; 24 | 25 | public interface ProductService { 26 | 27 | /** 28 | * Get product by aws name, e.g. "Amazon Elastic Compute Cloud", "AWS Elastic MapReduce" 29 | * @param name 30 | * @return product 31 | */ 32 | Product getProductByAwsName(String name); 33 | 34 | /** 35 | * Get product by name, e.g. ec2, emr 36 | * @param name 37 | * @return 38 | */ 39 | Product getProductByName(String name); 40 | 41 | /** 42 | * Get list of products from given names 43 | * @param names 44 | * @return list of products 45 | */ 46 | public List getProducts(List names); 47 | 48 | /** 49 | * Get list of products 50 | * @return list of products 51 | */ 52 | public Collection getProducts(); 53 | } 54 | -------------------------------------------------------------------------------- /web-app/WEB-INF/grails.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 20 | 21 | DefaultSecurityConfig 22 | grails.plugins.springsecurity.SpringSecurityService 23 | grails.plugins.springsecurity.SecurityTagLib 24 | DefaultLdapSecurityConfig 25 | BootStrap 26 | BuildConfig 27 | Config 28 | DataSource 29 | OccasionFilters 30 | RegionFilters 31 | SettingsFilters 32 | resources 33 | TrackingFilters 34 | UrlMappings 35 | com.netflix.ice.DashboardController 36 | 37 | 38 | HibernateGrailsPlugin 39 | SpringSecurityCoreGrailsPlugin 40 | SpringSecurityLdapGrailsPlugin 41 | 42 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Tag.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import java.io.Serializable; 21 | 22 | public abstract class Tag implements Comparable, Serializable { 23 | public static final Tag aggregated = new Tag("aggregated") { 24 | @Override 25 | public int compareTo(Tag t) { 26 | return this == t ? 0 : -1; 27 | } 28 | }; 29 | 30 | public final String name; 31 | Tag(String name) { 32 | this.name = name; 33 | } 34 | 35 | @Override 36 | public boolean equals(Object o) { 37 | if (o instanceof Tag) 38 | return this.name.equals(((Tag)o).name); 39 | else 40 | return false; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return this.name; 46 | } 47 | 48 | @Override 49 | public int hashCode() { 50 | return this.name.hashCode(); 51 | } 52 | 53 | public int compareTo(Tag t) { 54 | if (t == aggregated) 55 | return -t.compareTo(this); 56 | int result = ("a" + this.name).compareTo("a" + t.name); 57 | return result; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ApplicationGroupService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import java.util.Map; 21 | 22 | /** 23 | * Interface to manager user defined application groups. 24 | */ 25 | public interface ApplicationGroupService { 26 | 27 | void init(); 28 | 29 | /** 30 | * Get a map of application groups. Key will be the application group name. 31 | * @return map of application groups 32 | */ 33 | Map getApplicationGroups(); 34 | 35 | /** 36 | * Get application group by name. 37 | * @param name 38 | * @return application group or null if not exist 39 | */ 40 | ApplicationGroup getApplicationGroup(String name); 41 | 42 | /** 43 | * Save application group configuration. 44 | * @param applicationGroup 45 | * @return whether or not save was successfull 46 | */ 47 | boolean saveApplicationGroup(ApplicationGroup applicationGroup); 48 | 49 | /** 50 | * Delete application group. 51 | * @param name 52 | * @return whether or not delete was successfull 53 | */ 54 | boolean deleteApplicationGroup(String name); 55 | } 56 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/LineItemProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.netflix.ice.tag.Product; 21 | import java.util.List; 22 | import org.joda.time.DateTimeZone; 23 | import org.joda.time.format.DateTimeFormat; 24 | import org.joda.time.format.DateTimeFormatter; 25 | 26 | import java.util.Map; 27 | 28 | /** 29 | * Interface to process each line item in billing file. 30 | */ 31 | public interface LineItemProcessor { 32 | public static final DateTimeFormatter amazonBillingDateFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(DateTimeZone.UTC); 33 | public static final DateTimeFormatter amazonBillingDateFormat2 = DateTimeFormat.forPattern("yyyy/MM/dd HH:mm:ss").withZone(DateTimeZone.UTC); 34 | 35 | void initIndexes(boolean withTags, String[] header); 36 | List getHeader(); 37 | int getUserTagStartIndex(); 38 | long getEndMillis(String[] items); 39 | Result process(long startMilli, boolean processAll, ProcessorConfig config, String[] items, Map usageDataByProduct, Map costDataByProduct, Map ondemandRate); 40 | 41 | public static enum Result { 42 | delay, 43 | ignore, 44 | hourly, 45 | monthly, 46 | daily 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/ResourceGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | 23 | import java.util.List; 24 | import java.util.concurrent.ConcurrentMap; 25 | 26 | public class ResourceGroup extends Tag { 27 | private ResourceGroup (String name) { 28 | super(name); 29 | } 30 | private static ConcurrentMap resourceGroups = Maps.newConcurrentMap(); 31 | 32 | public static ResourceGroup getResourceGroup(String name) { 33 | ResourceGroup resourceGroup = resourceGroups.get(name); 34 | if (resourceGroup == null) { 35 | resourceGroups.putIfAbsent(name, new ResourceGroup(name)); 36 | resourceGroup = resourceGroups.get(name); 37 | } 38 | return resourceGroup; 39 | } 40 | 41 | public static List getResourceGroups(List names) { 42 | List result = Lists.newArrayList(); 43 | if (names != null) { 44 | for (String name: names) { 45 | ResourceGroup resourceGroup = resourceGroups.get(name); 46 | if (resourceGroup != null) 47 | result.add(resourceGroup); 48 | } 49 | } 50 | return result; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/Managers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.netflix.ice.common.*; 21 | import com.netflix.ice.tag.Product; 22 | 23 | import java.util.Collection; 24 | 25 | /** 26 | * Interface to manager all TagGroupManager and DataManager instances for different products 27 | */ 28 | public interface Managers { 29 | 30 | void init(); 31 | 32 | /** 33 | * 34 | * @return collection of products 35 | */ 36 | Collection getProducts(); 37 | 38 | /** 39 | * 40 | * @param product 41 | * @return TagGroupManager instance for specified product 42 | */ 43 | TagGroupManager getTagGroupManager(Product product); 44 | 45 | /** 46 | * 47 | * @param product 48 | * @param consolidateType 49 | * @return cost DataManager instance for specified product and consolidateType 50 | */ 51 | DataManager getCostManager(Product product, ConsolidateType consolidateType); 52 | 53 | /** 54 | * 55 | * @param product 56 | * @param consolidateType 57 | * @return usage DataManager instance for specified product and consolidateType 58 | */ 59 | DataManager getUsageManager(Product product, ConsolidateType consolidateType); 60 | 61 | /** 62 | * shutdown all manager instances 63 | */ 64 | void shutdown(); 65 | } 66 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/InstanceOs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public enum InstanceOs { 21 | linux("", "", "Linux/UNIX"), 22 | sqlserverweb(".sqlserverweb", ":0202", "Windows with SQL Server Web"), 23 | sqlserverstd(".sqlserverstd", ":0006", "Windows with SQL Server Standard"), 24 | sles(".sles", ":000g", "SUSE Linux"), 25 | rhel(".rhel", ":0010", "Red Hat Enterprise Linux"), 26 | rhbl(".rhbl", ":00g0", "Red Hat BYOL Linux"), 27 | windows(".windows", ":0002", "Windows"), 28 | dw(".dw", ":0001", "redshift"), 29 | others(".others", ":others", "others"); 30 | 31 | public final String usageType; 32 | public final String code; 33 | public final String description; 34 | 35 | InstanceOs(String usageType, String code, String description) { 36 | this.usageType = usageType; 37 | this.code = code.toLowerCase(); 38 | this.description = description.toLowerCase(); 39 | } 40 | 41 | public static InstanceOs withCode(String code) { 42 | for (InstanceOs os: InstanceOs.values()) { 43 | if (code.toLowerCase().equals(os.code)) 44 | return os; 45 | } 46 | return others; 47 | } 48 | 49 | public static InstanceOs withDescription(String description) { 50 | for (InstanceOs os: InstanceOs.values()) { 51 | if (description.toLowerCase().startsWith(os.description)) 52 | return os; 53 | } 54 | return others; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Product.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public class Product extends Tag { 21 | public static final Product cloudfront = new Product("cloudfront"); 22 | public static final Product cloudhsm = new Product("cloudhsm"); 23 | public static final Product cloudwatch = new Product("cloudwatch"); 24 | public static final Product data_transfer = new Product("data_transfer"); 25 | public static final Product direct_connect = new Product("direct_connect"); 26 | public static final Product dynamodb = new Product("dynamodb"); 27 | public static final Product ebs = new Product("ebs"); 28 | public static final Product ec2 = new Product("ec2"); 29 | public static final Product ec2_instance = new Product("ec2_instance"); 30 | public static final Product eip = new Product("eip"); 31 | public static final Product elasticache = new Product("elasticache"); 32 | public static final Product emr = new Product("emr"); 33 | public static final Product glacier = new Product("glacier"); 34 | public static final Product monitor = new Product("monitor"); 35 | public static final Product rds = new Product("rds"); 36 | public static final Product redshift = new Product("redshift"); 37 | public static final Product route53 = new Product("route53"); 38 | public static final Product s3 = new Product("s3"); 39 | public static final Product simpledb = new Product("simpledb"); 40 | public static final Product ses = new Product("ses"); 41 | public static final Product sns = new Product("sns"); 42 | public static final Product sqs = new Product("sqs"); 43 | public static final Product sws = new Product("sws"); 44 | public static final Product vpc = new Product("vpc"); 45 | 46 | public Product (String name) { 47 | super(name); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ReadOnlyData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.netflix.ice.common.TagGroup; 22 | 23 | import java.io.DataInput; 24 | import java.io.IOException; 25 | import java.util.Collection; 26 | import java.util.List; 27 | 28 | public class ReadOnlyData { 29 | double[][] data; 30 | private Collection tagGroups; 31 | 32 | public ReadOnlyData(double[][] data, Collection tagGroups) { 33 | this.data = data; 34 | this.tagGroups = tagGroups; 35 | } 36 | 37 | public double[] getData(int i) { 38 | return data[i]; 39 | } 40 | 41 | public int getNum() { 42 | return data.length; 43 | } 44 | 45 | public Collection getTagGroups() { 46 | return tagGroups; 47 | } 48 | 49 | public static class Serializer { 50 | 51 | public static ReadOnlyData deserialize(DataInput in) throws IOException { 52 | 53 | int numKeys = in.readInt(); 54 | List keys = Lists.newArrayList(); 55 | for (int j = 0; j < numKeys; j++) { 56 | keys.add(TagGroup.Serializer.deserialize(ReaderConfig.getInstance(), in)); 57 | } 58 | 59 | int num = in.readInt(); 60 | double[][] data = new double[num][]; 61 | for (int i = 0; i < num; i++) { 62 | data[i] = new double[keys.size()]; 63 | boolean hasData = in.readBoolean(); 64 | if (hasData) { 65 | for (int j = 0; j < keys.size(); j++) { 66 | double v = in.readDouble(); 67 | if (v != 0) { 68 | data[i][j] = v; 69 | } 70 | } 71 | } 72 | } 73 | 74 | return new ReadOnlyData(data, keys); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Get Ice in a ready-to-run state on a fresh AWI instance. 4 | 5 | GRAILS_VERSION=2.2.1 6 | 7 | # Install prerequisites 8 | sudo yum -y install git java-1.6.0-openjdk-devel.x86_64 wget 9 | 10 | INSTALL_DIR=$(pwd) 11 | 12 | cd 13 | 14 | HOME_DIR=$(pwd) 15 | 16 | # Prep grails in such a way that we only download it once 17 | if [ ! -x ".grails/wrapper/${GRAILS_VERSION}/grails-${GRAILS_VERSION}" ]; then 18 | mkdir -p .grails/wrapper/ 19 | cd .grails/wrapper/ 20 | # (Fetch) 21 | wget http://dist.springframework.org.s3.amazonaws.com/release/GRAILS/grails-${GRAILS_VERSION}.zip -O grails-${GRAILS_VERSION}-download.zip 22 | mkdir ${GRAILS_VERSION} 23 | cd ${GRAILS_VERSION} 24 | # ("Install") 25 | unzip ../grails-${GRAILS_VERSION}-download.zip 26 | fi 27 | 28 | GRAILS_HOME=${HOME_DIR}/.grails/wrapper/${GRAILS_VERSION}/grails-${GRAILS_VERSION}/ 29 | PATH=$PATH:${HOME_DIR}/.grails/wrapper/${GRAILS_VERSION}/grails-${GRAILS_VERSION}/bin/ 30 | 31 | # Get ice 32 | cd ${INSTALL_DIR} 33 | 34 | if [ -x '.git' ]; then 35 | # We already have it; update to latest git 36 | git pull 37 | else 38 | # We don't have it at all yet; clone the repo 39 | git clone https://github.com/Netflix/ice.git 40 | cd ice 41 | INSTALL_DIR=$(pwd) 42 | fi 43 | 44 | # Initialize Ice with Grails 45 | grails ${JAVA_OPTS} wrapper 46 | 47 | # (Bug: Ice can't deal with this file existing and being empty.) 48 | rm grails-app/i18n/messages.properties 49 | 50 | # Create our local work directories (both for processing and reading) 51 | mkdir ${HOME_DIR}/ice_processor 52 | mkdir ${HOME_DIR}/ice_reader 53 | 54 | # Set up the config file 55 | cp src/java/sample.properties src/java/ice.properties 56 | echo Please enter the name of the bucket Ice will read Amazon billing information from: 57 | while [ "$BILLBUCKET" == "" ] 58 | do 59 | echo -n "-> " 60 | read -r BILLBUCKET 61 | done 62 | 63 | echo Please enter the name of the bucket Ice will write processed billing information to: 64 | while [ "$PROCBUCKET" == "" ] 65 | do 66 | echo -n "-> " 67 | read -r PROCBUCKET 68 | done 69 | sed -rie 's/=billing_s3bucketprefix\//=/; s/\/mnt\//\/home\/ec2-user\//; s/=work_s3bucketprefix\//=/; s/^ice.account.*//; s/=billing_s3bucketname/='${BILLBUCKET}'/; s/=work_s3bucketname/='${PROCBUCKET}'/' src/java/ice.properties 70 | 71 | echo Ice is now ready to run as a processor. If you want to run the reader, edit: 72 | echo ~/ice/src/java/ice.properties 73 | echo and alter the appropriate flags. You can now start Ice by running the following from the Ice root directory: 74 | echo ./grailsw -Djava.net.preferIPv4Stack=true -Dice.s3AccessKeyId=\ -Dice.s3SecretKey=\ run-app 75 | -------------------------------------------------------------------------------- /conf/system/setenv.sh: -------------------------------------------------------------------------------- 1 | # Determine the max heap size based on the instance type. 2 | # See: http://aws.amazon.com/ec2/instance-types/ 3 | # 4 | # For nodes in the datacenter we rely on the convention that the hostname 5 | # indicates the amount of memory. 6 | function heapSize { 7 | instanceType=$(hostname) 8 | if [ "$EC2_INSTANCE_TYPE" != "" ]; then 9 | instanceType="$EC2_INSTANCE_TYPE" 10 | fi 11 | 12 | case $instanceType in 13 | m1.small) echo "1g" ;; # 1.7GB 14 | m1.large) echo "6g" ;; # 7.5GB 15 | m1.xlarge) echo "12g" ;; # 15GB 16 | t1.micro) echo "256m" ;; # 613MB 17 | m2.xlarge) echo "14g" ;; # 17.1GB 18 | m2.2xlarge) echo "30g" ;; # 34.2GB 19 | m2.4xlarge) echo "60g" ;; # 68.4GB 20 | c1.medium) echo "1g" ;; # 1.7GB 21 | c1.xlarge) echo "4g" ;; # 7GB 22 | cc1.4xlarge) echo "20g" ;; # 23GB 23 | cg1.4xlarge) echo "20g" ;; # 22GB 24 | *-4g*) echo "3g" ;; 25 | *-8g*) echo "6g" ;; 26 | *-16g*) echo "14g" ;; 27 | *-24g*) echo "21g" ;; 28 | *-32g*) echo "28g" ;; 29 | *-64g*) echo "60g" ;; 30 | *-72g*) echo "68g" ;; 31 | *) echo "1g" ;; # who knows, try for a gig 32 | esac 33 | } 34 | 35 | # Create the java heap size options. We always set min and max heap size to 36 | # be the same, this ensures that the program will crash right away if there 37 | # is not enough memory. 38 | function javaOptsHeapSize { 39 | permPercent=$1 40 | max=$(heapSize) 41 | permSize="$(echo "$max" | awk "{print int(\$1 * $permPercent)}")g" 42 | echo "-Xms$max -Xmx$max -XX:PermSize=$permSize -XX:MaxPermSize=$permSize" 43 | } 44 | 45 | GCLOG=/apps/tomcat/logs/gc.log 46 | CATALINA_PID=/apps/tomcat/logs/catalina.pid 47 | JRE_HOME=/apps/java 48 | JAVA_HOME=$JRE_HOME 49 | if [ "$1" == "start" ]; then 50 | JAVA_OPTS=" \ 51 | -javaagent:/apps/appagent/javaagent.jar \ 52 | -verbosegc \ 53 | -XX:+PrintGCDetails \ 54 | -XX:+PrintGCTimeStamps \ 55 | -Xloggc:$GCLOG \ 56 | $(javaOptsHeapSize 0.1) \ 57 | -Xss8192k \ 58 | -Dnetflix.logging.realtimetracers=false \ 59 | -Dlog4j.appender.RTA=org.apache.log4j.varia.NullAppender \ 60 | -Dlog4j.appender.CHUKWA=org.apache.log4j.varia.NullAppender \ 61 | -Dlog4j.appender.CHUKWA.threshold=OFF \ 62 | -Dlog4j.logger.net.spy.memcached.internal.OperationFuture=FATAL \ 63 | -Dnet.spy.log.LoggerImpl=net.spy.memcached.compat.log.Log4JLogger \ 64 | -Dnetflix.environment=test 65 | -Dcom.sun.management.jmxremote.ssl=false \ 66 | -Dcom.sun.management.jmxremote.port=7500 \ 67 | -Dcom.sun.management.jmxremote.authenticate=false \ 68 | -Dnetflix.appinfo.name=${NETFLIX_APP}" 69 | 70 | logrotate -f /apps/tomcat/bin/.logrotate.conf 71 | else 72 | JAVA_OPTS="" 73 | fi 74 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/DataWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.netflix.ice.common.AwsUtils; 21 | import com.netflix.ice.common.TagGroup; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.io.*; 26 | import java.util.Collection; 27 | import java.util.TreeMap; 28 | 29 | public class DataWriter { 30 | private final static Logger logger = LoggerFactory.getLogger(DataWriter.class); 31 | 32 | 33 | private TreeMap> tagGroups; 34 | private ProcessorConfig config = ProcessorConfig.getInstance(); 35 | private String dbName; 36 | private File file; 37 | private ReadWriteData data; 38 | 39 | DataWriter(String name, boolean loadData) throws Exception { 40 | 41 | dbName = name; 42 | file = new File(config.localDir, dbName); 43 | if (loadData) { 44 | AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, file); 45 | } 46 | 47 | if (file.exists()) { 48 | DataInputStream in = new DataInputStream(new FileInputStream(file)); 49 | try { 50 | data = ReadWriteData.Serializer.deserialize(in); 51 | } 52 | finally { 53 | in.close(); 54 | } 55 | } 56 | else { 57 | data = new ReadWriteData(); 58 | } 59 | } 60 | 61 | ReadWriteData getData() { 62 | return data; 63 | } 64 | 65 | void archive() throws IOException { 66 | archive(data); 67 | } 68 | 69 | void archive(ReadWriteData data) throws IOException { 70 | 71 | DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); 72 | try { 73 | ReadWriteData.Serializer.serialize(out, data); 74 | } 75 | finally { 76 | out.close(); 77 | } 78 | 79 | logger.info(this.dbName + " uploading to s3..."); 80 | AwsUtils.upload(config.workS3BucketName, config.workS3BucketPrefix, config.localDir, dbName); 81 | logger.info(this.dbName + " uploading done."); 82 | } 83 | } 84 | 85 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicResourceService.java: -------------------------------------------------------------------------------- 1 | package com.netflix.ice.basic; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.netflix.ice.common.ResourceService; 5 | import com.netflix.ice.processor.ProcessorConfig; 6 | import com.netflix.ice.reader.ReaderConfig; 7 | import com.netflix.ice.tag.Account; 8 | import com.netflix.ice.tag.Product; 9 | import com.netflix.ice.tag.Region; 10 | import org.apache.commons.lang.StringUtils; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.List; 15 | 16 | public class BasicResourceService extends ResourceService { 17 | protected Logger logger = LoggerFactory.getLogger(BasicResourceService.class); 18 | private ProcessorConfig processorConfig; 19 | public static final String MEDISTRANO_TESTS = "medistrano_tests"; 20 | 21 | @Override 22 | public void init() { 23 | logger.info("Retrieving processor config for custom BasicResourceService"); 24 | processorConfig = ProcessorConfig.getInstance(); 25 | } 26 | 27 | @Override 28 | public String getResource(Account account, Region region, Product product, String resourceId, String[] lineItem, long millisStart) { 29 | List header = processorConfig.lineItemProcessor.getHeader(); 30 | 31 | // Use bucket name for S3 resources 32 | if (product == Product.s3) { 33 | return resourceId.toLowerCase(); 34 | } 35 | 36 | String productName = ""; 37 | String result = ""; 38 | int productIndex = header.indexOf("user:Product"); 39 | 40 | if (productIndex > 0 && lineItem.length > productIndex && StringUtils.isNotEmpty(lineItem[productIndex])) { 41 | productName = lineItem[productIndex]; 42 | 43 | if (productName.contains("ztestz")) { 44 | return MEDISTRANO_TESTS; 45 | } else { 46 | for (String tag: processorConfig.customTags) { 47 | int index = header.indexOf(tag); 48 | if (index > 0 && lineItem.length > index && StringUtils.isNotEmpty(lineItem[index])) { 49 | result = StringUtils.isEmpty(result) ? lineItem[index] : result + "_" + lineItem[index]; 50 | } 51 | } 52 | } 53 | } 54 | 55 | return StringUtils.isEmpty(result) ? product.name.toLowerCase() : result.toLowerCase(); 56 | } 57 | 58 | @Override 59 | public List> getProductsWithResources() { 60 | List> result = Lists.newArrayList(); 61 | for (Product product: ReaderConfig.getInstance().productService.getProducts()) { 62 | result.add(Lists.newArrayList(product)); 63 | } 64 | return result; 65 | } 66 | 67 | @Override 68 | public void commit() { 69 | 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/TagGroupWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.google.common.collect.Maps; 21 | import com.netflix.ice.common.AwsUtils; 22 | import com.netflix.ice.common.TagGroup; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.io.*; 27 | import java.util.Collection; 28 | import java.util.TreeMap; 29 | 30 | public class TagGroupWriter { 31 | private final static Logger logger = LoggerFactory.getLogger(TagGroupWriter.class); 32 | public final static String DB_PREFIX = "tagdb_"; 33 | 34 | private TreeMap> tagGroups; 35 | private ProcessorConfig config = ProcessorConfig.getInstance(); 36 | private String dbName; 37 | private File file; 38 | 39 | TagGroupWriter(String name) throws Exception { 40 | 41 | dbName = DB_PREFIX + name; 42 | file = new File(config.localDir, dbName); 43 | logger.info("creating TagGroupWriter for " + file); 44 | AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, file); 45 | 46 | if (file.exists()) { 47 | DataInputStream in = new DataInputStream(new FileInputStream(file)); 48 | try { 49 | tagGroups = TagGroup.Serializer.deserializeTagGroups(config, in); 50 | } 51 | finally { 52 | if (in != null) 53 | in.close(); 54 | } 55 | } 56 | else { 57 | tagGroups = Maps.newTreeMap(); 58 | } 59 | } 60 | 61 | void archive(Long monthMilli,Collection tagGroups) throws IOException { 62 | this.tagGroups.put(monthMilli, tagGroups); 63 | 64 | DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); 65 | try { 66 | TagGroup.Serializer.serializeTagGroups(out, this.tagGroups); 67 | } 68 | finally { 69 | out.close(); 70 | } 71 | 72 | logger.info(dbName + " uploading to s3..."); 73 | AwsUtils.upload(config.workS3BucketName, config.workS3BucketPrefix, config.localDir, dbName); 74 | logger.info(dbName + " uploading done."); 75 | } 76 | } 77 | 78 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/ReservationService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.amazonaws.services.ec2.model.ReservedInstances; 21 | import com.netflix.ice.common.TagGroup; 22 | import com.netflix.ice.tag.Region; 23 | import com.netflix.ice.tag.UsageType; 24 | 25 | import java.util.Collection; 26 | import java.util.Map; 27 | 28 | /** 29 | * Interface for reservations. 30 | */ 31 | public interface ReservationService { 32 | 33 | void init(); 34 | 35 | /** 36 | * Get all tag groups with reservations 37 | * @return 38 | */ 39 | Collection getTaGroups(); 40 | 41 | /** 42 | * 43 | * @return 44 | */ 45 | Ec2InstanceReservationPrice.ReservationPeriod getReservationPeriod(); 46 | 47 | /** 48 | * 49 | * @return 50 | */ 51 | Ec2InstanceReservationPrice.ReservationUtilization getReservationUtilization(); 52 | 53 | /** 54 | * Get reservation info. 55 | * @param time 56 | * @param tagGroup 57 | * @return 58 | */ 59 | ReservationInfo getReservation( 60 | long time, 61 | TagGroup tagGroup); 62 | 63 | /** 64 | * Some companies may get different price tiers at different times depending on reservation cost. 65 | * This method is to get the latest hourly price including amortized upfront for given, time, region and usage type. 66 | * @param time 67 | * @param region 68 | * @param usageType 69 | * @return 70 | */ 71 | double getLatestHourlyTotalPrice( 72 | long time, 73 | Region region, 74 | UsageType usageType); 75 | 76 | /** 77 | * Called by ReservationCapacityPoller to update reservations. 78 | * @param reservations 79 | */ 80 | void updateEc2Reservations(Map reservations); 81 | 82 | 83 | public static class ReservationInfo { 84 | public final int capacity; 85 | public final double upfrontAmortized; 86 | public final double reservationHourlyCost; 87 | 88 | public ReservationInfo (int capacity, double upfrontAmortized, double reservationHourlyCost) { 89 | this.capacity = capacity; 90 | this.upfrontAmortized = upfrontAmortized; 91 | this.reservationHourlyCost = reservationHourlyCost; 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Operation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | 23 | import java.util.List; 24 | import java.util.concurrent.ConcurrentMap; 25 | 26 | public class Operation extends Tag { 27 | 28 | protected int seq = Integer.MAX_VALUE; 29 | private Operation (String name) { 30 | super(name); 31 | } 32 | private static ConcurrentMap operations = Maps.newConcurrentMap(); 33 | 34 | public static final ReservationOperation ondemandInstances = new ReservationOperation("OndemandInstances", 0); 35 | public static final ReservationOperation reservedInstances = new ReservationOperation("ReservedInstances", 1); 36 | public static final ReservationOperation borrowedInstances = new ReservationOperation("BorrowedInstances", 2); 37 | public static final ReservationOperation lentInstances = new ReservationOperation("LentInstances", 3); 38 | public static final ReservationOperation unusedInstances = new ReservationOperation("UnusedInstances", 4); 39 | public static final ReservationOperation upfrontAmortized = new ReservationOperation("UpfrontAmortized", 5); 40 | 41 | public static Operation getOperation(String name) { 42 | 43 | Operation operation = operations.get(name); 44 | if (operation == null) { 45 | operations.putIfAbsent(name, new Operation(name)); 46 | operation = operations.get(name); 47 | } 48 | 49 | return operation; 50 | } 51 | 52 | public static List getOperations(List names) { 53 | List result = Lists.newArrayList(); 54 | for (String name: names) 55 | result.add(operations.get(name)); 56 | return result; 57 | } 58 | 59 | public static class ReservationOperation extends Operation { 60 | private ReservationOperation(String name, int seq) { 61 | super(name); 62 | this.seq = seq; 63 | operations.put(name, this); 64 | } 65 | } 66 | 67 | @Override 68 | public int compareTo(Tag t) { 69 | if (t instanceof Operation) { 70 | Operation o = (Operation)t; 71 | int result = this.seq - o.seq; 72 | return result == 0 ? this.name.compareTo(t.name) : result; 73 | } 74 | else 75 | return super.compareTo(t); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/AccountService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import com.netflix.ice.tag.Account; 21 | import com.netflix.ice.tag.Zone; 22 | 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | public interface AccountService { 27 | /** 28 | * Get account by AWS id. The AWS id is usually an un-readable 12 digit string. 29 | * @param accountId 30 | * @return Account object associated with the account id 31 | */ 32 | Account getAccountById(String accountId); 33 | 34 | /** 35 | * Get account by account name. The account name is a user defined readable string. 36 | * @param accountName 37 | * @return Account object associated with the account name 38 | */ 39 | Account getAccountByName(String accountName); 40 | 41 | /** 42 | * Get a list of accounts from given account names. 43 | * @param accountNames 44 | * @return List of accounts 45 | */ 46 | List getAccounts(List accountNames); 47 | 48 | /** 49 | * If you don't have reserved instances, you can return an empty map. 50 | * @return Map of accounts. The keys are owner accounts, the values are list of borrowing accounts. 51 | */ 52 | Map> getReservationAccounts(); 53 | 54 | /** 55 | * If you don't need to poll reservation capacity through ec2 API for other accounts, you can return an empty map. 56 | * @return Map of account access roles. The keys are reservation owner accounts, 57 | * the values are assumed roles to call ec2 describeReservedInstances on each reservation owner account. 58 | */ 59 | Map getReservationAccessRoles(); 60 | 61 | /** 62 | * If you don't need to poll reservation capacity through ec2 API for other accounts, ir if you don't use external ids, 63 | * you can return an empty map. 64 | * @return Map of account access external ids. The keys are reservation owner accounts, 65 | * the values are external ids to call ec2 describeReservedInstances on each reservation owner account. 66 | */ 67 | Map getReservationAccessExternalIds(); 68 | 69 | /** 70 | * If you don't share reserved instances among multiple accounts, you can return the input zone. 71 | * @param mapAccount 72 | * @param account 73 | * @param zone 74 | * @return Mapped zone of mapAccount. 75 | */ 76 | Zone getAccountMappedZone(Account mapAccount, Account account, Zone zone); 77 | } 78 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ApplicationGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | import org.json.JSONArray; 23 | import org.json.JSONException; 24 | import org.json.JSONObject; 25 | import org.json.JSONTokener; 26 | 27 | import java.util.Iterator; 28 | import java.util.List; 29 | import java.util.Map; 30 | 31 | public class ApplicationGroup { 32 | public final Map> data; 33 | public final String name; 34 | public final String owner; 35 | 36 | // todo: remove this constructor 37 | public ApplicationGroup(String name, String owner, List ec2s, List s3s) { 38 | this.name = name; 39 | this.owner = owner; 40 | this.data = Maps.newHashMap(); 41 | this.data.put("rds", Lists.newArrayList()); 42 | this.data.put("ec2", ec2s); 43 | this.data.put("s3", s3s); 44 | } 45 | 46 | public ApplicationGroup(String name, String owner, Map> data) { 47 | this.name = name; 48 | this.owner = owner; 49 | this.data = data; 50 | } 51 | 52 | public ApplicationGroup(String jsonStr) throws JSONException { 53 | data = Maps.newHashMap(); 54 | JSONObject json = new JSONObject(new JSONTokener(jsonStr)); 55 | name = json.getString("name"); 56 | owner = json.getString("owner"); 57 | json = json.getJSONObject("data"); 58 | Iterator keys = json.keys(); 59 | while (keys.hasNext()) { 60 | String key = keys.next(); 61 | JSONArray jsonArray = json.getJSONArray(key); 62 | List values = Lists.newArrayList(); 63 | for (int i = 0; i < jsonArray.length(); i++) 64 | values.add(jsonArray.getString(i)); 65 | data.put(key, values); 66 | } 67 | } 68 | 69 | public JSONObject getJSON() throws JSONException { 70 | JSONObject json = new JSONObject(); 71 | json.put("name", name); 72 | json.put("owner", owner); 73 | json.put("data", new JSONObject(data)); 74 | 75 | return json; 76 | } 77 | 78 | @Override 79 | public String toString() { 80 | try { 81 | return getJSON().toString(); 82 | } 83 | catch (Exception e) { 84 | return e.getMessage(); 85 | } 86 | } 87 | 88 | public String getDisplayName() { 89 | return "Application Group " + name; 90 | } 91 | 92 | public String getLink() { 93 | return "dashboard/appgroup#appgroup=" + name; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/test/com/netflix/ice/basic/EddaResourceServiceTest.java: -------------------------------------------------------------------------------- 1 | package com.netflix.ice.basic; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.util.Properties; 6 | 7 | import org.junit.Test; 8 | 9 | import com.amazonaws.util.json.JSONArray; 10 | import com.netflix.ice.tag.Product; 11 | 12 | 13 | /** 14 | * Note: These tests require a running instance of Edda running at the url configured in ice.properties, so they 15 | * strictly speaking not unit tests any more... 16 | */ 17 | public class EddaResourceServiceTest { 18 | 19 | @Test 20 | public void test() throws Exception { 21 | EddaResourceService service = new EddaResourceService(new Properties()); 22 | 23 | service.init(); 24 | 25 | // does nothing really... 26 | service.commit(); 27 | 28 | assertNotNull(service.getProductsWithResources()); 29 | 30 | assertEquals("Product-name for unsupported resource", "somename", service.getResource(null, null, new Product("somename"), null, null, 0)); 31 | assertEquals("Error for empty resourceId", "Error", service.getResource(null, null, Product.ec2, null, null, 0)); 32 | assertEquals("Error for empty resourceId", "Error", service.getResource(null, null, Product.ec2, "", null, 0)); 33 | 34 | assertEquals("Unknown for resourceIds that we do not find", "Unknown", service.getResource(null, null, Product.ec2, "someunknowninstance", null, 0)); 35 | 36 | JSONArray instances = service.readInstanceArray(); 37 | 38 | String resource = service.getResource(null, null, Product.ec2, instances.getString(0), null, 0); 39 | assertFalse("Not Error for an actual instance", "Error".equals(resource)); 40 | 41 | resource = service.getResource(null, null, Product.ec2_instance, instances.getString(0), null, 0); 42 | assertFalse("Not Error for an actual instance", "Error".equals(resource)); 43 | 44 | for(int i = 0;i < instances.length();i++) { 45 | resource = service.getResource(null, null, Product.ec2_instance, instances.getString(i), null, 0); 46 | assertFalse("Not Error for an actual instance", "Error".equals(resource)); 47 | } 48 | } 49 | 50 | @Test 51 | public void testWrongURL() throws Exception { 52 | // use a normal setup for retrieving the instances 53 | EddaResourceService service = new EddaResourceService(new Properties()); 54 | JSONArray instances = service.readInstanceArray(); 55 | 56 | // overwrite config with an invalid hostname 57 | Properties prop = new Properties(); 58 | prop.setProperty("ice.eddaresourceservice.url", "http://invalidhostname:18081/edda/api/v2/"); 59 | service = new EddaResourceService(prop); 60 | 61 | // now the retrieved resources should return an error even for valid instances 62 | String resource = service.getResource(null, null, Product.ec2, instances.getString(0), null, 0); 63 | assertTrue("Error even for an actual instance when using wrong URL", "Error".equals(resource)); 64 | 65 | // overwrite config with an invalid URL 66 | prop.setProperty("ice.eddaresourceservice.url", "sasie://invalidhostname:18081/edda/api/v2/"); 67 | service = new EddaResourceService(prop); 68 | 69 | // now the retrieved resources should return an error even for valid instances 70 | resource = service.getResource(null, null, Product.ec2, instances.getString(0), null, 0); 71 | assertTrue("Error even for an actual instance when using wrong URL", "Error".equals(resource)); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/Config.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import com.amazonaws.auth.AWSCredentialsProvider; 21 | import org.joda.time.DateTime; 22 | import org.joda.time.DateTimeZone; 23 | 24 | import java.util.Properties; 25 | 26 | public abstract class Config { 27 | 28 | public final String workS3BucketName; 29 | public final String workS3BucketPrefix; 30 | public final String localDir; 31 | public final AccountService accountService; 32 | public final ProductService productService; 33 | public final ResourceService resourceService; 34 | public final DateTime startDate; 35 | public final AWSCredentialsProvider credentialsProvider; 36 | 37 | /** 38 | * 39 | * @param properties (required) 40 | * @param credentialsProvider (required) 41 | * @param accountService (required) 42 | * @param productService (required) 43 | * @param resourceService (optional) 44 | */ 45 | public Config( 46 | Properties properties, 47 | AWSCredentialsProvider credentialsProvider, 48 | AccountService accountService, 49 | ProductService productService, 50 | ResourceService resourceService) { 51 | if (properties.getProperty(IceOptions.START_MILLIS) == null) throw new IllegalArgumentException("IceOptions.START_MILLIS must be specified"); 52 | if (properties == null) throw new IllegalArgumentException("properties must be specified"); 53 | if (credentialsProvider == null) throw new IllegalArgumentException("credentialsProvider must be specified"); 54 | if (accountService == null) throw new IllegalArgumentException("accountService must be specified"); 55 | if (productService == null) throw new IllegalArgumentException("productService must be specified"); 56 | 57 | DateTime startDate = new DateTime(Long.parseLong(properties.getProperty(IceOptions.START_MILLIS)), DateTimeZone.UTC); 58 | workS3BucketName = properties.getProperty(IceOptions.WORK_S3_BUCKET_NAME); 59 | workS3BucketPrefix = properties.getProperty(IceOptions.WORK_S3_BUCKET_PREFIX, "ice/"); 60 | localDir = properties.getProperty(IceOptions.LOCAL_DIR, "/mnt/ice"); 61 | 62 | if (workS3BucketName == null) throw new IllegalArgumentException("IceOptions.WORK_S3_BUCKET_NAME must be specified"); 63 | 64 | this.credentialsProvider = credentialsProvider; 65 | this.startDate = startDate; 66 | this.accountService = accountService; 67 | this.productService = productService; 68 | this.resourceService = resourceService; 69 | 70 | AwsUtils.init(credentialsProvider); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/ResourceService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import com.netflix.ice.tag.Account; 21 | import com.netflix.ice.tag.Product; 22 | import com.netflix.ice.tag.Region; 23 | 24 | import java.util.List; 25 | 26 | /** 27 | * Please see a sample of subclass in SampleMapDbResourceService class. 28 | */ 29 | public abstract class ResourceService { 30 | 31 | /** 32 | * Subclass can choose different technologies to store the mapping of resource ids and resource group names. 33 | * E.g. SampleMapDbResourceService used MapDB to maintain a mapping of instance ids and application names. 34 | * You can initialize you mapping here. 35 | * You can also get the reference to ProcessorConfig instance here. 36 | */ 37 | abstract public void init(); 38 | 39 | /** 40 | * Get resource group name. Subclass can maintain a mapping of resource ids and resource group names. 41 | * Users can also choose to add user-defined tags in the billing file. E.g. in SampleMapDbResourceService, 42 | * the auto-scaling-group name is used to generate the resource group name. 43 | * @param account 44 | * @param region 45 | * @param product 46 | * @param resourceId: depending on product, resourceId could be: 47 | * 1) instance id, if product is ec2_instance. You can use Edda (https://github.com/Netflix/edda) to query application name from instance id. 48 | * 2) volumn id, if product is ebs. You can use Edda (https://github.com/Netflix/edda) to query instance id from volumn id, then application name from instance id. 49 | * 3) s3 bucket name if product is s3 50 | * 4) db name if product is rds 51 | * 5) etc. 52 | * @param lineItem: the line item in the billing file. You can access your user-defined tags here. 53 | * @param millisStart 54 | * @return 55 | */ 56 | public String getResource(Account account, Region region, Product product, String resourceId, String[] lineItem, long millisStart){ 57 | return product.name; 58 | } 59 | 60 | /** 61 | * Get products with resources. See example in SampleMapDbResourceService. This method will be used by UI to list 62 | * products with resources. 63 | * @return List of list of products. The inner list of products share the same resource groups. 64 | * E.g. in SampleMapDbResourceService, for products (ec2, ec2_instance, ebs), the resource group names are application names. 65 | */ 66 | abstract public List> getProductsWithResources(); 67 | 68 | /** 69 | * Commit resource mappings. This method will be called at the end of billing file processing to commit your mappings. 70 | */ 71 | abstract public void commit(); 72 | } 73 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/UsageType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.io.DataInput; 26 | import java.io.DataOutput; 27 | import java.io.IOException; 28 | import java.util.List; 29 | import java.util.concurrent.ConcurrentMap; 30 | 31 | public class UsageType extends Tag { 32 | private static final Logger logger = LoggerFactory.getLogger(Operation.class); 33 | public final String unit; 34 | 35 | private UsageType (String name, String unit) { 36 | super(name); 37 | this.unit = unit; 38 | } 39 | private static ConcurrentMap usageTypes = Maps.newConcurrentMap(); 40 | 41 | public static void serialize(DataOutput out, UsageType usageType) throws IOException { 42 | out.writeUTF(usageType.name); 43 | out.writeUTF(usageType.unit); 44 | } 45 | 46 | public static UsageType deserialize(DataInput in) throws IOException { 47 | String name = in.readUTF(); 48 | String unit = in.readUTF(); 49 | 50 | UsageType usageType = usageTypes.get(name); 51 | if (usageType == null) { 52 | usageTypes.putIfAbsent(name, new UsageType(name, unit)); 53 | usageType = usageTypes.get(name); 54 | } 55 | else if (!usageType.unit.equals(unit)) { 56 | logger.error("found different units for " + usageType + usageType.unit + " " + unit); 57 | } 58 | return usageType; 59 | } 60 | 61 | public static UsageType getUsageType(String name, Operation operation, String description) { 62 | String unit = ""; 63 | if (name.contains("Bytes") || name.contains("ByteHrs") || description.contains("GB")) 64 | unit = "GB"; 65 | if (operation instanceof Operation.ReservationOperation) 66 | unit = "hours"; 67 | 68 | return getUsageType(name, unit); 69 | } 70 | 71 | public static UsageType getUsageType(String name, String unit) { 72 | 73 | UsageType usageType = usageTypes.get(name); 74 | if (usageType == null) { 75 | usageTypes.putIfAbsent(name, new UsageType(name, unit)); 76 | usageType = usageTypes.get(name); 77 | } 78 | else if (!usageType.unit.equals(unit)) { 79 | logger.error("found different units for " + usageType + usageType.unit + " " + unit); 80 | } 81 | return usageType; 82 | } 83 | 84 | public static List getUsageTypes(List names) { 85 | List result = Lists.newArrayList(); 86 | for (String name: names) 87 | result.add(usageTypes.get(name)); 88 | return result; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /grails-app/views/dashboard/editappgroup.gsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | 3 | Copyright 2013 Netflix, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | --%> 18 | 19 | <%@ page contentType="text/html;charset=UTF-8" %> 20 | 21 | 22 | 23 | Edit Application Group 24 | 25 | 26 |
27 | 28 |
{{message}}
29 | 30 |

CreateEdit Application Group {{appgroup.name}}

31 | 32 |
33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 49 | 53 | 57 | 58 | 59 |
Group Name:
Email:
Selected {{row.displayName}}:
46 | 47 | 48 |




50 |
51 | 52 | 		Rest of {{row.displayName}}:
54 | 55 | 56 |
60 |
61 | 62 | 63 | 64 | Cancel 65 |
66 |
67 | 68 |
69 | 70 | -------------------------------------------------------------------------------- /grails-app/conf/Config.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // locations to search for config files that get merged into the main config 18 | // config files can either be Java properties files or ConfigSlurper scripts 19 | 20 | grails.config.locations = [ 21 | 22 | ] 23 | 24 | // if(System.properties["${appName}.config.location"]) { 25 | // grails.config.locations << "file:" + System.properties["${appName}.config.location"] 26 | // } 27 | 28 | grails.project.groupId = appName // change this to alter the default package name and Maven publishing destination 29 | grails.mime.file.extensions = true // enables the parsing of file extensions from URLs into the request format 30 | grails.mime.use.accept.header = false 31 | grails.mime.types = [html: ['text/html', 'application/xhtml+xml'], 32 | xml: ['text/xml', 'application/xml'], 33 | text: 'text/plain', 34 | js: 'text/javascript', 35 | rss: 'application/rss+xml', 36 | atom: 'application/atom+xml', 37 | css: 'text/css', 38 | csv: 'text/csv', 39 | all: '*/*', 40 | json: ['application/json', 'text/json'], 41 | form: 'application/x-www-form-urlencoded', 42 | multipartForm: 'multipart/form-data' 43 | ] 44 | 45 | // URL Mapping Cache Max Size, defaults to 5000 46 | //grails.urlmapping.cache.maxsize = 1000 47 | 48 | // The default codec used to encode data with ${} 49 | grails.views.default.codec = "none" // none, html, base64 50 | grails.views.gsp.encoding = "UTF-8" 51 | grails.converters.encoding = "UTF-8" 52 | // enable Sitemesh preprocessing of GSP pages 53 | grails.views.gsp.sitemesh.preprocess = true 54 | // scaffolding templates configuration 55 | grails.scaffolding.templates.domainSuffix = 'Instance' 56 | 57 | // Set to false to use the new Grails 1.2 JSONBuilder in the render method 58 | grails.json.legacy.builder = false 59 | // enabled native2ascii conversion of i18n properties files 60 | grails.enable.native2ascii = true 61 | // whether to install the java.util.logging bridge for sl4j. Disable for AppEngine! 62 | grails.logging.jul.usebridge = true 63 | // packages to include in Spring bean scanning 64 | grails.spring.bean.packages = [] 65 | 66 | // set per-environment serverURL stem for creating absolute links 67 | environments { 68 | production { 69 | } 70 | development { 71 | grails.serverURL = "http://localhost:8080/${appName}" 72 | } 73 | test { 74 | grails.serverURL = "http://localhost:8080/${appName}" 75 | } 76 | 77 | } 78 | 79 | log4j = { 80 | error 'org.codehaus.groovy.grails.web.servlet', // controllers 81 | 'org.codehaus.groovy.grails.web.pages', // GSP 82 | 'org.codehaus.groovy.grails.web.sitemesh', // layouts 83 | 'org.codehaus.groovy.grails.web.mapping.filter', // URL mapping 84 | 'org.codehaus.groovy.grails.web.mapping', // URL mapping 85 | 'org.codehaus.groovy.grails.commons', // core / classloading 86 | 'org.codehaus.groovy.grails.plugins', // plugins 87 | 'org.codehaus.groovy.grails.orm.hibernate' // hibernate integration 88 | 89 | warn 'org.mortbay.log' 90 | 91 | root { 92 | info() 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicAccountService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.ice.basic; 17 | 18 | import com.google.common.collect.Lists; 19 | import com.google.common.collect.Maps; 20 | import com.netflix.ice.common.AccountService; 21 | import com.netflix.ice.tag.Account; 22 | import com.netflix.ice.tag.Zone; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | public class BasicAccountService implements AccountService { 30 | 31 | Logger logger = LoggerFactory.getLogger(getClass()); 32 | 33 | private Map accountsById = Maps.newConcurrentMap(); 34 | private Map accountsByName = Maps.newConcurrentMap(); 35 | private Map> reservationAccounts = Maps.newHashMap(); 36 | private Map reservationAccessRoles = Maps.newHashMap(); 37 | private Map reservationAccessExternalIds = Maps.newHashMap(); 38 | 39 | public BasicAccountService(List accounts, Map> reservationAccounts, 40 | Map reservationAccessRoles, Map reservationAccessExternalIds) { 41 | this.reservationAccounts = reservationAccounts; 42 | this.reservationAccessRoles = reservationAccessRoles; 43 | this.reservationAccessExternalIds = reservationAccessExternalIds; 44 | for (Account account: accounts) { 45 | accountsByName.put(account.name, account); 46 | accountsById.put(account.id, account); 47 | } 48 | } 49 | 50 | public Account getAccountById(String accountId) { 51 | Account account = accountsById.get(accountId); 52 | if (account == null) { 53 | account = new Account(accountId, accountId); 54 | accountsByName.put(account.name, account); 55 | accountsById.put(account.id, account); 56 | logger.info("created account " + accountId + "."); 57 | } 58 | return account; 59 | } 60 | 61 | public Account getAccountByName(String accountName) { 62 | Account account = accountsByName.get(accountName); 63 | if (account == null) { 64 | account = new Account(accountName, accountName); 65 | accountsByName.put(account.name, account); 66 | accountsById.put(account.id, account); 67 | } 68 | return account; 69 | } 70 | 71 | public List getAccounts(List accountNames) { 72 | List result = Lists.newArrayList(); 73 | for (String name: accountNames) 74 | result.add(accountsByName.get(name)); 75 | return result; 76 | } 77 | 78 | public Map> getReservationAccounts() { 79 | return reservationAccounts; 80 | } 81 | 82 | public Map getReservationAccessRoles() { 83 | return reservationAccessRoles; 84 | } 85 | 86 | 87 | public Map getReservationAccessExternalIds() { 88 | return reservationAccessExternalIds; 89 | } 90 | 91 | public Zone getAccountMappedZone(Account mapAccount, Account account, Zone zone) { 92 | return zone; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Region.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | 23 | import java.util.List; 24 | import java.util.concurrent.ConcurrentMap; 25 | 26 | public class Region extends Tag { 27 | 28 | public static final Region US_EAST_1 = new Region("us-east-1", "USE1"); 29 | public static final Region US_WEST_1 = new Region("us-west-1", "USW1"); 30 | public static final Region US_WEST_2 = new Region("us-west-2", "USW2"); 31 | public static final Region EU_WEST_1 = new Region("eu-west-1", "EU"); 32 | public static final Region AP_SOUTHEAST_1 = new Region("ap-southeast-1", "APS1"); 33 | public static final Region AP_SOUTHEAST_2 = new Region("ap-southeast-2", "APS2"); 34 | public static final Region AP_NORTHEAST_1 = new Region("ap-northeast-1","APN1"); 35 | public static final Region SA_EAST_1 = new Region("sa-east-1", "SAE1"); 36 | 37 | private static ConcurrentMap regionsByName = Maps.newConcurrentMap(); 38 | private static ConcurrentMap regionsByShortName = Maps.newConcurrentMap(); 39 | 40 | static { 41 | regionsByShortName.put(US_EAST_1.shortName, US_EAST_1); 42 | regionsByShortName.put(US_WEST_1.shortName, US_WEST_1); 43 | regionsByShortName.put(US_WEST_2.shortName, US_WEST_2); 44 | regionsByShortName.put(EU_WEST_1.shortName, EU_WEST_1); 45 | regionsByShortName.put(AP_SOUTHEAST_1.shortName, AP_SOUTHEAST_1); 46 | regionsByShortName.put(AP_SOUTHEAST_2.shortName, AP_SOUTHEAST_2); 47 | regionsByShortName.put(AP_NORTHEAST_1.shortName, AP_NORTHEAST_1); 48 | regionsByShortName.put(SA_EAST_1.shortName, SA_EAST_1); 49 | 50 | regionsByName.put(US_EAST_1.name, US_EAST_1); 51 | regionsByName.put(US_WEST_1.name, US_WEST_1); 52 | regionsByName.put(US_WEST_2.name, US_WEST_2); 53 | regionsByName.put(EU_WEST_1.name, EU_WEST_1); 54 | regionsByName.put(AP_SOUTHEAST_1.name, AP_SOUTHEAST_1); 55 | regionsByName.put(AP_SOUTHEAST_2.name, AP_SOUTHEAST_2); 56 | regionsByName.put(AP_NORTHEAST_1.name, AP_NORTHEAST_1); 57 | regionsByName.put(SA_EAST_1.name, SA_EAST_1); 58 | } 59 | 60 | public final String shortName; 61 | List zones = Lists.newArrayList(); 62 | 63 | private Region(String name, String shortName) { 64 | super(name); 65 | this.shortName = shortName; 66 | } 67 | 68 | public List getZones() { 69 | return Lists.newArrayList(zones); 70 | } 71 | 72 | void addZone(Zone zone) { 73 | zones.add(zone); 74 | } 75 | 76 | public static Region getRegionByShortName(String shortName) { 77 | return regionsByShortName.get(shortName); 78 | } 79 | 80 | public static Region getRegionByName(String name) { 81 | return regionsByName.get(name); 82 | } 83 | 84 | public static List getRegions(List names) { 85 | List result = Lists.newArrayList(); 86 | for (String name: names) 87 | result.add(regionsByName.get(name)); 88 | return result; 89 | } 90 | 91 | public static List getAllRegions() { 92 | return Lists.newArrayList(regionsByName.values()); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/Poller.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.io.InterruptedIOException; 24 | import java.nio.channels.ClosedByInterruptException; 25 | import java.util.concurrent.CountDownLatch; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | public abstract class Poller { 29 | 30 | protected Logger logger = LoggerFactory.getLogger(getClass()); 31 | private Thread pollerThread; 32 | private CountDownLatch threadDoneSignal; 33 | 34 | private void doWork(int initialDelaySec, int delaySec, boolean fixedRate) { 35 | logger.info("poller starting..."); 36 | boolean first = true; 37 | long sleepTime = delaySec * 1000L; 38 | while (true) { 39 | try { 40 | if (first) { 41 | if (initialDelaySec > 0) { 42 | Thread.sleep(initialDelaySec * 1000L); 43 | } 44 | first = false; 45 | } 46 | else if (sleepTime > 0) 47 | Thread.sleep(sleepTime); 48 | 49 | long startMillis = System.currentTimeMillis(); 50 | poll(); 51 | sleepTime = fixedRate ? delaySec * 1000L - (System.currentTimeMillis() - startMillis) : delaySec * 1000L; 52 | } 53 | catch (InterruptedException e) { 54 | break; 55 | } 56 | catch (InterruptedIOException e) { 57 | break; 58 | } 59 | catch (ClosedByInterruptException e) { 60 | break; 61 | } 62 | catch (Exception e) { 63 | logger.error("Error polling", e); 64 | } 65 | } 66 | threadDoneSignal.countDown(); 67 | logger.info("poller stopping."); 68 | } 69 | 70 | protected abstract void poll() throws Exception; 71 | 72 | protected String getThreadName() { 73 | return getClass().getName(); 74 | } 75 | 76 | public void start() { 77 | start(0, 3600, false); 78 | } 79 | 80 | public void start(final int delaySec) { 81 | start(0, delaySec, false); 82 | } 83 | 84 | public void start(final int initialDelaySec, final int delaySec, final boolean fixedRate) { 85 | threadDoneSignal = new CountDownLatch(1); 86 | pollerThread = new Thread(new Runnable() { 87 | public void run() { 88 | doWork(initialDelaySec, delaySec, fixedRate); 89 | } 90 | }, getThreadName()); 91 | pollerThread.start(); 92 | logger.info("poller thread for " + getThreadName() + " started..."); 93 | } 94 | 95 | public void shutdown() { 96 | logger.info("shutting down... trying to interrupt poller thread..."); 97 | boolean done = false; 98 | int numTries = 0; 99 | while (!done) { 100 | pollerThread.interrupt(); 101 | try { 102 | done = threadDoneSignal.await(10, TimeUnit.SECONDS); 103 | } 104 | catch (InterruptedException e) { 105 | // ingore InterruptedException here 106 | } 107 | if (!done) { 108 | numTries = numTries + 1; 109 | logger.warn("trying to interrupt write thread again " + numTries); 110 | } 111 | else { 112 | logger.info("shutted down successfully."); 113 | } 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/java/sample.properties: -------------------------------------------------------------------------------- 1 | 2 | # whether or not to start processor 3 | ice.processor=true 4 | 5 | # whether or not to start reader/UI 6 | ice.reader=false 7 | 8 | # whether or not to start reservation capacity poller 9 | ice.reservationCapacityPoller=false 10 | 11 | # reservation period, possible values are oneyear, threeyear 12 | ice.reservationPeriod=threeyear 13 | # reservation utilization, possible values are LIGHT, MEDIUM, HEAVY 14 | ice.reservationUtilization=HEAVY 15 | 16 | # url prefix, e.g. http://ice.netflix.com/. Will be used in alert emails. 17 | ice.urlPrefix= 18 | 19 | # from email address 20 | ice.fromEmail= 21 | 22 | # ec2 ondemand hourly cost threshold to send alert email. The alert email will be sent at most once per day. 23 | ice.ondemandCostAlertThreshold=250 24 | 25 | # ec2 ondemand hourly cost alert emails, separated by "," 26 | ice.ondemandCostAlertEmails= 27 | 28 | # modify the following 5 properties according to your billing files configuration. if you have multiple payer accounts, you will need to specify multiple values for each property. 29 | # s3 bucket name where the billing files are. multiple bucket names are delimited by ",". Ice must have read access to billing s3 bucket. 30 | ice.billing_s3bucketname=billing_s3bucketname1,billing_s3bucketname2 31 | # prefix of the billing files. multiple prefixes are delimited by "," 32 | ice.billing_s3bucketprefix=, 33 | # specify your payer account id here if across-accounts IAM role access is used. multiple account ids are delimited by ",". "ice.billing_payerAccountId=,222222222222" means assumed role access is only used for the second bucket. 34 | #ice.billing_payerAccountId=,123456789012 35 | # specify the assumed role name here if you use IAM role access to read from billing s3 bucket. multiple role names are delimited by ",". "ice.billing_accessRoleName=,ice" means assumed role access is only used for the second bucket. 36 | #ice.billing_accessRoleName=,ice 37 | # specify external id here if it is used. multiple external ids are delimited by ",". if you don't use external id, you can leave this property unset. 38 | #ice.billing_accessExternalId= 39 | 40 | # specify your custom tags here. Multiple tags are delimited by ",". If specified, BasicResourceService will be used to generate resource groups for you. 41 | # PLEASE MAKE SURE you have limited number (e.g. < 100) of unique value combinations from your custom tags, otherwise Ice performance will be greatly affected. 42 | #ice.customTags=tag1,tag2 43 | 44 | # start date in millis from when you want to start processing the billing files 45 | ice.startmillis=1364774400000 46 | 47 | # you company name. it will be used by UI 48 | ice.companyName=Your Company Name 49 | 50 | # s3 bucket name where Ice can store output files. Ice must have read and write access to billing s3 bucket. 51 | ice.work_s3bucketname=work_s3bucketname 52 | # prefix of Ice output files 53 | ice.work_s3bucketprefix=ice/ 54 | 55 | # local directory for Ice processor. the directory must exist. 56 | ice.processor.localDir=/mnt/ice_processor 57 | 58 | # local directory for Ice reader. the directory must exist. 59 | ice.reader.localDir=/mnt/ice_reader 60 | 61 | # monthly data cache size for Ice reader. 62 | ice.monthlycachesize=12 63 | 64 | # change the follow account settings 65 | ice.account.account1=123456789011 66 | ice.account.account2=123456789012 67 | ice.account.account3=123456789013 68 | ice.account.account4=123456789014 69 | ice.account.account5=123456789015 70 | ice.account.account6=123456789016 71 | 72 | # set reservation owner accounts. "ice.owneraccount.account2=account3,account4" means reservations in account2 can be shared by account3 and account4 73 | # if reservation capacity poller is enabled, the poller will try to poll reservation capacity through ec2 API (desribeReservedInstances) for each reservation owner account. 74 | ice.owneraccount.account1= 75 | ice.owneraccount.account2=account3,account4 76 | ice.owneraccount.account5=account6 77 | 78 | # if reservation capacity poller needs to use IAM role to access ec2 API, set the assumed role here for each reservation owner account 79 | ice.owneraccount.account1.role=ice 80 | ice.owneraccount.account2.role=ice 81 | ice.owneraccount.account5.role=ice 82 | 83 | # if reservation capacity poller needs to use IAM role to access ec2 API and external id is used, set the external id here for each reservation owner account. otherwise you can leave it unset. 84 | ice.owneraccount.account1.externalId= 85 | ice.owneraccount.account2.externalId= 86 | ice.owneraccount.account5.externalId= -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/SampleMapDbResourceService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.netflix.ice.common.ProductService; 22 | import com.netflix.ice.common.ResourceService; 23 | import com.netflix.ice.processor.ProcessorConfig; 24 | import com.netflix.ice.tag.Account; 25 | import com.netflix.ice.tag.Product; 26 | import com.netflix.ice.tag.Region; 27 | import org.apache.commons.lang.StringUtils; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.util.List; 32 | 33 | /** 34 | * 35 | */ 36 | public class SampleMapDbResourceService extends ResourceService { 37 | 38 | public static final String UNKNOWN = "unknown"; 39 | private static final Logger logger = LoggerFactory.getLogger(SampleMapDbResourceService.class); 40 | private static List> productsWithResources = Lists.>newArrayList( 41 | Lists.newArrayList(Product.ec2, Product.ec2_instance, Product.ebs), 42 | Lists.newArrayList(Product.rds), 43 | Lists.newArrayList(Product.s3)); 44 | MapDb instanceDb; 45 | ProcessorConfig config; 46 | 47 | public void init() { 48 | config = ProcessorConfig.getInstance(); 49 | instanceDb = new MapDb("instances"); 50 | } 51 | 52 | @Override 53 | public void commit() { 54 | instanceDb.commit(); 55 | } 56 | 57 | @Override 58 | public List> getProductsWithResources() { 59 | return productsWithResources; 60 | } 61 | 62 | @Override 63 | public String getResource(Account account, Region region, Product product, String resourceId, String[] lineItem, long millisStart) { 64 | 65 | if (product == Product.ec2 || product == Product.ec2_instance || product == Product.ebs || product == Product.cloudwatch) { 66 | return getEc2Resource(account, region, resourceId, lineItem, millisStart); 67 | } 68 | else if (product == Product.rds) { 69 | return getRdsResource(account, region, resourceId, lineItem, millisStart); 70 | } 71 | else if (product == Product.s3) { 72 | return getS3Resource(account, region, resourceId, lineItem, millisStart); 73 | } 74 | else if (product == Product.eip) { 75 | return null; 76 | } 77 | else { 78 | return resourceId; 79 | } 80 | } 81 | 82 | protected String getEc2Resource(Account account, Region region, String resourceId, String[] lineItem, long millisStart) { 83 | String autoScalingGroupName = lineItem.length > config.lineItemProcessor.getUserTagStartIndex() ? 84 | lineItem[config.lineItemProcessor.getUserTagStartIndex()] : null; 85 | 86 | if (StringUtils.isEmpty(autoScalingGroupName)) { 87 | return UNKNOWN; 88 | } 89 | else if (resourceId.startsWith("i-")) { 90 | String appName = autoScalingGroupName.length() > 5 ? autoScalingGroupName.substring(0, 5) : autoScalingGroupName; 91 | instanceDb.SetResource(account, region, resourceId, appName, millisStart); 92 | return autoScalingGroupName; 93 | } 94 | else { 95 | return UNKNOWN; 96 | } 97 | } 98 | 99 | protected String getRdsResource(Account account, Region region, String resourceId, String[] lineItem, long millisStart) { 100 | if (resourceId.indexOf(":db:") > 0) 101 | return resourceId.substring(resourceId.indexOf(":db:") + 4); 102 | else 103 | return resourceId; 104 | } 105 | 106 | protected String getS3Resource(Account account, Region region, String resourceId, String[] lineItem, long millisStart) { 107 | return resourceId; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/MapDb.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.amazonaws.services.s3.AmazonS3Client; 21 | import com.amazonaws.services.s3.model.S3ObjectSummary; 22 | import com.netflix.ice.common.AwsUtils; 23 | import com.netflix.ice.processor.ProcessorConfig; 24 | import com.netflix.ice.tag.Account; 25 | import com.netflix.ice.tag.Region; 26 | import org.apache.commons.lang.StringUtils; 27 | import org.mapdb.DB; 28 | import org.mapdb.DBMaker; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import java.io.File; 33 | import java.io.FilenameFilter; 34 | import java.util.Map; 35 | 36 | public class MapDb { 37 | private final Logger logger = LoggerFactory.getLogger(this.getClass()); 38 | private DB db; 39 | private Map items; 40 | private long numItemsToCommit = 0; 41 | private ProcessorConfig config; 42 | private String dbName; 43 | 44 | MapDb(String name) { 45 | this.config = ProcessorConfig.getInstance(); 46 | 47 | this.dbName = "db_" + name; 48 | File file = new File(config.localDir, dbName); 49 | if (!file.exists()) { 50 | AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); 51 | for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + this.dbName).getObjectSummaries()) { 52 | File dbFile = new File(config.localDir, s3ObjectSummary.getKey().substring(config.workS3BucketPrefix.length())); 53 | AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, dbFile); 54 | } 55 | } 56 | this.db = DBMaker.newFileDB(new File(config.localDir, this.dbName)).make(); 57 | try { 58 | this.items = db.createHashMap(name, false, null, null); 59 | } 60 | catch (IllegalArgumentException e) { 61 | this.items = db.getHashMap(name); 62 | logger.info("found " + this.items.size() + " items from mapdb for " + name); 63 | } 64 | } 65 | 66 | String getResource(Account account, Region region, String resourceId) { 67 | return this.items.get(resourceId + "|" + account + "|" + region); 68 | } 69 | 70 | void SetResource(Account account, Region region, String resourceId, String resource, long millisStart) { 71 | if (StringUtils.isEmpty(resource)) 72 | return; 73 | 74 | String key = resourceId + "|" + account + "|" + region; 75 | String resourceInDb = this.items.get(key); 76 | 77 | if (resourceInDb == null) { 78 | this.items.put(key, resource); 79 | 80 | numItemsToCommit ++; 81 | if (numItemsToCommit >= 1000) { 82 | this.commit(); 83 | numItemsToCommit = 0; 84 | } 85 | } 86 | else if (!resourceInDb.equals(resource)) { 87 | logger.error("different resources " + resourceInDb + " " + resource + " for " + resourceId); 88 | this.items.put(key, resource); 89 | resourceInDb = resource; 90 | 91 | numItemsToCommit ++; 92 | if (numItemsToCommit >= 1000) { 93 | this.commit(); 94 | numItemsToCommit = 0; 95 | } 96 | } 97 | } 98 | 99 | void commit() { 100 | this.db.commit(); 101 | upload(); 102 | logger.info("committed " + this.items.size() + "."); 103 | } 104 | 105 | void upload() { 106 | AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); 107 | 108 | File dir = new File(config.localDir); 109 | File[] files = dir.listFiles(new FilenameFilter() { 110 | public boolean accept(File file, String fileName) { 111 | return fileName.startsWith(dbName); 112 | } 113 | }); 114 | for (File file: files) 115 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + file.getName(), file); 116 | 117 | for (File file: files) 118 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "copy" + file.getName(), file); 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/TagGroupManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.netflix.ice.tag.*; 21 | import org.joda.time.Interval; 22 | 23 | import java.util.Collection; 24 | import java.util.Map; 25 | 26 | /** 27 | * Interface to manager tag groups. 28 | */ 29 | public interface TagGroupManager { 30 | 31 | /** 32 | * Get all accounts that meet query in tagLists. 33 | * @param tagLists 34 | * @return collection of accounts 35 | */ 36 | Collection getAccounts(TagLists tagLists); 37 | 38 | /** 39 | * Get all regions that meet query in tagLists. 40 | * @param tagLists 41 | * @return collection of regions 42 | */ 43 | Collection getRegions(TagLists tagLists); 44 | 45 | /** 46 | * Get all zones that meet query in tagLists. 47 | * @param tagLists 48 | * @return collection of zones 49 | */ 50 | Collection getZones(TagLists tagLists); 51 | 52 | /** 53 | * Get all products that meet query in tagLists. 54 | * @param tagLists 55 | * @return collection of products 56 | */ 57 | Collection getProducts(TagLists tagLists); 58 | 59 | /** 60 | * Get all operations that meet query in tagLists. 61 | * @param tagLists 62 | * @return collection of operations 63 | */ 64 | Collection getOperations(TagLists tagLists); 65 | 66 | /** 67 | * Get all usage types that meet query in tagLists. 68 | * @param tagLists 69 | * @return collection of usage types 70 | */ 71 | Collection getUsageTypes(TagLists tagLists); 72 | 73 | /** 74 | * Get all resource groups that meet query in tagLists. 75 | * @param tagLists 76 | * @return collection of resource groups 77 | */ 78 | Collection getResourceGroups(TagLists tagLists); 79 | 80 | /** 81 | * Get all accounts that meet query in tagLists and in specifed interval. 82 | * @param interval 83 | * @param tagLists 84 | * @return collection of accounts 85 | */ 86 | Collection getAccounts(Interval interval, TagLists tagLists); 87 | 88 | /** 89 | * Get all regions that meet query in tagLists and in specifed interval. 90 | * @param interval 91 | * @param tagLists 92 | * @return collection of regions 93 | */ 94 | Collection getRegions(Interval interval, TagLists tagLists); 95 | 96 | /** 97 | * Get all zones that meet query in tagLists and in specifed interval. 98 | * @param interval 99 | * @param tagLists 100 | * @return collection of zones 101 | */ 102 | Collection getZones(Interval interval, TagLists tagLists); 103 | 104 | /** 105 | * Get all products that meet query in tagLists and in specifed interval. 106 | * @param interval 107 | * @param tagLists 108 | * @return collection of products 109 | */ 110 | Collection getProducts(Interval interval, TagLists tagLists); 111 | 112 | /** 113 | * Get all operations that meet query in tagLists and in specifed interval. 114 | * @param interval 115 | * @param tagLists 116 | * @return collection of operations 117 | */ 118 | Collection getOperations(Interval interval, TagLists tagLists); 119 | 120 | /** 121 | * Get all usage types that meet query in tagLists and in specifed interval. 122 | * @param interval 123 | * @param tagLists 124 | * @return collection of usage types 125 | */ 126 | Collection getUsageTypes(Interval interval, TagLists tagLists); 127 | 128 | /** 129 | * Get all resource groups that meet query in tagLists and in specifed interval. 130 | * @param interval 131 | * @param tagLists 132 | * @return collection of resource groups 133 | */ 134 | Collection getResourceGroups(Interval interval, TagLists tagLists); 135 | 136 | /** 137 | * Get overlapping interval 138 | * @param interval 139 | * @return overlapping interval 140 | */ 141 | Interval getOverlapInterval(Interval interval); 142 | 143 | /** 144 | * Get map of tag lists based on group by. 145 | * @param interval 146 | * @param tagLists 147 | * @param groupBy 148 | * @param forReservation 149 | * @return 150 | */ 151 | Map getTagListsMap(Interval interval, TagLists tagLists, TagType groupBy, boolean forReservation); 152 | } 153 | -------------------------------------------------------------------------------- /grails-app/conf/BuildConfig.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import org.apache.ivy.plugins.resolver.FileSystemResolver 17 | import org.apache.ivy.plugins.resolver.URLResolver 18 | 19 | grails.project.work.dir = 'work' 20 | grails.project.class.dir = 'target/classes' 21 | grails.project.test.class.dir = 'target/test-classes' 22 | grails.project.test.reports.dir = 'target/test-reports' 23 | grails.project.war.file = "target/${appName}.war" 24 | 25 | grails.project.dependency.resolution = { 26 | // inherit Grails' default dependencies 27 | inherits("global") { 28 | } 29 | log "warn" // log level of Ivy resolver, either 'error', 'warn', 'info', 'debug' or 'verbose' 30 | repositories { 31 | grailsPlugins() 32 | grailsHome() 33 | grailsCentral() 34 | mavenCentral() 35 | 36 | // Optional custom repository for dependencies. 37 | Closure internalRepo = { 38 | String repoUrl = 'http://artifacts/ext-releases-local' 39 | String artifactPattern = '[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier]).[ext]' 40 | String ivyPattern = '[organisation]/[module]/[revision]/[module]-[revision]-ivy.[ext]' 41 | URLResolver urlLibResolver = new URLResolver() 42 | urlLibResolver.with { 43 | name = repoUrl 44 | addArtifactPattern("${repoUrl}/${artifactPattern}") 45 | addIvyPattern("${repoUrl}/${ivyPattern}") 46 | m2compatible = true 47 | } 48 | resolver urlLibResolver 49 | 50 | String localDir = System.getenv('IVY_LOCAL_REPO') ?: "${System.getProperty('user.home')}/ivy2-local" 51 | FileSystemResolver localLibResolver = new FileSystemResolver() 52 | localLibResolver.with { 53 | name = localDir 54 | addArtifactPattern("${localDir}/${artifactPattern}") 55 | addIvyPattern("${localDir}/${ivyPattern}") 56 | } 57 | resolver localLibResolver 58 | } 59 | // Comment or uncomment the next line to toggle the use of an internal artifacts repository. 60 | //internalRepo() 61 | } 62 | 63 | dependencies { 64 | 65 | compile( 66 | // Amazon Web Services programmatic interface 67 | 'com.amazonaws:aws-java-sdk:1.4.4.1', 68 | ) { 69 | // AWS defines their dependencies as open-ended, which causes problems when resolving. 70 | // See http://stackoverflow.com/a/7990573/869 71 | transitive = false 72 | } 73 | 74 | compile( 75 | // Transitive dependencies of aws-java-sdk, but also used directly. 76 | // It would be great if we could upgrade httpcore and httpclient, but we can't until the AWS Java SDK 77 | // upgrades its dependencies. If we simply upgrade these, then some Amazon calls fail. 78 | 'org.apache.httpcomponents:httpcore:4.1', 79 | 'org.apache.httpcomponents:httpclient:4.1.1', 80 | 81 | // Explicitly including aws-java-sdk transitive dependencies 82 | 'org.codehaus.jackson:jackson-core-asl:1.8.9', 83 | 'org.codehaus.jackson:jackson-mapper-asl:1.8.9', 84 | 85 | // Extra collection types and utilities 86 | 'commons-collections:commons-collections:3.2.1', 87 | 88 | // Easier Java from of the Apache Foundation 89 | 'commons-lang:commons-lang:2.4', 90 | 91 | // Easier Java from Joshua Bloch and Google 92 | 'com.google.guava:guava:14.0', 93 | 94 | // Send emails about system errors and task completions 95 | 'javax.mail:mail:1.4.1', 96 | 97 | // Better date API 98 | 'joda-time:joda-time:2.0', 99 | 100 | 'net.sourceforge.javacsv:javacsv:2.0', 101 | 102 | 'org.apache.poi:poi-ooxml:3.7', 103 | 'org.codehaus.woodstox:wstx-asl:3.2.9', 104 | 'jfree:jfreechart:1.0.13', 105 | 'org.json:json:20090211', 106 | 'org.mapdb:mapdb:0.9.1' 107 | 108 | ) { // Exclude superfluous and dangerous transitive dependencies 109 | excludes( 110 | // Some libraries bring older versions of JUnit as a transitive dependency and that can interfere 111 | // with Grails' built in JUnit 112 | 'junit', 113 | 114 | 'mockito-core', 115 | ) 116 | } 117 | } 118 | 119 | plugins { 120 | build ":tomcat:$grailsVersion" 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/ReadWriteData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | import com.google.common.collect.Sets; 23 | import com.netflix.ice.common.TagGroup; 24 | 25 | import java.io.DataInput; 26 | import java.io.DataOutput; 27 | import java.io.IOException; 28 | import java.util.Collection; 29 | import java.util.List; 30 | import java.util.Map; 31 | import java.util.Set; 32 | 33 | public class ReadWriteData { 34 | private List> data; 35 | 36 | public ReadWriteData() { 37 | data = Lists.newArrayList(); 38 | } 39 | 40 | private ReadWriteData(List> data) { 41 | this.data = data; 42 | } 43 | 44 | public int getNum() { 45 | return data.size(); 46 | } 47 | 48 | void cutData(int num) { 49 | if (data.size() > num) 50 | data = data.subList(0, num); 51 | } 52 | 53 | public Map getData(int i) { 54 | return getCreateData(data, i); 55 | } 56 | 57 | void setData(List> newData, int startIndex, boolean merge) { 58 | for (int i = 0; i < newData.size(); i++) { 59 | int index = startIndex + i; 60 | 61 | if (index > data.size()) { 62 | getCreateData(data, index-1); 63 | } 64 | if (index >= data.size()) { 65 | data.add(newData.get(i)); 66 | } 67 | else { 68 | if (merge) { 69 | Map existed = data.get(index); 70 | for (Map.Entry entry: newData.get(i).entrySet()) { 71 | existed.put(entry.getKey(), entry.getValue()); 72 | } 73 | } 74 | else { 75 | data.set(index, newData.get(i)); 76 | } 77 | } 78 | } 79 | } 80 | 81 | static Map getCreateData(List> data, int i) { 82 | if (i >= data.size()) { 83 | for (int j = data.size(); j <= i; j++) { 84 | data.add(Maps.newHashMap()); 85 | } 86 | } 87 | return data.get(i); 88 | } 89 | 90 | public Collection getTagGroups() { 91 | Set keys = Sets.newTreeSet(); 92 | 93 | for (Map map: data) { 94 | keys.addAll(map.keySet()); 95 | } 96 | 97 | return keys; 98 | } 99 | 100 | public static class Serializer { 101 | public static void serialize(DataOutput out, ReadWriteData data) throws IOException { 102 | 103 | Collection keys = data.getTagGroups(); 104 | out.writeInt(keys.size()); 105 | for (TagGroup tagGroup: keys) { 106 | TagGroup.Serializer.serialize(out, tagGroup); 107 | } 108 | 109 | out.writeInt(data.data.size()); 110 | for (int i = 0; i < data.data.size(); i++) { 111 | Map map = data.getData(i); 112 | out.writeBoolean(map.size() > 0); 113 | if (map.size() > 0) { 114 | for (TagGroup tagGroup: keys) { 115 | Double v = map.get(tagGroup); 116 | out.writeDouble(v == null ? 0 : v); 117 | } 118 | } 119 | } 120 | } 121 | 122 | public static ReadWriteData deserialize(DataInput in) throws IOException { 123 | 124 | int numKeys = in.readInt(); 125 | List keys = Lists.newArrayList(); 126 | for (int j = 0; j < numKeys; j++) { 127 | keys.add(TagGroup.Serializer.deserialize(ProcessorConfig.getInstance(), in)); 128 | } 129 | 130 | List> data = Lists.newArrayList(); 131 | int num = in.readInt(); 132 | for (int i = 0; i < num; i++) { 133 | Map map = Maps.newHashMap(); 134 | boolean hasData = in.readBoolean(); 135 | if (hasData) { 136 | for (int j = 0; j < keys.size(); j++) { 137 | double v = in.readDouble(); 138 | if (v != 0) { 139 | map.put(keys.get(j), v); 140 | } 141 | } 142 | } 143 | data.add(map); 144 | } 145 | 146 | return new ReadWriteData(data); 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /grails-app/views/dashboard/summary.gsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | 3 | Copyright 2013 Netflix, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | --%> 18 | 19 | <%@ page contentType="text/html;charset=UTF-8" %> 20 | 21 | 22 | 23 | Aws Usage Summary 24 | 25 | 26 |
27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 41 | 45 | 49 | 53 | 57 | 61 | 62 |
Group ByAccountRegionProductOperationUsageType
39 | 40 | 42 | 43 |
44 | 		46 | 47 |
48 | 		50 | 51 |
52 | 		54 | 55 |
56 | 		58 | 59 |
60 |
63 |
64 | 65 | Submit 68 |
69 | 70 |
71 |
72 |
73 |
74 | SHOW ALL 75 | HIDE ALL 76 | 77 |
78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 95 | 96 | 97 |
{{legendName}}{{monthFormat(month)}}
{{header.name}}
{{legend.name}}		 92 | {{currencySign}} {{data[legend.name][header.index] | number:2}} 93 | {{currencySign}} {{data[legend.name][header.index] | number:2}} 94 |
98 |
99 | 100 |
101 | 102 | -------------------------------------------------------------------------------- /grails-app/views/layouts/main.gsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | 3 | Copyright 2013 Netflix, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | --%> 18 | 19 | <%@ page import="com.netflix.ice.reader.ReaderConfig" %> 20 | 21 | 22 | 23 | 24 | <g:layoutTitle default="${ReaderConfig.getInstance().companyName} AWS Usage Dashboard"/> 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |
33 |
34 | 35 | 36 | ${ReaderConfig.getInstance().companyName} AWS Usage Dashboard 37 | 38 | 39 |
40 |
41 | 72 |
73 | 74 | 76 | 77 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicS3ApplicationGroupService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.amazonaws.services.s3.AmazonS3Client; 21 | import com.amazonaws.services.s3.model.ObjectMetadata; 22 | import com.google.common.collect.Maps; 23 | import com.netflix.ice.common.AwsUtils; 24 | import com.netflix.ice.reader.ApplicationGroup; 25 | import com.netflix.ice.reader.ApplicationGroupService; 26 | import com.netflix.ice.reader.ReaderConfig; 27 | import org.apache.commons.io.IOUtils; 28 | import org.json.JSONException; 29 | import org.json.JSONObject; 30 | import org.json.JSONTokener; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | import java.io.ByteArrayInputStream; 35 | import java.io.InputStream; 36 | import java.util.Iterator; 37 | import java.util.Map; 38 | 39 | public class BasicS3ApplicationGroupService implements ApplicationGroupService { 40 | private final static Logger logger = LoggerFactory.getLogger(BasicS3ApplicationGroupService.class); 41 | private AmazonS3Client s3Client; 42 | private ReaderConfig config; 43 | 44 | public void init() { 45 | this.config = ReaderConfig.getInstance(); 46 | s3Client = AwsUtils.getAmazonS3Client(); 47 | } 48 | 49 | private String getJson(Map appgroups) throws JSONException { 50 | JSONObject json = new JSONObject(); 51 | for (String name: appgroups.keySet()) { 52 | ApplicationGroup appgroup = appgroups.get(name); 53 | json.put(name, appgroup.getJSON()); 54 | } 55 | 56 | return json.toString(); 57 | } 58 | 59 | public Map getApplicationGroups() { 60 | String jsonStr; 61 | try { 62 | InputStream in = s3Client.getObject(config.workS3BucketName, config.workS3BucketPrefix + "appgroups").getObjectContent(); 63 | jsonStr = IOUtils.toString(in); 64 | in.close(); 65 | } 66 | catch (Exception e) { 67 | logger.error("Error reading from appgroups file", e); 68 | try { 69 | InputStream in = s3Client.getObject(config.workS3BucketName, config.workS3BucketPrefix + "copy_appgroups").getObjectContent(); 70 | jsonStr = IOUtils.toString(in); 71 | in.close(); 72 | } 73 | catch (Exception r) { 74 | logger.error("Error reading from copy_appgroups file", r); 75 | return Maps.newHashMap(); 76 | } 77 | } 78 | 79 | try { 80 | JSONObject json = new JSONObject(new JSONTokener(jsonStr)); 81 | Map appgroups = Maps.newHashMap(); 82 | Iterator keys = json.keys(); 83 | while (keys.hasNext()) { 84 | String key = keys.next(); 85 | String str = json.getString(key); 86 | appgroups.put(key, new ApplicationGroup(str)); 87 | } 88 | 89 | return appgroups; 90 | } 91 | catch (JSONException e) { 92 | logger.error("Error reading appgroups from json...", e); 93 | return Maps.newHashMap(); 94 | } 95 | } 96 | 97 | public ApplicationGroup getApplicationGroup(String name) { 98 | Map appgroups = getApplicationGroups(); 99 | return appgroups.get(name); 100 | } 101 | 102 | public boolean saveApplicationGroup(ApplicationGroup appgroup) { 103 | Map appgroups = getApplicationGroups(); 104 | appgroups.put(appgroup.name, appgroup); 105 | 106 | try { 107 | String json = getJson(appgroups); 108 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "appgroups", IOUtils.toInputStream(json), new ObjectMetadata()); 109 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "copy_appgroups", IOUtils.toInputStream(json), new ObjectMetadata()); 110 | 111 | BasicS3ApplicationGroupService.logger.info("saved appgroup " + appgroup); 112 | return true; 113 | } 114 | catch (JSONException e) { 115 | logger.error("Error saving appgroup " + appgroup, e); 116 | return false; 117 | } 118 | } 119 | 120 | public boolean deleteApplicationGroup(String name) { 121 | Map appgroups = getApplicationGroups(); 122 | ApplicationGroup appgroup = appgroups.remove(name); 123 | 124 | try { 125 | String json = getJson(appgroups); 126 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "appgroups", new ByteArrayInputStream(json.getBytes()), new ObjectMetadata()); 127 | 128 | BasicS3ApplicationGroupService.logger.info("delete appgroup " + name + " " + appgroup); 129 | return true; 130 | } 131 | catch (JSONException e) { 132 | logger.error("Error deleting appgroup " + appgroup, e); 133 | return false; 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicProductService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | import com.netflix.ice.common.ProductService; 23 | import com.netflix.ice.tag.Product; 24 | 25 | import java.util.Collection; 26 | import java.util.List; 27 | import java.util.concurrent.ConcurrentMap; 28 | 29 | public class BasicProductService implements ProductService { 30 | 31 | private static ConcurrentMap productsByAwsName = Maps.newConcurrentMap(); 32 | private static ConcurrentMap productsByName = Maps.newConcurrentMap(); 33 | 34 | public static final BasicProduct cloudfront = new BasicProduct(Product.cloudfront, "CloudFront"); 35 | public static final BasicProduct cloudhsm = new BasicProduct(Product.cloudhsm, "CloudHSM"); 36 | public static final BasicProduct cloudwatch = new BasicProduct(Product.cloudwatch, "CloudWatch"); 37 | public static final BasicProduct data_transfer = new BasicProduct(Product.data_transfer, "Data Transfer"); 38 | public static final BasicProduct direct_connect = new BasicProduct(Product.direct_connect, "Direct Connect"); 39 | public static final BasicProduct dynamodb = new BasicProduct(Product.dynamodb, "DynamoDB"); 40 | public static final BasicProduct ebs = new BasicProduct(Product.ebs, "ebs"); 41 | public static final BasicProduct ec2 = new BasicProduct(Product.ec2, "Elastic Compute Cloud"); 42 | public static final BasicProduct ec2_instance = new BasicProduct(Product.ec2_instance, "ec2_instance"); 43 | public static final BasicProduct eip = new BasicProduct(Product.eip, "eip"); 44 | public static final BasicProduct elasticache = new BasicProduct(Product.elasticache, "ElastiCache"); 45 | public static final BasicProduct emr = new BasicProduct(Product.emr, "Elastic MapReduce"); 46 | public static final BasicProduct glacier = new BasicProduct(Product.glacier, "Glacier"); 47 | public static final BasicProduct rds = new BasicProduct(Product.rds, "RDS Service"); 48 | public static final BasicProduct redshift = new BasicProduct(Product.redshift, "Redshift"); 49 | public static final BasicProduct route53 = new BasicProduct(Product.route53, "Route 53"); 50 | public static final BasicProduct s3 = new BasicProduct(Product.s3, "Simple Storage Service"); 51 | public static final BasicProduct simpledb = new BasicProduct(Product.simpledb, "SimpleDB"); 52 | public static final BasicProduct ses = new BasicProduct(Product.ses, "Simple Email Service"); 53 | public static final BasicProduct sns = new BasicProduct(Product.sns, "Simple Notification Service"); 54 | public static final BasicProduct sqs = new BasicProduct(Product.sqs, "Simple Queue Service"); 55 | public static final BasicProduct sws = new BasicProduct(Product.sws, "Simple Workflow Service"); 56 | public static final BasicProduct vpc = new BasicProduct(Product.vpc, "Virtual Private Cloud"); 57 | public static final BasicProduct monitor = new BasicProduct(Product.monitor, "monitor"); 58 | 59 | private static BasicProduct[] products = new BasicProduct[]{cloudfront, cloudhsm, cloudwatch, data_transfer, direct_connect, dynamodb, ebs, ec2, ec2_instance, eip, elasticache, emr, glacier, rds, redshift, route53, s3, simpledb, ses, sns, sqs, sws, vpc}; 60 | 61 | static { 62 | for (BasicProduct product: products) { 63 | productsByAwsName.put("AWS " + product.awsName, product.product); 64 | productsByAwsName.put("Amazon " + product.awsName, product.product); 65 | productsByName.put(product.product.name, product.product); 66 | } 67 | productsByAwsName.put(monitor.awsName, monitor.product); 68 | productsByName.put(monitor.product.name, monitor.product); 69 | } 70 | 71 | public Product getProductByAwsName(String awsName) { 72 | Product product = productsByAwsName.get(awsName); 73 | if (product == null) { 74 | product = new Product(awsName); 75 | productsByAwsName.put(awsName, product); 76 | productsByName.put(awsName, product); 77 | } 78 | return product; 79 | } 80 | 81 | public Product getProductByName(String name) { 82 | Product product = productsByName.get(name); 83 | if (product == null) { 84 | product = new Product(name); 85 | productsByAwsName.put(name, product); 86 | productsByName.put(name, product); 87 | } 88 | return product; 89 | } 90 | 91 | public List getProducts(List names) { 92 | List result = Lists.newArrayList(); 93 | for (String name: names) 94 | result.add(productsByName.get(name)); 95 | return result; 96 | } 97 | 98 | public Collection getProducts() { 99 | return productsByName.values(); 100 | } 101 | 102 | private static class BasicProduct { 103 | private final String awsName; 104 | private final Product product; 105 | BasicProduct(Product product, String awsName) { 106 | this.product = product; 107 | this.awsName = awsName; 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/ProcessorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.amazonaws.auth.AWSCredentialsProvider; 21 | import com.netflix.ice.common.*; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.util.Properties; 26 | 27 | public class ProcessorConfig extends Config { 28 | private static final Logger logger = LoggerFactory.getLogger(ProcessorConfig.class); 29 | private static ProcessorConfig instance; 30 | private static ReservationCapacityPoller reservationCapacityPoller; 31 | private static BillingFileProcessor billingFileProcessor; 32 | 33 | public final String[] billingAccountIds; 34 | public final String[] billingS3BucketNames; 35 | public final String[] billingS3BucketPrefixes; 36 | public final String[] billingAccessRoleNames; 37 | public final String[] billingAccessExternalIds; 38 | 39 | public final String[] customTags; 40 | public final ReservationService reservationService; 41 | public final LineItemProcessor lineItemProcessor; 42 | public final Randomizer randomizer; 43 | public final double costPerMonitorMetricPerHour; 44 | 45 | public final String useCostForResourceGroup; 46 | 47 | /** 48 | * 49 | * @param properties (required) 50 | * @param accountService (required) 51 | * @param productService (required) 52 | * @param reservationService (required) 53 | * @param resourceService (optional) 54 | * @param lineItemProcessor (required) 55 | * @param randomizer (optional) 56 | */ 57 | public ProcessorConfig( 58 | Properties properties, 59 | AWSCredentialsProvider credentialsProvider, 60 | AccountService accountService, 61 | ProductService productService, 62 | ReservationService reservationService, 63 | ResourceService resourceService, 64 | LineItemProcessor lineItemProcessor, 65 | Randomizer randomizer) { 66 | 67 | super(properties, credentialsProvider, accountService, productService, resourceService); 68 | 69 | if (reservationService == null) throw new IllegalArgumentException("reservationService must be specified"); 70 | if (lineItemProcessor == null) throw new IllegalArgumentException("lineItemProcessor must be specified"); 71 | 72 | this.reservationService = reservationService; 73 | this.lineItemProcessor = lineItemProcessor; 74 | this.randomizer = randomizer; 75 | 76 | if (properties.getProperty(IceOptions.COST_PER_MONITORMETRIC_PER_HOUR) != null) 77 | this.costPerMonitorMetricPerHour = Double.parseDouble(properties.getProperty(IceOptions.COST_PER_MONITORMETRIC_PER_HOUR)); 78 | else 79 | this.costPerMonitorMetricPerHour = 0; 80 | 81 | billingS3BucketNames = properties.getProperty(IceOptions.BILLING_S3_BUCKET_NAME).split(","); 82 | billingS3BucketPrefixes = properties.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, "").split(","); 83 | billingAccountIds = properties.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, "").split(","); 84 | billingAccessRoleNames = properties.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, "").split(","); 85 | billingAccessExternalIds = properties.getProperty(IceOptions.BILLING_ACCESS_EXTERNALID, "").split(","); 86 | 87 | customTags = properties.getProperty(IceOptions.CUSTOM_TAGS, "").split(","); 88 | 89 | useCostForResourceGroup = properties.getProperty(IceOptions.RESOURCE_GROUP_COST, "modeled"); 90 | 91 | ProcessorConfig.instance = this; 92 | 93 | reservationService.init(); 94 | if (resourceService != null) 95 | resourceService.init(); 96 | 97 | billingFileProcessor = new BillingFileProcessor( 98 | properties.getProperty(IceOptions.URL_PREFIX), 99 | properties.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD) == null ? null : Double.parseDouble(properties.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD)), 100 | properties.getProperty(IceOptions.FROM_EMAIL), 101 | properties.getProperty(IceOptions.ONDEMAND_COST_ALERT_EMAILS)); 102 | } 103 | 104 | public void start (ReservationCapacityPoller reservationCapacityPoller) { 105 | logger.info("starting up..."); 106 | 107 | ProcessorConfig.reservationCapacityPoller = reservationCapacityPoller; 108 | if (reservationCapacityPoller != null) 109 | reservationCapacityPoller.start(); 110 | 111 | while (reservationCapacityPoller != null && !reservationCapacityPoller.updatedConfig()) { 112 | try { 113 | Thread.sleep(10000L); 114 | } 115 | catch (InterruptedException e) { 116 | } 117 | } 118 | 119 | billingFileProcessor.start(300); 120 | } 121 | 122 | public void shutdown() { 123 | logger.info("Shutting down..."); 124 | 125 | billingFileProcessor.shutdown(); 126 | reservationCapacityPoller.shutdown(); 127 | } 128 | 129 | /** 130 | * 131 | * @return singlton instance 132 | */ 133 | public static ProcessorConfig getInstance() { 134 | return instance; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/IceOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | public class IceOptions { 21 | 22 | /** 23 | * Data start date in millis. 24 | */ 25 | public static final String START_MILLIS = "ice.startmillis"; 26 | 27 | /** 28 | * Property for company name. It must be specified in ReaderConfig. 29 | */ 30 | public static final String COMPANY_NAME = "ice.companyName"; 31 | 32 | /** 33 | * Property for currency sign. The default value is "$". 34 | */ 35 | public static final String CURRENCY_SIGN = "ice.currencySign"; 36 | 37 | /** 38 | * Property for currency rate. The default value is "1". 39 | */ 40 | public static final String CURRENCY_RATE = "ice.currencyRate"; 41 | 42 | /** 43 | * s3 bucket name where billing files are located. For multiple payer accounts, multiple bucket names can be specified delimited by comma ",". 44 | * Only read permission is needed. It must be specified in Config. 45 | */ 46 | public static final String BILLING_S3_BUCKET_NAME = "ice.billing_s3bucketname"; 47 | 48 | /** 49 | * Prefix of billing files in billing s3 bucket. For multiple payer accounts, multiple bucket prefixes can be specified delimited by comma ",". 50 | * It must be specified in Config. 51 | */ 52 | public static final String BILLING_S3_BUCKET_PREFIX = "ice.billing_s3bucketprefix"; 53 | 54 | /** 55 | * Payer account id. Must be specified if across-accounts role is used to access billing files. For multiple payer accounts, acocunt ids can 56 | * be specified delimited by comma ",". 57 | */ 58 | public static final String BILLING_PAYER_ACCOUNT_ID = "ice.billing_payerAccountId"; 59 | 60 | /** 61 | * Billing file access role name to assume. Must be specified if across-accounts role is used to access billing files. For multiple payer accounts, 62 | * role names can be specified delimited by comma ",". 63 | */ 64 | public static final String BILLING_ACCESS_ROLENAME = "ice.billing_accessRoleName"; 65 | 66 | /** 67 | * Billing file access external ID. It is optional. Specify it if cross-accounts role is used to access billing files and external id is needed. 68 | * For multiple payer accounts, external ids can be specified delimited by comma ",". 69 | */ 70 | public static final String BILLING_ACCESS_EXTERNALID = "ice.billing_accessExternalId"; 71 | 72 | /** 73 | * User can configure their custom tags. 74 | */ 75 | public static final String CUSTOM_TAGS = "ice.customTags"; 76 | 77 | /** 78 | * s3 bucket name where output files are to be store. Both read and write permissions are needed. It must be specified in Config. 79 | */ 80 | public static final String WORK_S3_BUCKET_NAME = "ice.work_s3bucketname"; 81 | 82 | /** 83 | * Prefix of output files in output s3 bucket. It must be specified in Config. 84 | */ 85 | public static final String WORK_S3_BUCKET_PREFIX = "ice.work_s3bucketprefix"; 86 | 87 | /** 88 | * Local directory. It must be specified in Config. 89 | */ 90 | public static final String LOCAL_DIR = "ice.localDir"; 91 | 92 | /** 93 | * Monthly data cache size for reader. Default is 12. 94 | */ 95 | public static final String MONTHLY_CACHE_SIZE = "ice.monthlycachesize"; 96 | 97 | /** 98 | * Cost per monitor metric per hour, It's optional. 99 | */ 100 | public static final String COST_PER_MONITORMETRIC_PER_HOUR = "ice.cost_per_monitormetric_per_hour"; 101 | 102 | /** 103 | * url prefix, e.g. http://ice.netflix.com/ 104 | */ 105 | public static final String URL_PREFIX = "ice.urlPrefix"; 106 | 107 | /** 108 | * from email address. It must be registered in aws ses. 109 | */ 110 | public static final String FROM_EMAIL = "ice.fromEmail"; 111 | 112 | /** 113 | * ec2 ondemand hourly cost threshold to send alert email. The alert email will be sent at most once per day. 114 | */ 115 | public static final String ONDEMAND_COST_ALERT_THRESHOLD = "ice.ondemandCostAlertThreshold"; 116 | 117 | /** 118 | * ec2 ondemand hourly cost alert emails, separated by "," 119 | */ 120 | public static final String ONDEMAND_COST_ALERT_EMAILS = "ice.ondemandCostAlertEmails"; 121 | 122 | /** 123 | * What pricing data ice should use when calculating usage costs for resource groups 124 | */ 125 | public static final String RESOURCE_GROUP_COST = "ice.resourceGroupCost"; 126 | 127 | /** 128 | * Enable weekly cost email per application groups 129 | */ 130 | public static final String WEEKLYEMAILS = "ice.weeklyCostEmails"; 131 | 132 | /** 133 | * from email address for weekly cost emails. Must be registered in aws ses. 134 | */ 135 | public static final String WEEKLYFROM = "ice.weeklyCostEmails_fromEmail"; 136 | 137 | /** 138 | * bcc email address for weekly cost emails. 139 | */ 140 | public static final String WEEKLYBCC = "ice.weeklyCostEmails_bccEmail"; 141 | 142 | /** 143 | * from email to use when test flag is enabled. 144 | */ 145 | public static final String WEEKLYTEST = "ice.weeklyCostEmails_testEmail"; 146 | 147 | /** 148 | * from email to use when test flag is enabled. 149 | */ 150 | public static final String NUM_WEEKS_FOR_WEEKLYEMAILS = "ice.weeklyCostEmails_numWeeks"; 151 | } 152 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Zone.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | 23 | import java.util.Collection; 24 | import java.util.List; 25 | import java.util.concurrent.ConcurrentMap; 26 | 27 | public class Zone extends Tag { 28 | 29 | public final Region region; 30 | 31 | private Zone (Region region, String name) { 32 | super(name); 33 | this.region = region; 34 | region.addZone(this); 35 | } 36 | 37 | public static final Zone US_EAST_1A = new Zone(Region.US_EAST_1, "us-east-1a"); 38 | public static final Zone US_EAST_1B = new Zone(Region.US_EAST_1, "us-east-1b"); 39 | public static final Zone US_EAST_1C = new Zone(Region.US_EAST_1, "us-east-1c"); 40 | public static final Zone US_EAST_1D = new Zone(Region.US_EAST_1, "us-east-1d"); 41 | public static final Zone US_EAST_1E = new Zone(Region.US_EAST_1, "us-east-1e"); 42 | 43 | public static final Zone US_WEST_1A = new Zone(Region.US_WEST_1, "us-west-1a"); 44 | public static final Zone US_WEST_1B = new Zone(Region.US_WEST_1, "us-west-1b"); 45 | public static final Zone US_WEST_1C = new Zone(Region.US_WEST_1, "us-west-1c"); 46 | 47 | public static final Zone US_WEST_2A = new Zone(Region.US_WEST_2, "us-west-2a"); 48 | public static final Zone US_WEST_2B = new Zone(Region.US_WEST_2, "us-west-2b"); 49 | public static final Zone US_WEST_2C = new Zone(Region.US_WEST_2, "us-west-2c"); 50 | 51 | public static final Zone EU_WEST_1A = new Zone(Region.EU_WEST_1, "eu-west-1a"); 52 | public static final Zone EU_WEST_1B = new Zone(Region.EU_WEST_1, "eu-west-1b"); 53 | public static final Zone EU_WEST_1C = new Zone(Region.EU_WEST_1, "eu-west-1c"); 54 | 55 | public static final Zone SA_EAST_1A = new Zone(Region.SA_EAST_1, "sa-east-1a"); 56 | public static final Zone SA_EAST_1B = new Zone(Region.SA_EAST_1, "sa-east-1b"); 57 | 58 | public static final Zone AP_NORTHEAST_1A = new Zone(Region.AP_NORTHEAST_1, "ap-northeast-1a"); 59 | public static final Zone AP_NORTHEAST_1B = new Zone(Region.AP_NORTHEAST_1, "ap-northeast-1b"); 60 | public static final Zone AP_NORTHEAST_1C = new Zone(Region.AP_NORTHEAST_1, "ap-northeast-1c"); 61 | 62 | public static final Zone AP_SOUTHEAST_1A = new Zone(Region.AP_SOUTHEAST_1, "ap-southeast-1a"); 63 | public static final Zone AP_SOUTHEAST_1B = new Zone(Region.AP_SOUTHEAST_1, "ap-southeast-1b"); 64 | 65 | public static final Zone AP_SOUTHEAST_2A = new Zone(Region.AP_SOUTHEAST_2, "ap-southeast-2a"); 66 | public static final Zone AP_SOUTHEAST_2B = new Zone(Region.AP_SOUTHEAST_2, "ap-southeast-2b"); 67 | 68 | private static ConcurrentMap zonesByName = Maps.newConcurrentMap(); 69 | 70 | static { 71 | zonesByName.put(US_EAST_1A.name, US_EAST_1A); 72 | zonesByName.put(US_EAST_1B.name, US_EAST_1B); 73 | zonesByName.put(US_EAST_1C.name, US_EAST_1C); 74 | zonesByName.put(US_EAST_1D.name, US_EAST_1D); 75 | zonesByName.put(US_EAST_1E.name, US_EAST_1E); 76 | 77 | zonesByName.put(US_WEST_1A.name, US_WEST_1A); 78 | zonesByName.put(US_WEST_1B.name, US_WEST_1B); 79 | zonesByName.put(US_WEST_1C.name, US_WEST_1C); 80 | 81 | zonesByName.put(US_WEST_2A.name, US_WEST_2A); 82 | zonesByName.put(US_WEST_2B.name, US_WEST_2B); 83 | zonesByName.put(US_WEST_2C.name, US_WEST_2C); 84 | 85 | zonesByName.put(EU_WEST_1A.name, EU_WEST_1A); 86 | zonesByName.put(EU_WEST_1B.name, EU_WEST_1B); 87 | zonesByName.put(EU_WEST_1C.name, EU_WEST_1C); 88 | 89 | zonesByName.put(SA_EAST_1A.name, SA_EAST_1A); 90 | zonesByName.put(SA_EAST_1B.name, SA_EAST_1B); 91 | 92 | zonesByName.put(AP_NORTHEAST_1A.name, AP_NORTHEAST_1A); 93 | zonesByName.put(AP_NORTHEAST_1B.name, AP_NORTHEAST_1B); 94 | zonesByName.put(AP_NORTHEAST_1C.name, AP_NORTHEAST_1C); 95 | 96 | zonesByName.put(AP_SOUTHEAST_1A.name, AP_SOUTHEAST_1A); 97 | zonesByName.put(AP_SOUTHEAST_1B.name, AP_SOUTHEAST_1B); 98 | 99 | zonesByName.put(AP_SOUTHEAST_2A.name, AP_SOUTHEAST_2A); 100 | zonesByName.put(AP_SOUTHEAST_2B.name, AP_SOUTHEAST_2B); 101 | } 102 | 103 | public static void addZone(Zone zone) { 104 | Zone existedZone = zonesByName.putIfAbsent(zone.name, zone); 105 | if (existedZone != null) { 106 | throw new RuntimeException("Zone with shortname already exists " + existedZone); 107 | } 108 | } 109 | 110 | public static Zone getZone(String name, Region region) { 111 | if (name.isEmpty() || name.equals(region.name)) 112 | return null; 113 | Zone zone = zonesByName.get(name); 114 | if (zone == null) { 115 | zonesByName.putIfAbsent(name, new Zone(region, name)); 116 | zone = zonesByName.get(name); 117 | } 118 | return zone; 119 | } 120 | 121 | public static Zone getZone(String name) { 122 | return zonesByName.get(name); 123 | } 124 | 125 | public static Collection getZones() { 126 | return zonesByName.values(); 127 | } 128 | 129 | public static List getZones(List names) { 130 | List result = Lists.newArrayList(); 131 | for (String name: names) 132 | result.add(zonesByName.get(name)); 133 | return result; 134 | } 135 | } 136 | 137 | -------------------------------------------------------------------------------- /grailsw.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem ## 4 | @rem Grails JVM Bootstrap for Windows ## 5 | @rem ## 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set CLASS=org.grails.wrapper.GrailsWrapper 12 | 13 | if exist "%USERPROFILE%/.groovy/preinit.bat" call "%USERPROFILE%/.groovy/preinit.bat" 14 | 15 | @rem Determine the command interpreter to execute the "CD" later 16 | set COMMAND_COM="cmd.exe" 17 | if exist "%SystemRoot%\system32\cmd.exe" set COMMAND_COM="%SystemRoot%\system32\cmd.exe" 18 | if exist "%SystemRoot%\command.com" set COMMAND_COM="%SystemRoot%\command.com" 19 | 20 | @rem Use explicit find.exe to prevent cygwin and others find.exe from being used 21 | set FIND_EXE="find.exe" 22 | if exist "%SystemRoot%\system32\find.exe" set FIND_EXE="%SystemRoot%\system32\find.exe" 23 | if exist "%SystemRoot%\command\find.exe" set FIND_EXE="%SystemRoot%\command\find.exe" 24 | 25 | :check_JAVA_HOME 26 | @rem Make sure we have a valid JAVA_HOME 27 | if not "%JAVA_HOME%" == "" goto have_JAVA_HOME 28 | 29 | echo. 30 | echo ERROR: Environment variable JAVA_HOME has not been set. 31 | echo. 32 | echo Please set the JAVA_HOME variable in your environment to match the 33 | echo location of your Java installation. 34 | echo. 35 | goto end 36 | 37 | :have_JAVA_HOME 38 | @rem Remove trailing slash from JAVA_HOME if found 39 | if "%JAVA_HOME:~-1%"=="\" SET JAVA_HOME=%JAVA_HOME:~0,-1% 40 | 41 | @rem Validate JAVA_HOME 42 | %COMMAND_COM% /C DIR "%JAVA_HOME%" 2>&1 | %FIND_EXE% /I /C "%JAVA_HOME%" >nul 43 | if not errorlevel 1 goto check_GRAILS_HOME 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | echo. 51 | goto end 52 | 53 | :check_GRAILS_HOME 54 | @rem Define GRAILS_HOME if not set 55 | if "%GRAILS_HOME%" == "" set GRAILS_HOME=%DIRNAME%.. 56 | 57 | @rem Remove trailing slash from GRAILS_HOME if found 58 | if "%GRAILS_HOME:~-1%"=="\" SET GRAILS_HOME=%GRAILS_HOME:~0,-1% 59 | 60 | :init 61 | 62 | for %%x in ("%HOMEPATH%") do set SHORTHOME=%%~fsx 63 | if "x%GRAILS_AGENT_CACHE_DIR%" == "x" set GRAILS_AGENT_CACHE_DIR=%SHORTHOME%/.grails/2.2.1/ 64 | set SPRINGLOADED_PARAMS="profile=grails;cacheDir=%GRAILS_AGENT_CACHE_DIR%" 65 | if not exist "%GRAILS_AGENT_CACHE_DIR%" mkdir "%GRAILS_AGENT_CACHE_DIR%" 66 | 67 | set AGENT_STRING=-javaagent:wrapper/springloaded-core-1.1.1.jar -noverify -Dspringloaded=\"%SPRINGLOADED_PARAMS%\" 68 | set DISABLE_RELOADING= 69 | if "%GRAILS_OPTS%" == "" set GRAILS_OPTS=-server -Xmx768M -Xms64M -XX:PermSize=32m -XX:MaxPermSize=256m -Dfile.encoding=UTF-8 70 | 71 | @rem Get command-line arguments, handling Windows variants 72 | if "%@eval[2+2]" == "4" goto 4NT_args 73 | 74 | @rem Slurp the command line arguments. 75 | set CMD_LINE_ARGS= 76 | set CP= 77 | set INTERACTIVE=true 78 | 79 | :win9xME_args_slurp 80 | if "x%~1" == "x" goto execute 81 | set CURR_ARG=%~1 82 | if "%CURR_ARG:~0,2%" == "-D" ( 83 | set GRAILS_OPTS=%GRAILS_OPTS% %~1=%~2 84 | shift 85 | shift 86 | goto win9xME_args_slurp 87 | ) 88 | if "x%~1" == "x-cp" ( 89 | set CP=%~2 90 | shift 91 | shift 92 | goto win9xME_args_slurp 93 | ) 94 | if "x%~1" == "x-debug" ( 95 | set JAVA_OPTS=%JAVA_OPTS% -Xdebug -Xnoagent -Dgrails.full.stacktrace=true -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005 96 | shift 97 | goto win9xME_args_slurp 98 | ) 99 | if "x%~1" == "x-classpath" ( 100 | set CP=%~2 101 | shift 102 | shift 103 | goto win9xME_args_slurp 104 | ) 105 | if "x%~1" == "x-reloading" ( 106 | set AGENT=%AGENT_STRING% 107 | shift 108 | goto win9xME_args_slurp 109 | ) 110 | if "x%~1" == "xrun-app" ( 111 | set AGENT=%AGENT_STRING% 112 | set INTERACTIVE= 113 | set CMD_LINE_ARGS=%CMD_LINE_ARGS% %1 114 | shift 115 | goto win9xME_args_slurp 116 | ) 117 | if "x%~1" == "x-noreloading" ( 118 | set DISABLE_RELOADING=true 119 | shift 120 | goto win9xME_args_slurp 121 | ) 122 | set INTERACTIVE= 123 | set CMD_LINE_ARGS=%CMD_LINE_ARGS% %1 124 | shift 125 | goto win9xME_args_slurp 126 | 127 | :4NT_args 128 | @rem Get arguments from the 4NT Shell from JP Software 129 | set CMD_LINE_ARGS=%$ 130 | 131 | :execute 132 | @rem Setup the command line 133 | set STARTER_CLASSPATH=wrapper/grails-wrapper-runtime-2.2.1.jar;wrapper;. 134 | 135 | if exist "%USERPROFILE%/.groovy/init.bat" call "%USERPROFILE%/.groovy/init.bat" 136 | 137 | @rem Setting a classpath using the -cp or -classpath option means not to use 138 | @rem the global classpath. Groovy behaves then the same as the java interpreter 139 | 140 | if "x" == "x%CLASSPATH%" goto after_classpath 141 | set CP=%CP%;%CLASSPATH% 142 | :after_classpath 143 | 144 | if "x%DISABLE_RELOADING%" == "xtrue" ( 145 | set AGENT= 146 | ) else ( 147 | if "x%INTERACTIVE%" == "xtrue" ( 148 | set AGENT=%AGENT_STRING% 149 | ) 150 | ) 151 | 152 | set STARTER_MAIN_CLASS=org.grails.wrapper.GrailsWrapper 153 | set STARTER_CONF=%GRAILS_HOME%\conf\groovy-starter.conf 154 | 155 | set JAVA_EXE=%JAVA_HOME%\bin\java.exe 156 | set TOOLS_JAR=%JAVA_HOME%\lib\tools.jar 157 | 158 | set JAVA_OPTS=%GRAILS_OPTS% %JAVA_OPTS% %AGENT% 159 | 160 | set JAVA_OPTS=%JAVA_OPTS% -Dprogram.name="%PROGNAME%" 161 | set JAVA_OPTS=%JAVA_OPTS% -Dgrails.home="%GRAILS_HOME%" 162 | set JAVA_OPTS=%JAVA_OPTS% -Dgrails.version=2.2.1 163 | set JAVA_OPTS=%JAVA_OPTS% -Dbase.dir=. 164 | set JAVA_OPTS=%JAVA_OPTS% -Dtools.jar="%TOOLS_JAR%" 165 | set JAVA_OPTS=%JAVA_OPTS% -Dgroovy.starter.conf="%STARTER_CONF%" 166 | 167 | if exist "%USERPROFILE%/.groovy/postinit.bat" call "%USERPROFILE%/.groovy/postinit.bat" 168 | 169 | @rem Execute Grails 170 | CALL "%JAVA_EXE%" %JAVA_OPTS% -classpath "%STARTER_CLASSPATH%" %STARTER_MAIN_CLASS% --main %CLASS% --conf "%STARTER_CONF%" --classpath "%CP%" "%CMD_LINE_ARGS%" 171 | :end 172 | @rem End local scope for the variables with windows NT shell 173 | if "%OS%"=="Windows_NT" endlocal 174 | 175 | @rem Optional pause the batch file 176 | if "%GROOVY_BATCH_PAUSE%" == "on" pause 177 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/EddaResourceService.java: -------------------------------------------------------------------------------- 1 | package com.netflix.ice.basic; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.net.MalformedURLException; 6 | import java.net.URL; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Properties; 10 | 11 | import org.apache.commons.io.IOUtils; 12 | import org.apache.commons.lang.StringUtils; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import com.amazonaws.util.json.JSONArray; 17 | import com.amazonaws.util.json.JSONException; 18 | import com.amazonaws.util.json.JSONObject; 19 | import com.google.common.collect.Lists; 20 | import com.netflix.ice.common.ResourceService; 21 | import com.netflix.ice.tag.Account; 22 | import com.netflix.ice.tag.Product; 23 | import com.netflix.ice.tag.Region; 24 | 25 | 26 | /** 27 | * A ResourceService which queries an https://github.com/Netflix/edda instance for the 'Usage' tag of instances for breaking down 28 | * costs based on this tag. 29 | * 30 | * Recognizes configuration values "ice.eddaresourceservice.url" and "ice.eddaresourceservice.tag", i.e. 31 | * 32 | # Settings for our own Resource-Service 33 | ice.eddaresourceservice.url=http://172.16.110.80:8080 34 | ice.eddaresourceservice.tag=Usage 35 | 36 | * Note: You will need to register the service in Bootstrap.groovy when ProcessorConfig and ReaderConfig are instantiated. 37 | * 38 | * TODO: There is currently no caching done, so there might be a lot of requests fired off to Edda! 39 | */ 40 | public class EddaResourceService extends ResourceService { 41 | private static final ArrayList EC2_PRODUCTS = Lists.newArrayList(Product.ec2, Product.ec2_instance, Product.ebs); 42 | //private static final ArrayList RDS_PRODUCTS = Lists.newArrayList(Product.rds); 43 | //private static final ArrayList S3_PRODUCTS = Lists.newArrayList(Product.s3); 44 | 45 | private final static Logger logger = LoggerFactory.getLogger(EddaResourceService.class); 46 | 47 | @SuppressWarnings("unchecked") 48 | private static List> productsWithResources = 49 | Lists.>newArrayList(EC2_PRODUCTS/*, RDS_PRODUCTS*//*, S3_PRODUCTS*/); 50 | 51 | // read from properties 52 | protected String EDDA_ROOT_URL; 53 | protected String EDDA_TAG_NAME; 54 | 55 | //private final Properties prop; 56 | 57 | public EddaResourceService(Properties prop) { 58 | super(); 59 | //this.prop = prop; 60 | 61 | EDDA_ROOT_URL = prop.getProperty("ice.eddaresourceservice.url", "http://localhost:18081/edda/api/v2/"); 62 | EDDA_TAG_NAME = prop.getProperty("ice.eddaresourceservice.tag", "Usage"); 63 | } 64 | 65 | /* (non-Javadoc) 66 | * @see com.netflix.ice.common.ResourceService#init() 67 | */ 68 | @Override 69 | public void init() { 70 | logger.info("Initializing..."); 71 | } 72 | 73 | 74 | @Override 75 | public String getResource(Account account, Region region, Product product, String resourceId, String[] lineItem, 76 | long millisStart) { 77 | // currently we support ec2 78 | if(Product.ec2.equals(product) || Product.ec2_instance.equals(product)) { 79 | if(StringUtils.isEmpty(resourceId)) { 80 | logger.warn("Had empty resourceId"); 81 | return "Error"; 82 | } 83 | 84 | try { 85 | JSONArray instances = readInstanceArray(); 86 | boolean found = false; 87 | for(int i = 0;i < instances.length();i++) { 88 | String instance = instances.getString(i); 89 | if(resourceId.equals(instance)) { 90 | found = true; 91 | break; 92 | } 93 | } 94 | if(!found) { 95 | logger.warn("Did not find resourceId in edda: " + resourceId); 96 | return "Unknown"; 97 | } 98 | 99 | InputStream stream = new URL(EDDA_ROOT_URL + "view/instances/" + resourceId).openStream(); 100 | final String json; 101 | try { 102 | json = IOUtils.toString(stream); 103 | } finally { 104 | stream.close(); 105 | } 106 | 107 | JSONObject object = new JSONObject(json); 108 | JSONArray tags = object.getJSONArray("tags"); 109 | for(int i = 0;i < tags.length();i++) { 110 | JSONObject tag = tags.getJSONObject(i); 111 | String key = tag.getString("key"); 112 | if(key.equals(EDDA_TAG_NAME)) { 113 | String usage = tag.getString("value"); 114 | logger.debug("Found usage: " + usage + " for resource " + resourceId); 115 | return usage; 116 | } 117 | } 118 | 119 | logger.debug("Did not find tag 'Usage' for resource " + resourceId); 120 | return "Unknown"; 121 | } catch (JSONException e) { 122 | logger.warn("error parsing json", e); 123 | return "Error"; 124 | } catch (MalformedURLException e) { 125 | logger.warn("error parsing url", e); 126 | return "Error"; 127 | } catch (IOException e) { 128 | logger.warn("error fetching data from edda at " + EDDA_ROOT_URL, e); 129 | return "Error"; 130 | } 131 | } 132 | 133 | logger.debug("Product: " + product + " not handled, resourceId: " + resourceId); 134 | //logger.info("get resource for account " + account + " region " + region + " product " + product + " resource: " + resourceId + " lineItem: " + Arrays.toString(lineItem)); 135 | return super.getResource(account, region, product, resourceId, lineItem, millisStart); 136 | } 137 | 138 | 139 | /* (non-Javadoc) 140 | * @see com.netflix.ice.common.ResourceService#getProductsWithResources() 141 | */ 142 | @Override 143 | public List> getProductsWithResources() { 144 | logger.info("Register for products: " + productsWithResources + "..."); 145 | return productsWithResources; 146 | } 147 | 148 | /* (non-Javadoc) 149 | * @see com.netflix.ice.common.ResourceService#commit() 150 | */ 151 | @Override 152 | public void commit() { 153 | logger.info("Commit..."); 154 | } 155 | 156 | protected JSONArray readInstanceArray() throws IOException, MalformedURLException, JSONException { 157 | InputStream stream = new URL(EDDA_ROOT_URL + "view/instances").openStream(); 158 | final String json; 159 | try { 160 | json = IOUtils.toString(stream); 161 | } finally { 162 | stream.close(); 163 | } 164 | JSONArray instances = new JSONArray(json); 165 | return instances; 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /grails-app/views/dashboard/breakdown.gsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | 3 | Copyright 2013 Netflix, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | --%> 18 | 19 | <%@ page contentType="text/html;charset=UTF-8" %> 20 | 21 | 22 | 23 | Aws Usage Breakdown 24 | 25 | 26 |
27 |
{{message}}
28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 50 | 54 | 58 | 62 | 66 | 70 | 74 | 75 | 76 |
ShowAccountRegionProductOperationUsageType
41 |
End:
42 |
# of Spans:
43 |
Aggregate:
49 | 		51 |
52 | 53 | 		55 | 56 |
57 | 		59 | 60 |
61 | 		63 | 64 |
65 | 		67 | 68 |
69 | 		71 | 72 |
73 |
77 | 78 |
79 | 80 | Submit 83 |
84 | 85 |
86 |
87 | 88 |
89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 106 | 109 | 113 | 116 | 117 | 118 |
EditDelete{{groupBy.name}}{{dayFormat(period)}} {{getConsolidateName(data_consolidate)}}
{{header.name}}
104 | Edit 105 | 107 | Delete 108 | 110 | {{row.name}} 111 | {{row.name}} 112 | 114 | {{currencySign}} {{row[header.index] | number:2}} 115 |
119 |
120 | 121 |
122 | 123 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicManagers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.amazonaws.services.s3.AmazonS3Client; 21 | import com.amazonaws.services.s3.model.S3ObjectSummary; 22 | import com.google.common.collect.Maps; 23 | import com.google.common.collect.Sets; 24 | import com.netflix.ice.common.*; 25 | import com.netflix.ice.processor.TagGroupWriter; 26 | import com.netflix.ice.reader.*; 27 | import com.netflix.ice.tag.Product; 28 | 29 | import java.util.Collection; 30 | import java.util.Map; 31 | import java.util.Set; 32 | import java.util.TreeMap; 33 | 34 | /** 35 | * This class manages all BasicTagGroupManager and BasicDataManager instances. 36 | */ 37 | public class BasicManagers extends Poller implements Managers { 38 | private ReaderConfig config; 39 | 40 | private Set products = Sets.newHashSet(); 41 | private Map tagGroupManagers = Maps.newHashMap(); 42 | private TreeMap costManagers = Maps.newTreeMap(); 43 | private TreeMap usageManagers = Maps.newTreeMap(); 44 | 45 | public void shutdown() { 46 | for (BasicTagGroupManager tagGroupManager: tagGroupManagers.values()) { 47 | tagGroupManager.shutdown(); 48 | } 49 | for (BasicDataManager dataManager: costManagers.values()) { 50 | dataManager.shutdown(); 51 | } 52 | for (BasicDataManager dataManager: usageManagers.values()) { 53 | dataManager.shutdown(); 54 | } 55 | } 56 | 57 | public void init() { 58 | config = ReaderConfig.getInstance(); 59 | doWork(); 60 | start(300); 61 | } 62 | 63 | public Collection getProducts() { 64 | return products; 65 | } 66 | 67 | public TagGroupManager getTagGroupManager(Product product) { 68 | return tagGroupManagers.get(product); 69 | } 70 | 71 | public DataManager getCostManager(Product product, ConsolidateType consolidateType) { 72 | return costManagers.get(new Key(product, consolidateType)); 73 | } 74 | 75 | public DataManager getUsageManager(Product product, ConsolidateType consolidateType) { 76 | return usageManagers.get(new Key(product, consolidateType)); 77 | } 78 | 79 | @Override 80 | protected void poll() throws Exception { 81 | doWork(); 82 | } 83 | 84 | private void doWork() { 85 | 86 | logger.info("trying to find new tag group and data managers..."); 87 | Set products = Sets.newHashSet(this.products); 88 | Map tagGroupManagers = Maps.newHashMap(this.tagGroupManagers); 89 | TreeMap costManagers = Maps.newTreeMap(this.costManagers); 90 | TreeMap usageManagers = Maps.newTreeMap(this.usageManagers); 91 | 92 | Set newProducts = Sets.newHashSet(); 93 | AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); 94 | for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).getObjectSummaries()) { 95 | String key = s3ObjectSummary.getKey(); 96 | Product product; 97 | if (key.endsWith("_all")) { 98 | product = null; 99 | } 100 | else { 101 | String name = key.substring((config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).length()); 102 | product = config.productService.getProductByName(name); 103 | } 104 | if (!products.contains(product)) { 105 | products.add(product); 106 | newProducts.add(product); 107 | } 108 | } 109 | 110 | for (Product product: newProducts) { 111 | tagGroupManagers.put(product, new BasicTagGroupManager(product)); 112 | for (ConsolidateType consolidateType: ConsolidateType.values()) { 113 | Key key = new Key(product, consolidateType); 114 | costManagers.put(key, new BasicDataManager(product, consolidateType, true)); 115 | usageManagers.put(key, new BasicDataManager(product, consolidateType, false)); 116 | } 117 | } 118 | 119 | if (newProducts.size() > 0) { 120 | this.costManagers = costManagers; 121 | this.usageManagers = usageManagers; 122 | this.tagGroupManagers = tagGroupManagers; 123 | this.products = products; 124 | } 125 | } 126 | 127 | private static class Key implements Comparable { 128 | Product product; 129 | ConsolidateType consolidateType; 130 | Key(Product product, ConsolidateType consolidateType) { 131 | this.product = product; 132 | this.consolidateType = consolidateType; 133 | } 134 | 135 | public int compareTo(Key t) { 136 | int result = this.product == t.product ? 0 : (this.product == null ? 1 : (t.product == null ? -1 : t.product.compareTo(this.product))); 137 | if (result != 0) 138 | return result; 139 | return consolidateType.compareTo(t.consolidateType); 140 | } 141 | 142 | @Override 143 | public boolean equals(Object o) { 144 | if (o == null) 145 | return false; 146 | Key other = (Key)o; 147 | return 148 | this.product == other.product && 149 | this.consolidateType == other.consolidateType; 150 | } 151 | 152 | @Override 153 | public int hashCode() { 154 | final int prime = 31; 155 | int result = 1; 156 | if (product != null) 157 | result = prime * result + this.product.hashCode(); 158 | result = prime * result + this.consolidateType.hashCode(); 159 | 160 | return result; 161 | } 162 | } 163 | 164 | 165 | } 166 | -------------------------------------------------------------------------------- /conf/system/server.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 35 | 36 | 39 | 44 | 45 | 46 | 51 | 52 | 53 | 54 | 58 | 59 | 60 | 67 | 70 | 71 | 77 | 81 | 86 | 87 | 88 | 89 | 90 | 91 | 96 | 97 | 100 | 101 | 102 | 105 | 108 | 109 | 112 | 115 | 116 | 120 | 122 | 123 | 126 | 129 | 130 | 132 | 135 | 136 | 138 | 140 | 141 | 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ReaderConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.amazonaws.auth.AWSCredentialsProvider; 21 | import com.netflix.ice.basic.BasicWeeklyCostEmailService; 22 | import com.netflix.ice.common.*; 23 | import com.netflix.ice.tag.Product; 24 | import com.netflix.ice.tag.TagType; 25 | import org.joda.time.DateTime; 26 | import org.joda.time.DateTimeZone; 27 | import org.joda.time.Interval; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.util.Collection; 32 | import java.util.Properties; 33 | 34 | /** 35 | * COnfiguration class for reader/UI. 36 | */ 37 | public class ReaderConfig extends Config { 38 | private static ReaderConfig instance; 39 | private static final Logger logger = LoggerFactory.getLogger(ReaderConfig.class); 40 | 41 | public final String companyName; 42 | public final String currencySign; 43 | public final double currencyRate; 44 | public final ApplicationGroupService applicationGroupService; 45 | public final ThroughputMetricService throughputMetricService; 46 | public final BasicWeeklyCostEmailService costEmailService; 47 | public final Managers managers; 48 | public final int monthlyCacheSize; 49 | 50 | /** 51 | * 52 | * @param properties (required) 53 | * @param managers (required) 54 | * @param accountService (required) 55 | * @param productService (required) 56 | * @param resourceService (optional) 57 | * @param applicationGroupService (optional) 58 | * @param throughputMetricService (optional) 59 | */ 60 | public ReaderConfig( 61 | Properties properties, 62 | AWSCredentialsProvider credentialsProvider, 63 | Managers managers, 64 | AccountService accountService, 65 | ProductService productService, 66 | ResourceService resourceService, 67 | ApplicationGroupService applicationGroupService, 68 | ThroughputMetricService throughputMetricService, 69 | BasicWeeklyCostEmailService costEmailService) { 70 | super(properties, credentialsProvider, accountService, productService, resourceService); 71 | 72 | companyName = properties.getProperty(IceOptions.COMPANY_NAME, ""); 73 | currencySign = properties.getProperty(IceOptions.CURRENCY_SIGN, "$"); 74 | currencyRate = Double.parseDouble(properties.getProperty(IceOptions.CURRENCY_RATE, "1")); 75 | 76 | this.managers = managers; 77 | this.applicationGroupService = applicationGroupService; 78 | this.throughputMetricService = throughputMetricService; 79 | this.costEmailService = costEmailService; 80 | this.monthlyCacheSize = Integer.parseInt(properties.getProperty(IceOptions.MONTHLY_CACHE_SIZE, "12")); 81 | 82 | ReaderConfig.instance = this; 83 | 84 | // AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); 85 | // logger.info("Deleting all files..."); 86 | // List objectSummariesToDelete = AwsUtils.listAllObjects(instance.workS3BucketName, instance.workS3BucketPrefix); 87 | // for (S3ObjectSummary objectSummary : objectSummariesToDelete) { 88 | // 89 | // String fileKey = objectSummary.getKey(); 90 | // 91 | // String name = fileKey.substring(fileKey.lastIndexOf("/") + 1); 92 | // if (name.startsWith("cost_") || name.startsWith("usage_") || name.startsWith("tagdb_")) { 93 | // s3Client.deleteObject(instance.workS3BucketName, fileKey); 94 | // continue; 95 | // } 96 | // } 97 | 98 | if (throughputMetricService != null) 99 | throughputMetricService.init(); 100 | managers.init(); 101 | applicationGroupService.init(); 102 | } 103 | 104 | /** 105 | * 106 | * @return singlton instance 107 | */ 108 | public static ReaderConfig getInstance() { 109 | return instance; 110 | } 111 | 112 | public void start() { 113 | 114 | Managers managers = ReaderConfig.getInstance().managers; 115 | Collection products = managers.getProducts(); 116 | for (Product product: products) { 117 | TagGroupManager tagGroupManager = managers.getTagGroupManager(product); 118 | Interval interval = tagGroupManager.getOverlapInterval(new Interval(new DateTime(DateTimeZone.UTC).minusMonths(monthlyCacheSize), new DateTime(DateTimeZone.UTC))); 119 | if (interval == null) 120 | continue; 121 | for (ConsolidateType consolidateType: ConsolidateType.values()) { 122 | readData(product, managers.getCostManager(product, consolidateType), interval, consolidateType); 123 | readData(product, managers.getUsageManager(product, consolidateType), interval, consolidateType); 124 | } 125 | } 126 | 127 | if (costEmailService != null) 128 | costEmailService.start(); 129 | } 130 | 131 | public void shutdown() { 132 | logger.info("Shutting down..."); 133 | 134 | instance.managers.shutdown(); 135 | if (instance.costEmailService != null) 136 | instance.costEmailService.shutdown(); 137 | } 138 | 139 | private void readData(Product product, DataManager dataManager, Interval interval, ConsolidateType consolidateType) { 140 | if (consolidateType == ConsolidateType.hourly) { 141 | DateTime start = interval.getStart().withDayOfMonth(1).withMillisOfDay(0); 142 | do { 143 | int hours = dataManager.getDataLength(start); 144 | logger.info("found " + hours + " hours data for " + product + " " + interval); 145 | start = start.plusMonths(1); 146 | } 147 | while (start.isBefore(interval.getEnd())); 148 | } 149 | else if (consolidateType == ConsolidateType.daily) { 150 | DateTime start = interval.getStart().withDayOfYear(1).withMillisOfDay(0); 151 | do { 152 | dataManager.getDataLength(start); 153 | start = start.plusYears(1); 154 | } 155 | while (start.isBefore(interval.getEnd())); 156 | } 157 | else { 158 | dataManager.getData(interval, new TagLists(), TagType.Account, AggregateType.both, false); 159 | } 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicThroughputMetricService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.google.common.cache.*; 21 | import com.google.common.collect.Lists; 22 | import com.google.common.collect.Maps; 23 | import com.netflix.ice.common.AwsUtils; 24 | import com.netflix.ice.common.ConsolidateType; 25 | import com.netflix.ice.common.Poller; 26 | import com.netflix.ice.reader.ReaderConfig; 27 | import com.netflix.ice.reader.ThroughputMetricService; 28 | import org.apache.commons.io.IOUtils; 29 | import org.joda.time.DateTime; 30 | import org.joda.time.Hours; 31 | import org.joda.time.Interval; 32 | import org.joda.time.PeriodType; 33 | 34 | import java.io.File; 35 | import java.io.FileInputStream; 36 | import java.util.List; 37 | import java.util.Map; 38 | 39 | public class BasicThroughputMetricService extends Poller implements ThroughputMetricService { 40 | 41 | private String metricName; 42 | private String metricUnitName; 43 | private String factoredCostCurrencySign; 44 | private double factoredCostMultiply; 45 | private String filePrefix; 46 | private ReaderConfig config; 47 | 48 | private Map fileCache; 49 | private LoadingCache data; 50 | 51 | public BasicThroughputMetricService(String metricName, String metricUnitName, String factoredCostCurrencySign, double factoredCostMultiply, String filePrefix) { 52 | this.metricName = metricName; 53 | this.metricUnitName = metricUnitName; 54 | this.factoredCostCurrencySign = factoredCostCurrencySign; 55 | this.factoredCostMultiply = factoredCostMultiply; 56 | this.filePrefix = filePrefix; 57 | fileCache = Maps.newConcurrentMap(); 58 | } 59 | 60 | public void init() { 61 | config = ReaderConfig.getInstance(); 62 | data = CacheBuilder.newBuilder() 63 | .maximumSize(config.monthlyCacheSize) 64 | .removalListener(new RemovalListener() { 65 | public void onRemoval(RemovalNotification objectRemovalNotification) { 66 | fileCache.remove(objectRemovalNotification.getKey()); 67 | } 68 | }) 69 | .build( 70 | new CacheLoader() { 71 | public double[] load(DateTime monthDate) throws Exception { 72 | return loadData(monthDate); 73 | } 74 | }); 75 | this.start(); 76 | } 77 | 78 | @Override 79 | protected void poll() throws Exception { 80 | for (DateTime key: fileCache.keySet()) { 81 | File file = fileCache.get(key); 82 | boolean downloaded = AwsUtils.downloadFileIfChanged(config.workS3BucketName, config.workS3BucketPrefix, file, 0); 83 | if (downloaded) { 84 | logger.info("trying to re-read data for " + file); 85 | FileInputStream in = new FileInputStream(file); 86 | try { 87 | String[] strs = IOUtils.toString(in).split(","); 88 | double[] values = new double[strs.length]; 89 | for (int i = 0; i < strs.length; i++) 90 | values[i] = Double.parseDouble(strs[i]); 91 | 92 | data.put(key, values); 93 | logger.info("done re-read data for " + file); 94 | } 95 | finally { 96 | in.close(); 97 | } 98 | } 99 | } 100 | } 101 | 102 | private double[] loadData(DateTime monthDate) throws InterruptedException { 103 | while (true) { 104 | try { 105 | File file = new File(config.localDir, filePrefix + AwsUtils.monthDateFormat.print(monthDate)); 106 | AwsUtils.downloadFileIfChanged(config.workS3BucketName, config.workS3BucketPrefix, file, 0); 107 | 108 | FileInputStream in = new FileInputStream(file); 109 | try { 110 | String[] strs = IOUtils.toString(in).split(","); 111 | double[] values = new double[strs.length]; 112 | for (int i = 0; i < strs.length; i++) 113 | values[i] = Double.parseDouble(strs[i]); 114 | 115 | return values; 116 | } 117 | finally { 118 | in.close(); 119 | } 120 | } 121 | catch (Exception e) { 122 | logger.error("error in loading data for " + monthDate, e); 123 | Thread.sleep(1000*20L); 124 | } 125 | } 126 | } 127 | 128 | public String getMetricName() { 129 | return metricName; 130 | } 131 | 132 | public String getMetricUnitName() { 133 | return metricUnitName; 134 | } 135 | 136 | public String getFactoredCostCurrencySign() { 137 | return factoredCostCurrencySign; 138 | } 139 | 140 | public double getFactoredCostMultiply() { 141 | return factoredCostMultiply; 142 | } 143 | 144 | public double[] getData(Interval interval, ConsolidateType consolidateType) throws Exception { 145 | DateTime start = interval.getStart().withDayOfMonth(1).withMillisOfDay(0); 146 | DateTime end = interval.getEnd(); 147 | 148 | int num = interval.toPeriod(PeriodType.hours()).getHours(); 149 | if (interval.getStart().plusHours(num).isBefore(interval.getEnd())) 150 | num++; 151 | 152 | double[] hourly = new double[num]; 153 | List monthly = Lists.newArrayList(); 154 | do { 155 | double total = 0; 156 | int resultIndex = interval.getStart().isBefore(start) ? Hours.hoursBetween(interval.getStart(), start).getHours() : 0; 157 | int fromIndex = interval.getStart().isBefore(start) ? 0 : Hours.hoursBetween(start, interval.getStart()).getHours(); 158 | 159 | double[] data = this.data.get(start); 160 | while (resultIndex < num && fromIndex < data.length) { 161 | total += data[fromIndex]; 162 | hourly[resultIndex++] = data[fromIndex++]; 163 | } 164 | 165 | start = start.plusMonths(1); 166 | monthly.add(total); 167 | } 168 | while (start.isBefore(end)); 169 | 170 | int hoursInPeriod = (int) (consolidateType.millis / AwsUtils.hourMillis); 171 | num = consolidateType == ConsolidateType.monthly ? monthly.size() : (int) Math.ceil(1.0 * num / hoursInPeriod); 172 | double[] result = new double[num]; 173 | 174 | if (consolidateType == ConsolidateType.monthly) { 175 | for (int i = 0; i < num; i++) 176 | result[i] = monthly.get(i); 177 | } 178 | else { 179 | for (int i = 0; i < hourly.length; i++) 180 | result[i/hoursInPeriod] += hourly[i]; 181 | } 182 | return result; 183 | } 184 | } 185 | --------------------------------------------------------------------------------