├── grails-app ├── i18n │ └── messages.properties ├── views │ ├── error.gsp │ ├── dashboard │ │ ├── editappgroup.gsp │ │ ├── summary.gsp │ │ └── breakdown.gsp │ └── layouts │ │ └── main.gsp └── conf │ ├── UrlMappings.groovy │ ├── TrackingFilters.groovy │ ├── DataSource.groovy │ ├── Config.groovy │ └── BuildConfig.groovy ├── OSSMETADATA ├── screenshots ├── ss_detail.png ├── ss_summary.png ├── ss_breakdown_appgroup.png ├── ss_reservation_bytype.png └── ss_reservation_byreservation.png ├── web-app ├── images │ ├── spinner.gif │ ├── downarrowdark.gif │ ├── skin │ │ └── information.png │ └── tango │ │ ├── 16 │ │ ├── places │ │ │ └── user-trash.png │ │ ├── tools │ │ │ └── draw-freehand.png │ │ ├── actions │ │ │ └── document-save.png │ │ └── apps │ │ │ └── utilities-system-monitor.png │ │ ├── 24 │ │ ├── actions │ │ │ ├── edit-undo.png │ │ │ └── document-save.png │ │ └── places │ │ │ └── user-trash.png │ │ └── tango-icon-theme │ │ └── 32x32 │ │ └── actions │ │ ├── go-next.png │ │ └── go-previous.png ├── css │ ├── ui-lightness │ │ └── images │ │ │ ├── animated-overlay.gif │ │ │ ├── ui-icons_222222_256x240.png │ │ │ ├── ui-icons_228ef1_256x240.png │ │ │ ├── ui-icons_ef8c08_256x240.png │ │ │ ├── ui-icons_ffd27a_256x240.png │ │ │ ├── ui-icons_ffffff_256x240.png │ │ │ ├── ui-bg_flat_10_000000_40x100.png │ │ │ ├── ui-bg_glass_65_ffffff_1x400.png │ │ │ ├── ui-bg_glass_100_f6f6f6_1x400.png │ │ │ ├── ui-bg_glass_100_fdf5ce_1x400.png │ │ │ ├── ui-bg_gloss-wave_35_f6a828_500x100.png │ │ │ ├── ui-bg_diagonals-thick_18_b81900_40x40.png │ │ │ ├── ui-bg_diagonals-thick_20_666666_40x40.png │ │ │ ├── ui-bg_highlight-soft_100_eeeeee_1x100.png │ │ │ └── ui-bg_highlight-soft_75_ffe45c_1x100.png │ └── errors.css └── WEB-INF │ ├── sitemesh.xml │ ├── grails.xml │ └── applicationContext.xml ├── wrapper ├── grails-wrapper.properties ├── springloaded-1.2.1.RELEASE.jar └── grails-wrapper-runtime-2.4.4.jar ├── .travis.yml ├── src ├── java │ ├── log4j.properties │ ├── com │ │ └── netflix │ │ │ └── ice │ │ │ ├── reader │ │ │ ├── AggregateType.java │ │ │ ├── ThroughputMetricService.java │ │ │ ├── DataManager.java │ │ │ ├── ApplicationGroupService.java │ │ │ ├── Managers.java │ │ │ ├── ReadOnlyData.java │ │ │ ├── ApplicationGroup.java │ │ │ ├── TagGroupManager.java │ │ │ └── ReaderConfig.java │ │ │ ├── tag │ │ │ ├── ApplicationGroup.java │ │ │ ├── TagType.java │ │ │ ├── Account.java │ │ │ ├── ResourceGroup.java │ │ │ ├── InstanceOs.java │ │ │ ├── Tag.java │ │ │ ├── Product.java │ │ │ ├── UsageType.java │ │ │ └── Region.java │ │ │ ├── common │ │ │ ├── ConsolidateType.java │ │ │ ├── Randomizer.java │ │ │ ├── ProductService.java │ │ │ ├── AccountService.java │ │ │ ├── Config.java │ │ │ ├── ResourceService.java │ │ │ ├── Poller.java │ │ │ └── IceOptions.java │ │ │ ├── basic │ │ │ ├── BasicResourceService.java │ │ │ ├── BasicAccountService.java │ │ │ ├── SampleMapDbResourceService.java │ │ │ ├── MapDb.java │ │ │ ├── BasicS3ApplicationGroupService.java │ │ │ ├── BasicProductService.java │ │ │ ├── EddaResourceService.java │ │ │ └── BasicManagers.java │ │ │ └── processor │ │ │ ├── LineItemProcessor.java │ │ │ ├── DataWriter.java │ │ │ ├── TagGroupWriter.java │ │ │ ├── ReservationService.java │ │ │ ├── ReadWriteData.java │ │ │ └── ProcessorConfig.java │ └── sample.properties ├── groovy │ └── com │ │ └── netflix │ │ └── ice │ │ └── JSONConverter.groovy └── test │ └── com │ └── netflix │ └── ice │ └── basic │ └── EddaResourceServiceTest.java ├── .gitignore ├── application.properties ├── conf └── system │ ├── setenv.sh │ └── server.xml ├── install.sh └── grailsw.bat /grails-app/i18n/messages.properties: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=maintenance 2 | -------------------------------------------------------------------------------- /screenshots/ss_detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/screenshots/ss_detail.png -------------------------------------------------------------------------------- /screenshots/ss_summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/screenshots/ss_summary.png -------------------------------------------------------------------------------- /web-app/images/spinner.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/spinner.gif -------------------------------------------------------------------------------- /web-app/images/downarrowdark.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/downarrowdark.gif -------------------------------------------------------------------------------- /web-app/images/skin/information.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/skin/information.png -------------------------------------------------------------------------------- /screenshots/ss_breakdown_appgroup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/screenshots/ss_breakdown_appgroup.png -------------------------------------------------------------------------------- /screenshots/ss_reservation_bytype.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/screenshots/ss_reservation_bytype.png -------------------------------------------------------------------------------- /wrapper/grails-wrapper.properties: -------------------------------------------------------------------------------- 1 | wrapper.dist.url=http://dist.springframework.org.s3.amazonaws.com/release/GRAILS/ 2 | -------------------------------------------------------------------------------- /wrapper/springloaded-1.2.1.RELEASE.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/wrapper/springloaded-1.2.1.RELEASE.jar -------------------------------------------------------------------------------- /wrapper/grails-wrapper-runtime-2.4.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/wrapper/grails-wrapper-runtime-2.4.4.jar -------------------------------------------------------------------------------- /screenshots/ss_reservation_byreservation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/screenshots/ss_reservation_byreservation.png -------------------------------------------------------------------------------- /web-app/images/tango/16/places/user-trash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/16/places/user-trash.png -------------------------------------------------------------------------------- /web-app/images/tango/24/actions/edit-undo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/24/actions/edit-undo.png -------------------------------------------------------------------------------- /web-app/images/tango/24/places/user-trash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/24/places/user-trash.png -------------------------------------------------------------------------------- /web-app/images/tango/16/tools/draw-freehand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/16/tools/draw-freehand.png -------------------------------------------------------------------------------- /web-app/images/tango/16/actions/document-save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/16/actions/document-save.png -------------------------------------------------------------------------------- /web-app/images/tango/24/actions/document-save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/24/actions/document-save.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/animated-overlay.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/animated-overlay.gif -------------------------------------------------------------------------------- /web-app/images/tango/16/apps/utilities-system-monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/16/apps/utilities-system-monitor.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_222222_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-icons_222222_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_228ef1_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-icons_228ef1_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_ef8c08_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-icons_ef8c08_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_ffd27a_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-icons_ffd27a_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-icons_ffffff_256x240.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-icons_ffffff_256x240.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_flat_10_000000_40x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_flat_10_000000_40x100.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_glass_65_ffffff_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_glass_65_ffffff_1x400.png -------------------------------------------------------------------------------- /web-app/images/tango/tango-icon-theme/32x32/actions/go-next.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/tango-icon-theme/32x32/actions/go-next.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_glass_100_f6f6f6_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_glass_100_f6f6f6_1x400.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_glass_100_fdf5ce_1x400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_glass_100_fdf5ce_1x400.png -------------------------------------------------------------------------------- /web-app/images/tango/tango-icon-theme/32x32/actions/go-previous.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/images/tango/tango-icon-theme/32x32/actions/go-previous.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_gloss-wave_35_f6a828_500x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_gloss-wave_35_f6a828_500x100.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_diagonals-thick_18_b81900_40x40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_diagonals-thick_18_b81900_40x40.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_diagonals-thick_20_666666_40x40.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_diagonals-thick_20_666666_40x40.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_highlight-soft_100_eeeeee_1x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_highlight-soft_100_eeeeee_1x100.png -------------------------------------------------------------------------------- /web-app/css/ui-lightness/images/ui-bg_highlight-soft_75_ffe45c_1x100.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Teevity/ice/HEAD/web-app/css/ui-lightness/images/ui-bg_highlight-soft_75_ffe45c_1x100.png -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | language: groovy 4 | 5 | jdk: 6 | - oraclejdk7 7 | 8 | install: 9 | - export ICE_HOME=${TRAVIS_BUILD_DIR} 10 | - ./grailsw clean 11 | - yes | ./grailsw refresh-dependencies 12 | 13 | script: 14 | - bash -n install.sh 15 | - ./grailsw compile -------------------------------------------------------------------------------- /src/java/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | # stdout 4 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 5 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} - %-5p %C{1} - [%F:%L] %m%n -------------------------------------------------------------------------------- /grails-app/views/error.gsp: -------------------------------------------------------------------------------- 1 | 2 | 3 | <g:if env="development">Ice Runtime Exception</g:if><g:else>Error</g:else> 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /grails-app/conf/UrlMappings.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | class UrlMappings { 18 | 19 | static mappings = { 20 | "/$controller/$action?/$id?" {} 21 | "/" { controller = "dashboard"} 22 | "500" (view: '/error') 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # .gitignore for Grails 1.2 and 1.3 2 | 3 | # web application files 4 | /web-app/WEB-INF 5 | 6 | # IDE support files 7 | /.classpath 8 | /.launch 9 | /.project 10 | /.settings 11 | /*.launch 12 | /*.tmproj 13 | /ivy* 14 | /eclipse 15 | 16 | # default HSQL database files for production mode 17 | /prodDb.* 18 | 19 | # general HSQL database files 20 | *Db.properties 21 | *Db.script 22 | 23 | # logs 24 | /stacktrace.log 25 | /test/reports 26 | /logs 27 | 28 | # project release file 29 | /*.war 30 | 31 | # plugin release file 32 | /*.zip 33 | 34 | # older plugin install locations 35 | /plugins 36 | /web-app/plugins 37 | /web-app/WEB-INF/classes 38 | 39 | # "temporary" build files 40 | /target 41 | /target-eclipse 42 | /work 43 | 44 | # other 45 | *.iws 46 | 47 | #project specific: 48 | /src/java/ice.properties 49 | 50 | # OSX 51 | .DS_Store 52 | 53 | # FUSE files 54 | .fuse_hidden* -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/AggregateType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | public enum AggregateType { 21 | none, 22 | stats, 23 | data, 24 | both 25 | } 26 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/ApplicationGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public class ApplicationGroup extends Tag { 21 | 22 | public ApplicationGroup(String name) { 23 | super(name); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/TagType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public enum TagType { 21 | Account, 22 | Region, 23 | Zone, 24 | Product, 25 | Operation, 26 | UsageType, 27 | ResourceGroup, 28 | ApplicationGroup; 29 | } 30 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Account.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public class Account extends Tag { 21 | public final String id; 22 | 23 | public Account(String accountId, String accountName) { 24 | super(accountName); 25 | this.id = accountId; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/ConsolidateType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | public enum ConsolidateType { 21 | hourly(3600000L), 22 | daily(24*3600000L), 23 | weekly(24*7*3600000L), 24 | monthly(-1); 25 | 26 | public final long millis; 27 | 28 | private ConsolidateType(long millis) { 29 | this.millis = millis; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ThroughputMetricService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.netflix.ice.common.ConsolidateType; 21 | import org.joda.time.Interval; 22 | 23 | /** 24 | * Interface to provide throughput metrics. 25 | */ 26 | public interface ThroughputMetricService { 27 | void init(); 28 | String getMetricName(); 29 | String getMetricUnitName(); 30 | String getFactoredCostCurrencySign(); 31 | double getFactoredCostMultiply(); 32 | double[] getData(Interval interval, ConsolidateType consolidateType) throws Exception; 33 | } 34 | -------------------------------------------------------------------------------- /application.properties: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | app.grails.version=2.4.4 18 | app.name=ice 19 | app.servlet.version=2.4 20 | app.version=1.1.0 21 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 22 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.stdout.layout.ConversionPattern=%d %5p [%c] %m%n 24 | log4j.category.com.amazonaws.request=WARN 25 | log4j.category.com.netflix.cloud=DEBUG 26 | log4j.category.com.netflix.logging=WARN 27 | log4j.category.com.netflix.monitoring=WARN 28 | log4j.category.hibernate=ERROR 29 | log4j.category.httpclient=ERROR 30 | log4j.category.org.codehaus.groovy=WARN 31 | log4j.category.sun.reflect=WARN 32 | log4j.rootLogger=INFO, stdout 33 | -------------------------------------------------------------------------------- /grails-app/conf/TrackingFilters.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import java.util.regex.Pattern 17 | import org.joda.time.DateTime 18 | 19 | class TrackingFilters { 20 | 21 | static final Pattern nonBrowserUserAgents = Pattern.compile( 22 | '.*(libcurl|Python-urllib|Wget|HttpClient|lwp-request|Java).*') 23 | 24 | static filters = { 25 | allExceptError(controller: 'error', invert: true) { 26 | before = { 27 | 28 | String userAgent = request.getHeader('user-agent') 29 | if (userAgent?.contains('MSIE') && !userAgent.contains('chromeframe')) { 30 | request['ieWithoutChromeFrame'] = true 31 | } 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /web-app/WEB-INF/sitemesh.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 20 | 21 | 23 | 25 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/Randomizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import java.util.Map; 21 | 22 | /** 23 | * Used to generate random/fake data 24 | */ 25 | public interface Randomizer { 26 | 27 | /** 28 | * Get random/fake usage for given tag group. 29 | * @param time 30 | * @param tagGroup 31 | * @param usage 32 | * @return random/fake usage 33 | */ 34 | double randomizeUsage(long time, TagGroup tagGroup, double usage); 35 | 36 | /** 37 | * Get random/fake cost for given tag group. 38 | * @param tagGroup 39 | * @return random/fake cost 40 | */ 41 | double randomizeCost(TagGroup tagGroup); 42 | 43 | /** 44 | * Get map of resource distribution. 45 | * @return map of resource distribution 46 | */ 47 | Map getDistribution(TagGroup tagGroup); 48 | } 49 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/DataManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.netflix.ice.tag.Tag; 21 | import com.netflix.ice.tag.TagType; 22 | import org.joda.time.DateTime; 23 | import org.joda.time.Interval; 24 | 25 | import java.util.Map; 26 | 27 | /** 28 | * Interface to feed data to UI. 29 | */ 30 | public interface DataManager { 31 | 32 | /** 33 | * Get map of data. 34 | * @param interval 35 | * @param tagLists 36 | * @param groupBy 37 | * @param aggregate 38 | * @param forReservation 39 | * @return 40 | */ 41 | Map getData(Interval interval, TagLists tagLists, TagType groupBy, AggregateType aggregate, boolean forReservation); 42 | 43 | /** 44 | * Get data length. 45 | * @param start 46 | * @return 47 | */ 48 | int getDataLength(DateTime start); 49 | } 50 | -------------------------------------------------------------------------------- /src/groovy/com/netflix/ice/JSONConverter.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice 19 | 20 | import grails.converters.JSON 21 | import com.netflix.ice.tag.Tag 22 | import com.netflix.ice.reader.ApplicationGroup 23 | 24 | class JSONConverter { 25 | 26 | static void register() { 27 | JSON.registerObjectMarshaller(Tag) { Tag it -> 28 | return [name: it.name] 29 | } 30 | JSON.registerObjectMarshaller(ApplicationGroup) { ApplicationGroup appgroup -> 31 | def result = [name: appgroup.name, owner: appgroup.owner, data: [:]] 32 | for (String key: appgroup.data.keySet()) { 33 | result.data[key] = [] 34 | for (String v: appgroup.data.get(key)) { 35 | result.data[key].add([name: v]) 36 | } 37 | } 38 | return result 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /grails-app/conf/DataSource.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | /*dataSource { 17 | pooled = true 18 | driverClassName = "org.hsqldb.jdbcDriver" 19 | username = "sa" 20 | password = "" 21 | }*/ 22 | hibernate { 23 | cache.use_second_level_cache = true 24 | cache.use_query_cache = true 25 | cache.provider_class = 'net.sf.ehcache.hibernate.EhCacheProvider' 26 | } 27 | 28 | // environment specific settings 29 | environments { 30 | development { 31 | dataSource { 32 | dbCreate = "create-drop" // one of 'create', 'create-drop','update' 33 | //url = "jdbc:hsqldb:mem:devDB" 34 | } 35 | } 36 | test { 37 | dataSource { 38 | dbCreate = "update" 39 | //url = "jdbc:hsqldb:mem:testDb" 40 | } 41 | } 42 | production { 43 | dataSource { 44 | dbCreate = "update" 45 | //url = "jdbc:hsqldb:file:prodDb;shutdown=true" 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/ProductService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import com.netflix.ice.tag.Product; 21 | 22 | import java.util.Collection; 23 | import java.util.List; 24 | 25 | public interface ProductService { 26 | 27 | /** 28 | * Get product by aws name, e.g. "Amazon Elastic Compute Cloud", "AWS Elastic MapReduce" 29 | * @param name 30 | * @return product 31 | */ 32 | Product getProductByAwsName(String name); 33 | 34 | /** 35 | * Get product by name, e.g. ec2, emr 36 | * @param name 37 | * @return 38 | */ 39 | Product getProductByName(String name); 40 | 41 | /** 42 | * Get list of products from given names 43 | * @param names 44 | * @return list of products 45 | */ 46 | public List getProducts(List names); 47 | 48 | /** 49 | * Get list of products 50 | * @return list of products 51 | */ 52 | public Collection getProducts(); 53 | } 54 | -------------------------------------------------------------------------------- /web-app/WEB-INF/grails.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 20 | 21 | DefaultSecurityConfig 22 | grails.plugins.springsecurity.SpringSecurityService 23 | grails.plugins.springsecurity.SecurityTagLib 24 | DefaultLdapSecurityConfig 25 | BootStrap 26 | BuildConfig 27 | Config 28 | DataSource 29 | OccasionFilters 30 | RegionFilters 31 | SettingsFilters 32 | resources 33 | TrackingFilters 34 | UrlMappings 35 | com.netflix.ice.DashboardController 36 | 37 | 38 | HibernateGrailsPlugin 39 | SpringSecurityCoreGrailsPlugin 40 | SpringSecurityLdapGrailsPlugin 41 | 42 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicResourceService.java: -------------------------------------------------------------------------------- 1 | package com.netflix.ice.basic; 2 | 3 | import com.google.common.collect.Lists; 4 | import com.netflix.ice.common.ResourceService; 5 | import com.netflix.ice.processor.ProcessorConfig; 6 | import com.netflix.ice.reader.ReaderConfig; 7 | import com.netflix.ice.tag.Account; 8 | import com.netflix.ice.tag.Product; 9 | import com.netflix.ice.tag.Region; 10 | import org.apache.commons.lang.StringUtils; 11 | 12 | import java.util.List; 13 | 14 | public class BasicResourceService extends ResourceService { 15 | 16 | private ProcessorConfig processorConfig; 17 | 18 | @Override 19 | public void init() { 20 | processorConfig = ProcessorConfig.getInstance(); 21 | } 22 | 23 | @Override 24 | public String getResource(Account account, Region region, Product product, String resourceId, String[] lineItem, long millisStart) { 25 | List header = processorConfig.lineItemProcessor.getHeader(); 26 | 27 | String result = ""; 28 | for (String tag: processorConfig.customTags) { 29 | int index = header.indexOf(tag); 30 | if (index > 0 && lineItem.length > index && !StringUtils.isEmpty(lineItem[index])) 31 | result = StringUtils.isEmpty(result) ? lineItem[index] : result + "_" + lineItem[index]; 32 | } 33 | 34 | return StringUtils.isEmpty(result) ? product.name : result; 35 | } 36 | 37 | @Override 38 | public List> getProductsWithResources() { 39 | List> result = Lists.newArrayList(); 40 | for (Product product: ReaderConfig.getInstance().productService.getProducts()) { 41 | result.add(Lists.newArrayList(product)); 42 | } 43 | return result; 44 | } 45 | 46 | @Override 47 | public void commit() { 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ApplicationGroupService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import java.util.Map; 21 | 22 | /** 23 | * Interface to manager user defined application groups. 24 | */ 25 | public interface ApplicationGroupService { 26 | 27 | void init(); 28 | 29 | /** 30 | * Get a map of application groups. Key will be the application group name. 31 | * @return map of application groups 32 | */ 33 | Map getApplicationGroups(); 34 | 35 | /** 36 | * Get application group by name. 37 | * @param name 38 | * @return application group or null if not exist 39 | */ 40 | ApplicationGroup getApplicationGroup(String name); 41 | 42 | /** 43 | * Save application group configuration. 44 | * @param applicationGroup 45 | * @return whether or not save was successfull 46 | */ 47 | boolean saveApplicationGroup(ApplicationGroup applicationGroup); 48 | 49 | /** 50 | * Delete application group. 51 | * @param name 52 | * @return whether or not delete was successfull 53 | */ 54 | boolean deleteApplicationGroup(String name); 55 | } 56 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/ResourceGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | 23 | import java.util.List; 24 | import java.util.concurrent.ConcurrentMap; 25 | 26 | public class ResourceGroup extends Tag { 27 | private ResourceGroup (String name) { 28 | super(name); 29 | } 30 | private static ConcurrentMap resourceGroups = Maps.newConcurrentMap(); 31 | 32 | public static ResourceGroup getResourceGroup(String name) { 33 | ResourceGroup resourceGroup = resourceGroups.get(name); 34 | if (resourceGroup == null) { 35 | resourceGroups.putIfAbsent(name, new ResourceGroup(name)); 36 | resourceGroup = resourceGroups.get(name); 37 | } 38 | return resourceGroup; 39 | } 40 | 41 | public static List getResourceGroups(List names) { 42 | List result = Lists.newArrayList(); 43 | if (names != null) { 44 | for (String name: names) { 45 | ResourceGroup resourceGroup = resourceGroups.get(name); 46 | if (resourceGroup != null) 47 | result.add(resourceGroup); 48 | } 49 | } 50 | return result; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/LineItemProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.netflix.ice.tag.Product; 21 | import java.util.List; 22 | import org.joda.time.DateTimeZone; 23 | import org.joda.time.format.DateTimeFormat; 24 | import org.joda.time.format.DateTimeFormatter; 25 | 26 | import java.util.Map; 27 | 28 | /** 29 | * Interface to process each line item in billing file. 30 | */ 31 | public interface LineItemProcessor { 32 | public static final DateTimeFormatter amazonBillingDateFormat = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZone(DateTimeZone.UTC); 33 | public static final DateTimeFormatter amazonBillingDateFormat2 = DateTimeFormat.forPattern("yyyy/MM/dd HH:mm:ss").withZone(DateTimeZone.UTC); 34 | 35 | void initIndexes(ProcessorConfig config, boolean withTags, String[] header); 36 | List getHeader(); 37 | int getUserTagStartIndex(); 38 | long getEndMillis(String[] items); 39 | Result process(long startMilli, boolean processAll, ProcessorConfig config, String[] items, Map usageDataByProduct, Map costDataByProduct, Map ondemandRate); 40 | 41 | public static enum Result { 42 | delay, 43 | ignore, 44 | hourly, 45 | monthly, 46 | daily 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/Managers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.netflix.ice.common.*; 21 | import com.netflix.ice.tag.Product; 22 | 23 | import java.util.Collection; 24 | 25 | /** 26 | * Interface to manager all TagGroupManager and DataManager instances for different products 27 | */ 28 | public interface Managers { 29 | 30 | void init(); 31 | 32 | /** 33 | * 34 | * @return collection of products 35 | */ 36 | Collection getProducts(); 37 | 38 | /** 39 | * 40 | * @param product 41 | * @return TagGroupManager instance for specified product 42 | */ 43 | TagGroupManager getTagGroupManager(Product product); 44 | 45 | /** 46 | * 47 | * @param product 48 | * @param consolidateType 49 | * @return cost DataManager instance for specified product and consolidateType 50 | */ 51 | DataManager getCostManager(Product product, ConsolidateType consolidateType); 52 | 53 | /** 54 | * 55 | * @param product 56 | * @param consolidateType 57 | * @return usage DataManager instance for specified product and consolidateType 58 | */ 59 | DataManager getUsageManager(Product product, ConsolidateType consolidateType); 60 | 61 | /** 62 | * shutdown all manager instances 63 | */ 64 | void shutdown(); 65 | } 66 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/InstanceOs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public enum InstanceOs { 21 | linux("", "", "Linux/UNIX"), 22 | sqlserverweb(".sqlserverweb", ":0202", "Windows with SQL Server Web"), 23 | sqlserverstd(".sqlserverstd", ":0006", "Windows with SQL Server Standard"), 24 | sles(".sles", ":000g", "SUSE Linux"), 25 | rhel(".rhel", ":0010", "Red Hat Enterprise Linux"), 26 | rhbl(".rhbl", ":00g0", "Red Hat BYOL Linux"), 27 | windows(".windows", ":0002", "Windows"), 28 | dw(".dw", ":0001", "redshift"), 29 | others(".others", ":others", "others"); 30 | 31 | public final String usageType; 32 | public final String code; 33 | public final String description; 34 | 35 | InstanceOs(String usageType, String code, String description) { 36 | this.usageType = usageType; 37 | this.code = code.toLowerCase(); 38 | this.description = description.toLowerCase(); 39 | } 40 | 41 | public static InstanceOs withCode(String code) { 42 | for (InstanceOs os: InstanceOs.values()) { 43 | if (code.toLowerCase().equals(os.code)) 44 | return os; 45 | } 46 | return others; 47 | } 48 | 49 | public static InstanceOs withDescription(String description) { 50 | for (InstanceOs os: InstanceOs.values()) { 51 | if (description.toLowerCase().startsWith(os.description)) 52 | return os; 53 | } 54 | return others; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /web-app/WEB-INF/applicationContext.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 25 | 26 | 27 | Grails application factory bean 28 | 29 | 30 | 31 | 32 | A bean that manages Grails plugins 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 46 | 47 | utf-8 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /web-app/css/errors.css: -------------------------------------------------------------------------------- 1 | h1, h2 { 2 | margin: 10px 25px 5px; 3 | } 4 | 5 | h2 { 6 | font-size: 1.1em; 7 | } 8 | 9 | .filename { 10 | font-style: italic; 11 | } 12 | 13 | .exceptionMessage { 14 | margin: 10px; 15 | border: 1px solid #000; 16 | padding: 5px; 17 | background-color: #E9E9E9; 18 | } 19 | 20 | .stack, 21 | .snippet { 22 | margin: 0 25px 10px; 23 | } 24 | 25 | .stack, 26 | .snippet { 27 | border: 1px solid #ccc; 28 | -mox-box-shadow: 0 0 2px rgba(0,0,0,0.2); 29 | -webkit-box-shadow: 0 0 2px rgba(0,0,0,0.2); 30 | box-shadow: 0 0 2px rgba(0,0,0,0.2); 31 | } 32 | 33 | /* error details */ 34 | .error-details { 35 | border-top: 1px solid #FFAAAA; 36 | -mox-box-shadow: 0 0 2px rgba(0,0,0,0.2); 37 | -webkit-box-shadow: 0 0 2px rgba(0,0,0,0.2); 38 | box-shadow: 0 0 2px rgba(0,0,0,0.2); 39 | border-bottom: 1px solid #FFAAAA; 40 | -mox-box-shadow: 0 0 2px rgba(0,0,0,0.2); 41 | -webkit-box-shadow: 0 0 2px rgba(0,0,0,0.2); 42 | box-shadow: 0 0 2px rgba(0,0,0,0.2); 43 | background-color:#FFF3F3; 44 | line-height: 1.5; 45 | overflow: hidden; 46 | padding: 5px; 47 | padding-left:25px; 48 | } 49 | 50 | .error-details dt { 51 | clear: left; 52 | float: left; 53 | font-weight: bold; 54 | margin-right: 5px; 55 | } 56 | 57 | .error-details dt:after { 58 | content: ":"; 59 | } 60 | 61 | .error-details dd { 62 | display: block; 63 | } 64 | 65 | /* stack trace */ 66 | .stack { 67 | padding: 5px; 68 | overflow: auto; 69 | height: 150px; 70 | } 71 | 72 | /* code snippet */ 73 | .snippet { 74 | background-color: #fff; 75 | font-family: monospace; 76 | } 77 | 78 | .snippet .line { 79 | display: block; 80 | } 81 | 82 | .snippet .lineNumber { 83 | background-color: #ddd; 84 | color: #999; 85 | display: inline-block; 86 | margin-right: 5px; 87 | padding: 0 3px; 88 | text-align: right; 89 | width: 3em; 90 | } 91 | 92 | .snippet .error { 93 | background-color: #fff3f3; 94 | font-weight: bold; 95 | } 96 | 97 | .snippet .error .lineNumber { 98 | background-color: #faa; 99 | color: #333; 100 | font-weight: bold; 101 | } 102 | 103 | .snippet .line:first-child .lineNumber { 104 | padding-top: 5px; 105 | } 106 | 107 | .snippet .line:last-child .lineNumber { 108 | padding-bottom: 5px; 109 | } -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Tag.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import java.io.Serializable; 21 | 22 | public abstract class Tag implements Comparable, Serializable { 23 | public static final Tag aggregated = new Tag("aggregated") { 24 | @Override 25 | public int compareTo(Tag t) { 26 | return this == t ? 0 : -1; 27 | } 28 | }; 29 | 30 | public final String name; 31 | public final String s3Name; 32 | Tag(String name) { 33 | this.name = name; 34 | this.s3Name = Tag.toS3(name); 35 | } 36 | 37 | @Override 38 | public boolean equals(Object o) { 39 | if (o instanceof Tag) 40 | return this.name.equals(((Tag)o).name); 41 | else 42 | return false; 43 | } 44 | 45 | /** 46 | * Normalize a tagname suitable to be an S3 Filename 47 | */ 48 | public static String toS3(String name) { 49 | name = name.replaceAll("/","--"); 50 | return name; 51 | } 52 | 53 | /** 54 | * Normalize a tagname from an S3 Filename 55 | */ 56 | public static String fromS3(String name) { 57 | name = name.replaceAll("--","/"); 58 | return name; 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return this.name; 64 | } 65 | 66 | @Override 67 | public int hashCode() { 68 | return this.name.hashCode(); 69 | } 70 | 71 | public int compareTo(Tag t) { 72 | if (t == aggregated) 73 | return -t.compareTo(this); 74 | int result = ("a" + this.name).compareTo("a" + t.name); 75 | return result; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ReadOnlyData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.netflix.ice.common.TagGroup; 22 | 23 | import java.io.DataInput; 24 | import java.io.IOException; 25 | import java.util.Collection; 26 | import java.util.List; 27 | 28 | public class ReadOnlyData { 29 | double[][] data; 30 | private Collection tagGroups; 31 | 32 | public ReadOnlyData(double[][] data, Collection tagGroups) { 33 | this.data = data; 34 | this.tagGroups = tagGroups; 35 | } 36 | 37 | public double[] getData(int i) { 38 | return data[i]; 39 | } 40 | 41 | public int getNum() { 42 | return data.length; 43 | } 44 | 45 | public Collection getTagGroups() { 46 | return tagGroups; 47 | } 48 | 49 | public static class Serializer { 50 | 51 | public static ReadOnlyData deserialize(DataInput in) throws IOException { 52 | 53 | int numKeys = in.readInt(); 54 | List keys = Lists.newArrayList(); 55 | for (int j = 0; j < numKeys; j++) { 56 | keys.add(TagGroup.Serializer.deserialize(ReaderConfig.getInstance(), in)); 57 | } 58 | 59 | int num = in.readInt(); 60 | double[][] data = new double[num][]; 61 | for (int i = 0; i < num; i++) { 62 | data[i] = new double[keys.size()]; 63 | boolean hasData = in.readBoolean(); 64 | if (hasData) { 65 | for (int j = 0; j < keys.size(); j++) { 66 | double v = in.readDouble(); 67 | if (v != 0) { 68 | data[i][j] = v; 69 | } 70 | } 71 | } 72 | } 73 | 74 | return new ReadOnlyData(data, keys); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Product.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | public class Product extends Tag { 21 | public static final Product cloudfront = new Product("cloudfront"); 22 | public static final Product cloudhsm = new Product("cloudhsm"); 23 | public static final Product cloudwatch = new Product("cloudwatch"); 24 | public static final Product data_pipeline = new Product("data_pipeline"); 25 | public static final Product data_transfer = new Product("data_transfer"); 26 | public static final Product direct_connect = new Product("direct_connect"); 27 | public static final Product dynamodb = new Product("dynamodb"); 28 | public static final Product ebs = new Product("ebs"); 29 | public static final Product ec2 = new Product("ec2"); 30 | public static final Product ec2_instance = new Product("ec2_instance"); 31 | public static final Product eip = new Product("eip"); 32 | public static final Product elasticache = new Product("elasticache"); 33 | public static final Product emr = new Product("emr"); 34 | public static final Product glacier = new Product("glacier"); 35 | public static final Product monitor = new Product("monitor"); 36 | public static final Product rds = new Product("rds"); 37 | public static final Product redshift = new Product("redshift"); 38 | public static final Product route53 = new Product("route53"); 39 | public static final Product s3 = new Product("s3"); 40 | public static final Product simpledb = new Product("simpledb"); 41 | public static final Product ses = new Product("ses"); 42 | public static final Product sns = new Product("sns"); 43 | public static final Product sqs = new Product("sqs"); 44 | public static final Product storage_gateway = new Product("storage_gateway"); 45 | public static final Product sws = new Product("sws"); 46 | public static final Product vpc = new Product("vpc"); 47 | 48 | public Product (String name) { 49 | super(name); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /conf/system/setenv.sh: -------------------------------------------------------------------------------- 1 | # Determine the max heap size based on the instance type. 2 | # See: http://aws.amazon.com/ec2/instance-types/ 3 | # 4 | # For nodes in the datacenter we rely on the convention that the hostname 5 | # indicates the amount of memory. 6 | function heapSize { 7 | instanceType=$(hostname) 8 | if [ "$EC2_INSTANCE_TYPE" != "" ]; then 9 | instanceType="$EC2_INSTANCE_TYPE" 10 | fi 11 | 12 | case $instanceType in 13 | m1.small) echo "1g" ;; # 1.7GB 14 | m1.large) echo "6g" ;; # 7.5GB 15 | m1.xlarge) echo "12g" ;; # 15GB 16 | t1.micro) echo "256m" ;; # 613MB 17 | m2.xlarge) echo "14g" ;; # 17.1GB 18 | m2.2xlarge) echo "30g" ;; # 34.2GB 19 | m2.4xlarge) echo "60g" ;; # 68.4GB 20 | c1.medium) echo "1g" ;; # 1.7GB 21 | c1.xlarge) echo "4g" ;; # 7GB 22 | cc1.4xlarge) echo "20g" ;; # 23GB 23 | cg1.4xlarge) echo "20g" ;; # 22GB 24 | *-4g*) echo "3g" ;; 25 | *-8g*) echo "6g" ;; 26 | *-16g*) echo "14g" ;; 27 | *-24g*) echo "21g" ;; 28 | *-32g*) echo "28g" ;; 29 | *-64g*) echo "60g" ;; 30 | *-72g*) echo "68g" ;; 31 | *) echo "1g" ;; # who knows, try for a gig 32 | esac 33 | } 34 | 35 | # Create the java heap size options. We always set min and max heap size to 36 | # be the same, this ensures that the program will crash right away if there 37 | # is not enough memory. 38 | function javaOptsHeapSize { 39 | permPercent=$1 40 | max=$(heapSize) 41 | permSize="$(echo "$max" | awk "{print int(\$1 * $permPercent)}")g" 42 | echo "-Xms$max -Xmx$max -XX:PermSize=$permSize -XX:MaxPermSize=$permSize" 43 | } 44 | 45 | GCLOG=/apps/tomcat/logs/gc.log 46 | CATALINA_PID=/apps/tomcat/logs/catalina.pid 47 | JRE_HOME=/apps/java 48 | JAVA_HOME=$JRE_HOME 49 | if [ "$1" == "start" ]; then 50 | JAVA_OPTS=" \ 51 | -javaagent:/apps/appagent/javaagent.jar \ 52 | -verbosegc \ 53 | -XX:+PrintGCDetails \ 54 | -XX:+PrintGCTimeStamps \ 55 | -Xloggc:$GCLOG \ 56 | $(javaOptsHeapSize 0.1) \ 57 | -Xss8192k \ 58 | -Dnetflix.logging.realtimetracers=false \ 59 | -Dlog4j.appender.RTA=org.apache.log4j.varia.NullAppender \ 60 | -Dlog4j.appender.CHUKWA=org.apache.log4j.varia.NullAppender \ 61 | -Dlog4j.appender.CHUKWA.threshold=OFF \ 62 | -Dlog4j.logger.net.spy.memcached.internal.OperationFuture=FATAL \ 63 | -Dnet.spy.log.LoggerImpl=net.spy.memcached.compat.log.Log4JLogger \ 64 | -Dnetflix.environment=test 65 | -Dcom.sun.management.jmxremote.ssl=false \ 66 | -Dcom.sun.management.jmxremote.port=7500 \ 67 | -Dcom.sun.management.jmxremote.authenticate=false \ 68 | -Dnetflix.appinfo.name=${NETFLIX_APP}" 69 | 70 | logrotate -f /apps/tomcat/bin/.logrotate.conf 71 | else 72 | JAVA_OPTS="" 73 | fi 74 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/DataWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.netflix.ice.common.AwsUtils; 21 | import com.netflix.ice.common.TagGroup; 22 | import com.netflix.ice.tag.Tag; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.io.*; 27 | import java.util.Collection; 28 | import java.util.TreeMap; 29 | 30 | public class DataWriter { 31 | private final static Logger logger = LoggerFactory.getLogger(DataWriter.class); 32 | 33 | 34 | private TreeMap> tagGroups; 35 | private ProcessorConfig config = ProcessorConfig.getInstance(); 36 | private String dbName; 37 | private File file; 38 | private ReadWriteData data; 39 | 40 | DataWriter(String name, boolean loadData) throws Exception { 41 | 42 | name = Tag.toS3(name); 43 | dbName = name; 44 | file = new File(config.localDir, dbName); 45 | if (loadData) { 46 | AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, file); 47 | } 48 | 49 | if (file.exists()) { 50 | DataInputStream in = new DataInputStream(new FileInputStream(file)); 51 | try { 52 | data = ReadWriteData.Serializer.deserialize(in); 53 | } 54 | finally { 55 | in.close(); 56 | } 57 | } 58 | else { 59 | data = new ReadWriteData(); 60 | } 61 | } 62 | 63 | ReadWriteData getData() { 64 | return data; 65 | } 66 | 67 | void archive() throws IOException { 68 | archive(data); 69 | } 70 | 71 | void archive(ReadWriteData data) throws IOException { 72 | 73 | DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); 74 | try { 75 | ReadWriteData.Serializer.serialize(out, data); 76 | } 77 | finally { 78 | out.close(); 79 | } 80 | 81 | logger.info(this.dbName + " uploading to s3..."); 82 | AwsUtils.upload(config.workS3BucketName, config.workS3BucketPrefix, config.localDir, dbName); 83 | logger.info(this.dbName + " uploading done."); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/TagGroupWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.google.common.collect.Maps; 21 | import com.netflix.ice.common.AwsUtils; 22 | import com.netflix.ice.common.TagGroup; 23 | import com.netflix.ice.tag.Tag; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | import java.io.*; 28 | import java.util.Collection; 29 | import java.util.TreeMap; 30 | 31 | public class TagGroupWriter { 32 | private final static Logger logger = LoggerFactory.getLogger(TagGroupWriter.class); 33 | public final static String DB_PREFIX = "tagdb_"; 34 | 35 | private TreeMap> tagGroups; 36 | private ProcessorConfig config = ProcessorConfig.getInstance(); 37 | private String dbName; 38 | private File file; 39 | 40 | TagGroupWriter(String name) throws Exception { 41 | 42 | name = Tag.toS3(name); 43 | dbName = DB_PREFIX + name; 44 | file = new File(config.localDir, dbName); 45 | logger.info("creating TagGroupWriter for " + file); 46 | AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, file); 47 | 48 | if (file.exists()) { 49 | DataInputStream in = new DataInputStream(new FileInputStream(file)); 50 | try { 51 | tagGroups = TagGroup.Serializer.deserializeTagGroups(config, in); 52 | } 53 | finally { 54 | if (in != null) 55 | in.close(); 56 | } 57 | } 58 | else { 59 | tagGroups = Maps.newTreeMap(); 60 | } 61 | } 62 | 63 | void archive(Long monthMilli,Collection tagGroups) throws IOException { 64 | this.tagGroups.put(monthMilli, tagGroups); 65 | 66 | DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); 67 | try { 68 | TagGroup.Serializer.serializeTagGroups(out, this.tagGroups); 69 | } 70 | finally { 71 | out.close(); 72 | } 73 | 74 | logger.info(dbName + " uploading to s3..."); 75 | AwsUtils.upload(config.workS3BucketName, config.workS3BucketPrefix, config.localDir, dbName); 76 | logger.info(dbName + " uploading done."); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/AccountService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import com.netflix.ice.tag.Account; 21 | import com.netflix.ice.tag.Zone; 22 | 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | public interface AccountService { 27 | /** 28 | * Get account by AWS id. The AWS id is usually an un-readable 12 digit string. 29 | * @param accountId 30 | * @return Account object associated with the account id 31 | */ 32 | Account getAccountById(String accountId); 33 | 34 | /** 35 | * Get account by account name. The account name is a user defined readable string. 36 | * @param accountName 37 | * @return Account object associated with the account name 38 | */ 39 | Account getAccountByName(String accountName); 40 | 41 | /** 42 | * Get a list of accounts from given account names. 43 | * @param accountNames 44 | * @return List of accounts 45 | */ 46 | List getAccounts(List accountNames); 47 | 48 | /** 49 | * If you don't have reserved instances, you can return an empty map. 50 | * @return Map of accounts. The keys are owner accounts, the values are list of borrowing accounts. 51 | */ 52 | Map> getReservationAccounts(); 53 | 54 | /** 55 | * If you don't need to poll reservation capacity through ec2 API for other accounts, you can return an empty map. 56 | * @return Map of account access roles. The keys are reservation owner accounts, 57 | * the values are assumed roles to call ec2 describeReservedInstances on each reservation owner account. 58 | */ 59 | Map getReservationAccessRoles(); 60 | 61 | /** 62 | * If you don't need to poll reservation capacity through ec2 API for other accounts, ir if you don't use external ids, 63 | * you can return an empty map. 64 | * @return Map of account access external ids. The keys are reservation owner accounts, 65 | * the values are external ids to call ec2 describeReservedInstances on each reservation owner account. 66 | */ 67 | Map getReservationAccessExternalIds(); 68 | 69 | /** 70 | * @param account 71 | * @param zone 72 | * @return Whether or not external mappings are not available in specified account. 73 | */ 74 | boolean externalMappingExist(Account account, Zone zone); 75 | } 76 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/ReservationService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.amazonaws.services.ec2.model.ReservedInstances; 21 | import com.netflix.ice.common.TagGroup; 22 | import com.netflix.ice.tag.Region; 23 | import com.netflix.ice.tag.UsageType; 24 | 25 | import java.util.Collection; 26 | import java.util.Map; 27 | 28 | /** 29 | * Interface for reservations. 30 | */ 31 | public interface ReservationService { 32 | 33 | void init(); 34 | 35 | /** 36 | * Get all tag groups with reservations 37 | * @param utilization 38 | * @return 39 | */ 40 | Collection getTagGroups(Ec2InstanceReservationPrice.ReservationUtilization utilization); 41 | 42 | /** 43 | * 44 | * @return 45 | */ 46 | Ec2InstanceReservationPrice.ReservationUtilization getDefaultReservationUtilization(long time); 47 | 48 | /** 49 | * Get reservation info. 50 | * @param time 51 | * @param tagGroup 52 | * @param utilization 53 | * @return 54 | */ 55 | ReservationInfo getReservation( 56 | long time, 57 | TagGroup tagGroup, 58 | Ec2InstanceReservationPrice.ReservationUtilization utilization); 59 | 60 | /** 61 | * Some companies may get different price tiers at different times depending on reservation cost. 62 | * This method is to get the latest hourly price including amortized upfront for given, time, region and usage type. 63 | * @param time 64 | * @param region 65 | * @param usageType 66 | * @param utilization 67 | * @return 68 | */ 69 | double getLatestHourlyTotalPrice( 70 | long time, 71 | Region region, 72 | UsageType usageType, 73 | Ec2InstanceReservationPrice.ReservationUtilization utilization); 74 | 75 | /** 76 | * Called by ReservationCapacityPoller to update reservations. 77 | * @param reservations 78 | */ 79 | void updateEc2Reservations(Map reservations); 80 | 81 | 82 | public static class ReservationInfo { 83 | public final int capacity; 84 | public final double upfrontAmortized; 85 | public final double reservationHourlyCost; 86 | 87 | public ReservationInfo (int capacity, double upfrontAmortized, double reservationHourlyCost) { 88 | this.capacity = capacity; 89 | this.upfrontAmortized = upfrontAmortized; 90 | this.reservationHourlyCost = reservationHourlyCost; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/com/netflix/ice/basic/EddaResourceServiceTest.java: -------------------------------------------------------------------------------- 1 | package com.netflix.ice.basic; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.util.Properties; 6 | 7 | import org.junit.Test; 8 | 9 | import org.json.JSONArray; 10 | import com.netflix.ice.tag.Product; 11 | 12 | 13 | /** 14 | * Note: These tests require a running instance of Edda running at the url configured in ice.properties, so they 15 | * strictly speaking not unit tests any more... 16 | */ 17 | public class EddaResourceServiceTest { 18 | 19 | @Test 20 | public void test() throws Exception { 21 | EddaResourceService service = new EddaResourceService(new Properties()); 22 | 23 | service.init(); 24 | 25 | // does nothing really... 26 | service.commit(); 27 | 28 | assertNotNull(service.getProductsWithResources()); 29 | 30 | assertEquals("Product-name for unsupported resource", "somename", service.getResource(null, null, new Product("somename"), null, null, 0)); 31 | assertEquals("Error for empty resourceId", "Error", service.getResource(null, null, Product.ec2, null, null, 0)); 32 | assertEquals("Error for empty resourceId", "Error", service.getResource(null, null, Product.ec2, "", null, 0)); 33 | 34 | assertEquals("Unknown for resourceIds that we do not find", "Unknown", service.getResource(null, null, Product.ec2, "someunknowninstance", null, 0)); 35 | 36 | JSONArray instances = service.readInstanceArray(); 37 | 38 | String resource = service.getResource(null, null, Product.ec2, instances.getString(0), null, 0); 39 | assertFalse("Not Error for an actual instance", "Error".equals(resource)); 40 | 41 | resource = service.getResource(null, null, Product.ec2_instance, instances.getString(0), null, 0); 42 | assertFalse("Not Error for an actual instance", "Error".equals(resource)); 43 | 44 | for(int i = 0;i < instances.length();i++) { 45 | resource = service.getResource(null, null, Product.ec2_instance, instances.getString(i), null, 0); 46 | assertFalse("Not Error for an actual instance", "Error".equals(resource)); 47 | } 48 | } 49 | 50 | @Test 51 | public void testWrongURL() throws Exception { 52 | // use a normal setup for retrieving the instances 53 | EddaResourceService service = new EddaResourceService(new Properties()); 54 | JSONArray instances = service.readInstanceArray(); 55 | 56 | // overwrite config with an invalid hostname 57 | Properties prop = new Properties(); 58 | prop.setProperty("ice.eddaresourceservice.url", "http://invalidhostname:18081/edda/api/v2/"); 59 | service = new EddaResourceService(prop); 60 | 61 | // now the retrieved resources should return an error even for valid instances 62 | String resource = service.getResource(null, null, Product.ec2, instances.getString(0), null, 0); 63 | assertTrue("Error even for an actual instance when using wrong URL", "Error".equals(resource)); 64 | 65 | // overwrite config with an invalid URL 66 | prop.setProperty("ice.eddaresourceservice.url", "sasie://invalidhostname:18081/edda/api/v2/"); 67 | service = new EddaResourceService(prop); 68 | 69 | // now the retrieved resources should return an error even for valid instances 70 | resource = service.getResource(null, null, Product.ec2, instances.getString(0), null, 0); 71 | assertTrue("Error even for an actual instance when using wrong URL", "Error".equals(resource)); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Exit the script if an error occur 4 | set -e 5 | 6 | # Get Ice in a ready-to-run state on a fresh AWI instance. 7 | 8 | GRAILS_VERSION=2.4.4 9 | 10 | # Install prerequisites 11 | if [ -f /etc/redhat-release ]; then 12 | echo "Installing redhat packages" 13 | sudo yum -y install git java-1.7.0-openjdk-devel.x86_64 wget unzip 14 | 15 | elif [[ -f /etc/debian-release || -f /etc/debian_version ]];then 16 | echo "Installing debian packages" 17 | sudo apt-get -y install git openjdk-7-jdk wget unzip 18 | 19 | elif [[ -f /etc/issue && $(grep "Amazon Linux AMI" /etc/issue) ]]; then 20 | echo "Assuming AWS AMI, installing packages" 21 | sudo yum -y install git java-1.7.0-openjdk-devel.x86_64 wget unzip 22 | 23 | else 24 | echo "Unknown operating system. You may have to install Java 7 manually." 25 | fi 26 | 27 | INSTALL_DIR=$(pwd) 28 | 29 | cd 30 | 31 | HOME_DIR=$(pwd) 32 | 33 | # Prep grails in such a way that we only download it once 34 | if [ ! -x ".grails/wrapper/${GRAILS_VERSION}/grails-${GRAILS_VERSION}" ]; then 35 | mkdir -p .grails/wrapper/ 36 | cd .grails/wrapper/ 37 | # (Fetch) 38 | wget http://dist.springframework.org.s3.amazonaws.com/release/GRAILS/grails-${GRAILS_VERSION}.zip -O grails-${GRAILS_VERSION}-download.zip 39 | mkdir ${GRAILS_VERSION} 40 | cd ${GRAILS_VERSION} 41 | # ("Install") 42 | unzip ../grails-${GRAILS_VERSION}-download.zip 43 | fi 44 | 45 | GRAILS_HOME=${HOME_DIR}/.grails/wrapper/${GRAILS_VERSION}/grails-${GRAILS_VERSION}/ 46 | PATH=$PATH:${HOME_DIR}/.grails/wrapper/${GRAILS_VERSION}/grails-${GRAILS_VERSION}/bin/ 47 | 48 | # Get ice 49 | cd ${INSTALL_DIR} 50 | 51 | if [ -x '.git' ]; then 52 | # We already have it; update to latest git 53 | git pull 54 | else 55 | # We don't have it at all yet; clone the repo 56 | git clone https://github.com/Netflix/ice.git 57 | cd ice 58 | INSTALL_DIR=$(pwd) 59 | fi 60 | 61 | # Initialize Ice with Grails 62 | grails ${JAVA_OPTS} wrapper 63 | 64 | # (Bug: Ice can't deal with this file existing and being empty.) 65 | rm -f grails-app/i18n/messages.properties 66 | 67 | # Create our local work directories (both for processing and reading) 68 | mkdir -p ${HOME_DIR}/ice_processor 69 | mkdir -p ${HOME_DIR}/ice_reader 70 | 71 | # Set up the config file 72 | cp src/java/sample.properties src/java/ice.properties 73 | echo Please enter the name of the bucket Ice will read Amazon billing information from: 74 | while [ "$BILLBUCKET" == "" ] 75 | do 76 | echo -n "-> " 77 | read -r BILLBUCKET 78 | done 79 | 80 | echo Please enter the name of the bucket Ice will write processed billing information to: 81 | while [ "$PROCBUCKET" == "" ] 82 | do 83 | echo -n "-> " 84 | read -r PROCBUCKET 85 | done 86 | sed -ri 's/=billing_s3bucketprefix\//=/; s|\/mnt\/|'"${HOME_DIR}"'\/|; s/=work_s3bucketprefix\//=/; s/^ice.account.*//; s/=billing_s3bucketname1/='${BILLBUCKET}'/; s/=work_s3bucketname/='${PROCBUCKET}'/' src/java/ice.properties 87 | 88 | echo Ice is now ready to run as a processor. If you want to run the reader, edit: 89 | echo "${INSTALL_DIR}/src/java/ice.properties" 90 | echo and alter the appropriate flags. You can now start Ice by running the following from the Ice root directory: 91 | echo ./grailsw -Djava.net.preferIPv4Stack=true -Dice.s3AccessKeyId=\ -Dice.s3SecretKey=\ run-app 92 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ApplicationGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | import org.json.JSONArray; 23 | import org.json.JSONException; 24 | import org.json.JSONObject; 25 | import org.json.JSONTokener; 26 | 27 | import java.util.Iterator; 28 | import java.util.List; 29 | import java.util.Map; 30 | 31 | public class ApplicationGroup { 32 | public final Map> data; 33 | public final String name; 34 | public final String owner; 35 | 36 | // todo: remove this constructor 37 | public ApplicationGroup(String name, String owner, List ec2s, List s3s) { 38 | this.name = name; 39 | this.owner = owner; 40 | this.data = Maps.newHashMap(); 41 | this.data.put("rds", Lists.newArrayList()); 42 | this.data.put("ec2", ec2s); 43 | this.data.put("s3", s3s); 44 | } 45 | 46 | public ApplicationGroup(String name, String owner, Map> data) { 47 | this.name = name; 48 | this.owner = owner; 49 | this.data = data; 50 | } 51 | 52 | public ApplicationGroup(String jsonStr) throws JSONException { 53 | data = Maps.newHashMap(); 54 | JSONObject json = new JSONObject(new JSONTokener(jsonStr)); 55 | name = json.getString("name"); 56 | owner = json.getString("owner"); 57 | json = json.getJSONObject("data"); 58 | Iterator keys = json.keys(); 59 | while (keys.hasNext()) { 60 | String key = keys.next(); 61 | JSONArray jsonArray = json.getJSONArray(key); 62 | List values = Lists.newArrayList(); 63 | for (int i = 0; i < jsonArray.length(); i++) 64 | values.add(jsonArray.getString(i)); 65 | data.put(key, values); 66 | } 67 | } 68 | 69 | public JSONObject getJSON() throws JSONException { 70 | JSONObject json = new JSONObject(); 71 | json.put("name", name); 72 | json.put("owner", owner); 73 | json.put("data", new JSONObject(data)); 74 | 75 | return json; 76 | } 77 | 78 | @Override 79 | public String toString() { 80 | try { 81 | return getJSON().toString(); 82 | } 83 | catch (Exception e) { 84 | return e.getMessage(); 85 | } 86 | } 87 | 88 | public String getDisplayName() { 89 | return "Application Group " + name; 90 | } 91 | 92 | public String getLink() { 93 | return "dashboard/appgroup#appgroup=" + name; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/Config.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import com.amazonaws.auth.AWSCredentialsProvider; 21 | import org.joda.time.DateTime; 22 | import org.joda.time.DateTimeZone; 23 | 24 | import java.util.Properties; 25 | 26 | public abstract class Config { 27 | 28 | public final String workS3BucketName; 29 | public final String workS3BucketPrefix; 30 | public final String localDir; 31 | public final AccountService accountService; 32 | public final ProductService productService; 33 | public final ResourceService resourceService; 34 | public final DateTime startDate; 35 | public final AWSCredentialsProvider credentialsProvider; 36 | 37 | /** 38 | * 39 | * @param properties (required) 40 | * @param credentialsProvider (required) 41 | * @param accountService (required) 42 | * @param productService (required) 43 | * @param resourceService (optional) 44 | */ 45 | public Config( 46 | Properties properties, 47 | AWSCredentialsProvider credentialsProvider, 48 | AccountService accountService, 49 | ProductService productService, 50 | ResourceService resourceService) { 51 | if (properties.getProperty(IceOptions.START_MILLIS) == null) throw new IllegalArgumentException("IceOptions.START_MILLIS must be specified"); 52 | if (properties == null) throw new IllegalArgumentException("properties must be specified"); 53 | if (credentialsProvider == null) throw new IllegalArgumentException("credentialsProvider must be specified"); 54 | if (accountService == null) throw new IllegalArgumentException("accountService must be specified"); 55 | if (productService == null) throw new IllegalArgumentException("productService must be specified"); 56 | 57 | DateTime startDate = new DateTime(Long.parseLong(properties.getProperty(IceOptions.START_MILLIS)), DateTimeZone.UTC); 58 | workS3BucketName = properties.getProperty(IceOptions.WORK_S3_BUCKET_NAME); 59 | workS3BucketPrefix = properties.getProperty(IceOptions.WORK_S3_BUCKET_PREFIX, "ice/"); 60 | localDir = properties.getProperty(IceOptions.LOCAL_DIR, "/mnt/ice"); 61 | 62 | if (workS3BucketName == null) throw new IllegalArgumentException("IceOptions.WORK_S3_BUCKET_NAME must be specified"); 63 | 64 | this.credentialsProvider = credentialsProvider; 65 | this.startDate = startDate; 66 | this.accountService = accountService; 67 | this.productService = productService; 68 | this.resourceService = resourceService; 69 | 70 | AwsUtils.init(credentialsProvider); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/ResourceService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import com.netflix.ice.tag.Account; 21 | import com.netflix.ice.tag.Product; 22 | import com.netflix.ice.tag.Region; 23 | 24 | import java.util.List; 25 | 26 | /** 27 | * Please see a sample of subclass in SampleMapDbResourceService class. 28 | */ 29 | public abstract class ResourceService { 30 | 31 | /** 32 | * Subclass can choose different technologies to store the mapping of resource ids and resource group names. 33 | * E.g. SampleMapDbResourceService used MapDB to maintain a mapping of instance ids and application names. 34 | * You can initialize you mapping here. 35 | * You can also get the reference to ProcessorConfig instance here. 36 | */ 37 | abstract public void init(); 38 | 39 | /** 40 | * Get resource group name. Subclass can maintain a mapping of resource ids and resource group names. 41 | * Users can also choose to add user-defined tags in the billing file. E.g. in SampleMapDbResourceService, 42 | * the auto-scaling-group name is used to generate the resource group name. 43 | * @param account 44 | * @param region 45 | * @param product 46 | * @param resourceId: depending on product, resourceId could be: 47 | * 1) instance id, if product is ec2_instance. You can use Edda (https://github.com/Netflix/edda) to query application name from instance id. 48 | * 2) volumn id, if product is ebs. You can use Edda (https://github.com/Netflix/edda) to query instance id from volumn id, then application name from instance id. 49 | * 3) s3 bucket name if product is s3 50 | * 4) db name if product is rds 51 | * 5) etc. 52 | * @param lineItem: the line item in the billing file. You can access your user-defined tags here. 53 | * @param millisStart 54 | * @return 55 | */ 56 | public String getResource(Account account, Region region, Product product, String resourceId, String[] lineItem, long millisStart){ 57 | return product.name; 58 | } 59 | 60 | /** 61 | * Get products with resources. See example in SampleMapDbResourceService. This method will be used by UI to list 62 | * products with resources. 63 | * @return List of list of products. The inner list of products share the same resource groups. 64 | * E.g. in SampleMapDbResourceService, for products (ec2, ec2_instance, ebs), the resource group names are application names. 65 | */ 66 | abstract public List> getProductsWithResources(); 67 | 68 | /** 69 | * Commit resource mappings. This method will be called at the end of billing file processing to commit your mappings. 70 | */ 71 | abstract public void commit(); 72 | } 73 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/UsageType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.io.DataInput; 26 | import java.io.DataOutput; 27 | import java.io.IOException; 28 | import java.util.List; 29 | import java.util.concurrent.ConcurrentMap; 30 | 31 | public class UsageType extends Tag { 32 | private static final Logger logger = LoggerFactory.getLogger(Operation.class); 33 | public final String unit; 34 | 35 | private UsageType (String name, String unit) { 36 | super(name); 37 | this.unit = unit; 38 | } 39 | private static ConcurrentMap usageTypes = Maps.newConcurrentMap(); 40 | 41 | public static void serialize(DataOutput out, UsageType usageType) throws IOException { 42 | out.writeUTF(usageType.name); 43 | out.writeUTF(usageType.unit); 44 | } 45 | 46 | public static UsageType deserialize(DataInput in) throws IOException { 47 | String name = in.readUTF(); 48 | String unit = in.readUTF(); 49 | 50 | UsageType usageType = usageTypes.get(name); 51 | if (usageType == null) { 52 | usageTypes.putIfAbsent(name, new UsageType(name, unit)); 53 | usageType = usageTypes.get(name); 54 | } 55 | else if (!usageType.unit.equals(unit)) { 56 | logger.error("found different units for " + usageType + usageType.unit + " " + unit); 57 | } 58 | return usageType; 59 | } 60 | 61 | public static UsageType getUsageType(String name, Operation operation, String description) { 62 | String unit = ""; 63 | if (name.contains("Bytes") || name.contains("ByteHrs") || description.contains("GB")) 64 | unit = "GB"; 65 | if (operation instanceof Operation.ReservationOperation) 66 | unit = "hours"; 67 | 68 | return getUsageType(name, unit); 69 | } 70 | 71 | public static UsageType getUsageType(String name, String unit) { 72 | 73 | UsageType usageType = usageTypes.get(name); 74 | if (usageType == null) { 75 | usageTypes.putIfAbsent(name, new UsageType(name, unit)); 76 | usageType = usageTypes.get(name); 77 | } 78 | else if (!usageType.unit.equals(unit)) { 79 | logger.error("found different units for " + usageType + usageType.unit + " " + unit); 80 | } 81 | return usageType; 82 | } 83 | 84 | public static List getUsageTypes(List names) { 85 | List result = Lists.newArrayList(); 86 | for (String name: names) 87 | result.add(usageTypes.get(name)); 88 | return result; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /grails-app/views/dashboard/editappgroup.gsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | 3 | Copyright 2013 Netflix, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | --%> 18 | 19 | <%@ page contentType="text/html;charset=UTF-8" %> 20 | 21 | 22 | 23 | Edit Application Group 24 | 25 | 26 |
27 | 28 |
{{message}}
29 | 30 |

CreateEdit Application Group {{appgroup.name}}

31 | 32 |
33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 49 | 53 | 57 | 58 | 59 |
Group Name:
Email:
Selected {{row.displayName}}:
46 | 47 | 48 |




50 |
51 | 52 | 		Rest of {{row.displayName}}:
54 | 55 | 56 |
60 |
61 | 62 | 63 | 64 | Cancel 65 |
66 |
67 | 68 |
69 | 70 | -------------------------------------------------------------------------------- /grails-app/conf/Config.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // locations to search for config files that get merged into the main config 18 | // config files can either be Java properties files or ConfigSlurper scripts 19 | 20 | grails.config.locations = [] 21 | 22 | if(System.properties["${appName}.config.location"]) { 23 | grails.config.locations << "file:" + System.properties["${appName}.config.location"] 24 | } 25 | 26 | grails.project.groupId = appName // change this to alter the default package name and Maven publishing destination 27 | grails.mime.file.extensions = true // enables the parsing of file extensions from URLs into the request format 28 | grails.mime.use.accept.header = false 29 | grails.mime.types = [html: ['text/html', 'application/xhtml+xml'], 30 | xml: ['text/xml', 'application/xml'], 31 | text: 'text/plain', 32 | js: 'text/javascript', 33 | rss: 'application/rss+xml', 34 | atom: 'application/atom+xml', 35 | css: 'text/css', 36 | csv: 'text/csv', 37 | all: '*/*', 38 | json: ['application/json', 'text/json'], 39 | form: 'application/x-www-form-urlencoded', 40 | multipartForm: 'multipart/form-data' 41 | ] 42 | 43 | // URL Mapping Cache Max Size, defaults to 5000 44 | //grails.urlmapping.cache.maxsize = 1000 45 | 46 | // The default codec used to encode data with ${} 47 | grails.views.default.codec = "none" // none, html, base64 48 | grails.views.gsp.encoding = "UTF-8" 49 | grails.converters.encoding = "UTF-8" 50 | // enable Sitemesh preprocessing of GSP pages 51 | grails.views.gsp.sitemesh.preprocess = true 52 | // scaffolding templates configuration 53 | grails.scaffolding.templates.domainSuffix = 'Instance' 54 | 55 | // Set to false to use the new Grails 1.2 JSONBuilder in the render method 56 | grails.json.legacy.builder = false 57 | // enabled native2ascii conversion of i18n properties files 58 | grails.enable.native2ascii = true 59 | // whether to install the java.util.logging bridge for sl4j. Disable for AppEngine! 60 | grails.logging.jul.usebridge = true 61 | // packages to include in Spring bean scanning 62 | grails.spring.bean.packages = [] 63 | 64 | // set per-environment serverURL stem for creating absolute links 65 | environments { 66 | production { 67 | } 68 | development { 69 | grails.serverURL = "http://localhost:8080/${appName}" 70 | } 71 | test { 72 | grails.serverURL = "http://localhost:8080/${appName}" 73 | } 74 | 75 | } 76 | 77 | log4j = { 78 | error 'org.codehaus.groovy.grails.web.servlet', // controllers 79 | 'org.codehaus.groovy.grails.web.pages', // GSP 80 | 'org.codehaus.groovy.grails.web.sitemesh', // layouts 81 | 'org.codehaus.groovy.grails.web.mapping.filter', // URL mapping 82 | 'org.codehaus.groovy.grails.web.mapping', // URL mapping 83 | 'org.codehaus.groovy.grails.commons', // core / classloading 84 | 'org.codehaus.groovy.grails.plugins', // plugins 85 | 'org.codehaus.groovy.grails.orm.hibernate' // hibernate integration 86 | 87 | warn 'org.mortbay.log' 88 | 89 | root { 90 | info() 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicAccountService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.netflix.ice.basic; 17 | 18 | import com.google.common.collect.Lists; 19 | import com.google.common.collect.Maps; 20 | import com.netflix.ice.common.AccountService; 21 | import com.netflix.ice.tag.Account; 22 | import com.netflix.ice.tag.Zone; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.util.List; 27 | import java.util.Map; 28 | 29 | public class BasicAccountService implements AccountService { 30 | 31 | Logger logger = LoggerFactory.getLogger(getClass()); 32 | 33 | private Map accountsById = Maps.newConcurrentMap(); 34 | private Map accountsByName = Maps.newConcurrentMap(); 35 | private Map> reservationAccounts = Maps.newHashMap(); 36 | private Map reservationAccessRoles = Maps.newHashMap(); 37 | private Map reservationAccessExternalIds = Maps.newHashMap(); 38 | 39 | public BasicAccountService(List accounts, Map> reservationAccounts, 40 | Map reservationAccessRoles, Map reservationAccessExternalIds) { 41 | this.reservationAccounts = reservationAccounts; 42 | this.reservationAccessRoles = reservationAccessRoles; 43 | this.reservationAccessExternalIds = reservationAccessExternalIds; 44 | for (Account account: accounts) { 45 | accountsByName.put(account.name, account); 46 | accountsById.put(account.id, account); 47 | } 48 | } 49 | 50 | public Account getAccountById(String accountId) { 51 | Account account = accountsById.get(accountId); 52 | if (account == null) { 53 | account = new Account(accountId, accountId); 54 | accountsByName.put(account.name, account); 55 | accountsById.put(account.id, account); 56 | logger.info("created account " + accountId + "."); 57 | } 58 | return account; 59 | } 60 | 61 | public Account getAccountByName(String accountName) { 62 | Account account = accountsByName.get(accountName); 63 | // for accounts that were not mapped to names in ice.properties (ice.account.xxx), this check will make sure that 64 | // data/tags are updated properly once the mapping is established in ice.properties 65 | if (account == null) { 66 | account = accountsById.get(accountName); 67 | } 68 | if (account == null) { 69 | account = new Account(accountName, accountName); 70 | accountsByName.put(account.name, account); 71 | accountsById.put(account.id, account); 72 | } 73 | return account; 74 | } 75 | 76 | public List getAccounts(List accountNames) { 77 | List result = Lists.newArrayList(); 78 | for (String name: accountNames) 79 | result.add(accountsByName.get(name)); 80 | return result; 81 | } 82 | 83 | public Map> getReservationAccounts() { 84 | return reservationAccounts; 85 | } 86 | 87 | public Map getReservationAccessRoles() { 88 | return reservationAccessRoles; 89 | } 90 | 91 | 92 | public Map getReservationAccessExternalIds() { 93 | return reservationAccessExternalIds; 94 | } 95 | 96 | public boolean externalMappingExist(Account account, Zone zone) { 97 | return true; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/Poller.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.io.InterruptedIOException; 24 | import java.nio.channels.ClosedByInterruptException; 25 | import java.util.concurrent.CountDownLatch; 26 | import java.util.concurrent.TimeUnit; 27 | 28 | public abstract class Poller { 29 | 30 | protected Logger logger = LoggerFactory.getLogger(getClass()); 31 | private Thread pollerThread; 32 | private CountDownLatch threadDoneSignal; 33 | 34 | private void doWork(int initialDelaySec, int delaySec, boolean fixedRate) { 35 | logger.info("poller starting..."); 36 | boolean first = true; 37 | long sleepTime = delaySec * 1000L; 38 | while (true) { 39 | try { 40 | if (first) { 41 | if (initialDelaySec > 0) { 42 | Thread.sleep(initialDelaySec * 1000L); 43 | } 44 | first = false; 45 | } 46 | else if (sleepTime > 0) 47 | Thread.sleep(sleepTime); 48 | 49 | long startMillis = System.currentTimeMillis(); 50 | poll(); 51 | sleepTime = fixedRate ? delaySec * 1000L - (System.currentTimeMillis() - startMillis) : delaySec * 1000L; 52 | } 53 | catch (InterruptedException e) { 54 | break; 55 | } 56 | catch (InterruptedIOException e) { 57 | break; 58 | } 59 | catch (ClosedByInterruptException e) { 60 | break; 61 | } 62 | catch (Exception e) { 63 | logger.error("Error polling", e); 64 | } 65 | } 66 | threadDoneSignal.countDown(); 67 | logger.info("poller stopping."); 68 | } 69 | 70 | protected abstract void poll() throws Exception; 71 | 72 | protected String getThreadName() { 73 | return getClass().getName(); 74 | } 75 | 76 | public void start() { 77 | start(0, 3600, false); 78 | } 79 | 80 | public void start(final int delaySec) { 81 | start(0, delaySec, false); 82 | } 83 | 84 | public void start(final int initialDelaySec, final int delaySec, final boolean fixedRate) { 85 | threadDoneSignal = new CountDownLatch(1); 86 | pollerThread = new Thread(new Runnable() { 87 | public void run() { 88 | doWork(initialDelaySec, delaySec, fixedRate); 89 | } 90 | }, getThreadName()); 91 | pollerThread.start(); 92 | logger.info("poller thread for " + getThreadName() + " started..."); 93 | } 94 | 95 | public void shutdown() { 96 | logger.info("shutting down... trying to interrupt poller thread..."); 97 | boolean done = false; 98 | int numTries = 0; 99 | while (!done) { 100 | pollerThread.interrupt(); 101 | try { 102 | done = threadDoneSignal.await(10, TimeUnit.SECONDS); 103 | } 104 | catch (InterruptedException e) { 105 | // ingore InterruptedException here 106 | } 107 | if (!done) { 108 | numTries = numTries + 1; 109 | logger.warn("trying to interrupt write thread again " + numTries); 110 | } 111 | else { 112 | logger.info("shutted down successfully."); 113 | } 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/SampleMapDbResourceService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.netflix.ice.common.ProductService; 22 | import com.netflix.ice.common.ResourceService; 23 | import com.netflix.ice.processor.ProcessorConfig; 24 | import com.netflix.ice.tag.Account; 25 | import com.netflix.ice.tag.Product; 26 | import com.netflix.ice.tag.Region; 27 | import org.apache.commons.lang.StringUtils; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.util.List; 32 | 33 | /** 34 | * 35 | */ 36 | public class SampleMapDbResourceService extends ResourceService { 37 | 38 | public static final String UNKNOWN = "unknown"; 39 | private static final Logger logger = LoggerFactory.getLogger(SampleMapDbResourceService.class); 40 | private static List> productsWithResources = Lists.>newArrayList( 41 | Lists.newArrayList(Product.ec2, Product.ec2_instance, Product.ebs), 42 | Lists.newArrayList(Product.rds), 43 | Lists.newArrayList(Product.s3)); 44 | MapDb instanceDb; 45 | ProcessorConfig config; 46 | 47 | public void init() { 48 | config = ProcessorConfig.getInstance(); 49 | instanceDb = new MapDb("instances"); 50 | } 51 | 52 | @Override 53 | public void commit() { 54 | instanceDb.commit(); 55 | } 56 | 57 | @Override 58 | public List> getProductsWithResources() { 59 | return productsWithResources; 60 | } 61 | 62 | @Override 63 | public String getResource(Account account, Region region, Product product, String resourceId, String[] lineItem, long millisStart) { 64 | 65 | if (product == Product.ec2 || product == Product.ec2_instance || product == Product.ebs || product == Product.cloudwatch) { 66 | return getEc2Resource(account, region, resourceId, lineItem, millisStart); 67 | } 68 | else if (product == Product.rds) { 69 | return getRdsResource(account, region, resourceId, lineItem, millisStart); 70 | } 71 | else if (product == Product.s3) { 72 | return getS3Resource(account, region, resourceId, lineItem, millisStart); 73 | } 74 | else if (product == Product.eip) { 75 | return null; 76 | } 77 | else { 78 | return resourceId; 79 | } 80 | } 81 | 82 | protected String getEc2Resource(Account account, Region region, String resourceId, String[] lineItem, long millisStart) { 83 | String autoScalingGroupName = lineItem.length > config.lineItemProcessor.getUserTagStartIndex() ? 84 | lineItem[config.lineItemProcessor.getUserTagStartIndex()] : null; 85 | 86 | if (StringUtils.isEmpty(autoScalingGroupName)) { 87 | return UNKNOWN; 88 | } 89 | else if (resourceId.startsWith("i-")) { 90 | String appName = autoScalingGroupName.length() > 5 ? autoScalingGroupName.substring(0, 5) : autoScalingGroupName; 91 | instanceDb.SetResource(account, region, resourceId, appName, millisStart); 92 | return autoScalingGroupName; 93 | } 94 | else { 95 | return UNKNOWN; 96 | } 97 | } 98 | 99 | protected String getRdsResource(Account account, Region region, String resourceId, String[] lineItem, long millisStart) { 100 | if (resourceId.indexOf(":db:") > 0) 101 | return resourceId.substring(resourceId.indexOf(":db:") + 4); 102 | else 103 | return resourceId; 104 | } 105 | 106 | protected String getS3Resource(Account account, Region region, String resourceId, String[] lineItem, long millisStart) { 107 | return resourceId; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/java/sample.properties: -------------------------------------------------------------------------------- 1 | 2 | # whether or not to start processor 3 | ice.processor=true 4 | 5 | # whether or not to start reader/UI 6 | ice.reader=false 7 | 8 | # whether or not to start reservation capacity poller 9 | ice.reservationCapacityPoller=false 10 | 11 | # default reservation period, possible values are oneyear, threeyear 12 | ice.reservationPeriod=threeyear 13 | # default reservation utilization, possible values are LIGHT, MEDIUM, HEAVY. If you have both (LIGHT or MEDIUM) and HEAVY RIs, make sure you do not put HEAVY here. 14 | ice.reservationUtilization=MEDIUM 15 | 16 | # the highstock url 17 | ice.highstockUrl=https://code.highcharts.com/stock/4.2.1/highstock.js 18 | 19 | # url prefix, e.g. http://ice.netflix.com/. Will be used in alert emails. 20 | ice.urlPrefix= 21 | 22 | # from email address 23 | ice.fromEmail= 24 | 25 | # ec2 ondemand hourly cost threshold to send alert email. The alert email will be sent at most once per day. 26 | ice.ondemandCostAlertThreshold=250 27 | 28 | # ec2 ondemand hourly cost alert emails, separated by "," 29 | ice.ondemandCostAlertEmails= 30 | 31 | # modify the following 6 properties according to your billing files configuration. if you have multiple payer accounts, you will need to specify multiple values for each property. 32 | # s3 bucket name where the billing files are. multiple bucket names are delimited by ",". Ice must have read access to billing s3 bucket. 33 | ice.billing_s3bucketname=billing_s3bucketname1,billing_s3bucketname2 34 | # location for the billing bucket. It should be specified for buckets using v4 validation 35 | ice.billing_s3bucketregion=eu-west-1,eu-central-1 36 | # prefix of the billing files. multiple prefixes are delimited by "," 37 | ice.billing_s3bucketprefix=, 38 | # specify your payer account id here if across-accounts IAM role access is used. multiple account ids are delimited by ",". "ice.billing_payerAccountId=,222222222222" means assumed role access is only used for the second bucket. 39 | #ice.billing_payerAccountId=,123456789012 40 | # specify the assumed role name here if you use IAM role access to read from billing s3 bucket. multiple role names are delimited by ",". "ice.billing_accessRoleName=,ice" means assumed role access is only used for the second bucket. 41 | #ice.billing_accessRoleName=,ice 42 | # specify external id here if it is used. multiple external ids are delimited by ",". if you don't use external id, you can leave this property unset. 43 | #ice.billing_accessExternalId= 44 | 45 | # specify your custom tags here. Multiple tags are delimited by ",". If specified, BasicResourceService will be used to generate resource groups for you. 46 | # PLEASE MAKE SURE you have limited number (e.g. < 100) of unique value combinations from your custom tags, otherwise Ice performance will be greatly affected. 47 | #ice.customTags=tag1,tag2 48 | 49 | # start date in millis from when you want to start processing the billing files 50 | ice.startmillis=1364774400000 51 | 52 | # you company name. it will be used by UI 53 | ice.companyName=Your Company Name 54 | 55 | # s3 bucket name where Ice can store output files. Ice must have read and write access to billing s3 bucket. 56 | ice.work_s3bucketname=work_s3bucketname 57 | # prefix of Ice output files 58 | ice.work_s3bucketprefix=ice/ 59 | 60 | # local directory for Ice processor. the directory must exist. 61 | ice.processor.localDir=/mnt/ice_processor 62 | 63 | # local directory for Ice reader. the directory must exist. 64 | ice.reader.localDir=/mnt/ice_reader 65 | 66 | # monthly data cache size for Ice reader. 67 | ice.monthlycachesize=12 68 | 69 | # change the follow account settings 70 | ice.account.account1=123456789011 71 | ice.account.account2=123456789012 72 | ice.account.account3=123456789013 73 | ice.account.account4=123456789014 74 | ice.account.account5=123456789015 75 | ice.account.account6=123456789016 76 | 77 | # set reservation owner accounts. "ice.owneraccount.account2=account3,account4" means reservations in account2 can be shared by account3 and account4 78 | # if reservation capacity poller is enabled, the poller will try to poll reservation capacity through ec2 API (desribeReservedInstances) for each reservation owner account. 79 | ice.owneraccount.account1= 80 | ice.owneraccount.account2=account3,account4 81 | ice.owneraccount.account5=account6 82 | 83 | # if reservation capacity poller needs to use IAM role to access ec2 API, set the assumed role here for each reservation owner account 84 | ice.owneraccount.account1.role=ice 85 | ice.owneraccount.account2.role=ice 86 | ice.owneraccount.account5.role=ice 87 | 88 | # if reservation capacity poller needs to use IAM role to access ec2 API and external id is used, set the external id here for each reservation owner account. otherwise you can leave it unset. 89 | ice.owneraccount.account1.externalId= 90 | ice.owneraccount.account2.externalId= 91 | ice.owneraccount.account5.externalId= 92 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/MapDb.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.amazonaws.services.s3.AmazonS3Client; 21 | import com.amazonaws.services.s3.model.S3ObjectSummary; 22 | import com.netflix.ice.common.AwsUtils; 23 | import com.netflix.ice.processor.ProcessorConfig; 24 | import com.netflix.ice.tag.Account; 25 | import com.netflix.ice.tag.Region; 26 | import org.apache.commons.lang.StringUtils; 27 | import org.mapdb.DB; 28 | import org.mapdb.DBMaker; 29 | import org.slf4j.Logger; 30 | import org.slf4j.LoggerFactory; 31 | 32 | import java.io.File; 33 | import java.io.FilenameFilter; 34 | import java.util.Map; 35 | 36 | public class MapDb { 37 | private final Logger logger = LoggerFactory.getLogger(this.getClass()); 38 | private DB db; 39 | private Map items; 40 | private long numItemsToCommit = 0; 41 | private ProcessorConfig config; 42 | private String dbName; 43 | 44 | MapDb(String name) { 45 | this.config = ProcessorConfig.getInstance(); 46 | 47 | this.dbName = "db_" + name; 48 | File file = new File(config.localDir, dbName); 49 | if (!file.exists()) { 50 | AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); 51 | for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + this.dbName).getObjectSummaries()) { 52 | File dbFile = new File(config.localDir, s3ObjectSummary.getKey().substring(config.workS3BucketPrefix.length())); 53 | AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, dbFile); 54 | } 55 | } 56 | this.db = DBMaker.newFileDB(new File(config.localDir, this.dbName)).make(); 57 | try { 58 | this.items = db.createHashMap(name, false, null, null); 59 | } 60 | catch (IllegalArgumentException e) { 61 | this.items = db.getHashMap(name); 62 | logger.info("found " + this.items.size() + " items from mapdb for " + name); 63 | } 64 | } 65 | 66 | String getResource(Account account, Region region, String resourceId) { 67 | return this.items.get(resourceId + "|" + account + "|" + region); 68 | } 69 | 70 | void SetResource(Account account, Region region, String resourceId, String resource, long millisStart) { 71 | if (StringUtils.isEmpty(resource)) 72 | return; 73 | 74 | String key = resourceId + "|" + account + "|" + region; 75 | String resourceInDb = this.items.get(key); 76 | 77 | if (resourceInDb == null) { 78 | this.items.put(key, resource); 79 | 80 | numItemsToCommit ++; 81 | if (numItemsToCommit >= 1000) { 82 | this.commit(); 83 | numItemsToCommit = 0; 84 | } 85 | } 86 | else if (!resourceInDb.equals(resource)) { 87 | logger.error("different resources " + resourceInDb + " " + resource + " for " + resourceId); 88 | this.items.put(key, resource); 89 | resourceInDb = resource; 90 | 91 | numItemsToCommit ++; 92 | if (numItemsToCommit >= 1000) { 93 | this.commit(); 94 | numItemsToCommit = 0; 95 | } 96 | } 97 | } 98 | 99 | void commit() { 100 | this.db.commit(); 101 | upload(); 102 | logger.info("committed " + this.items.size() + "."); 103 | } 104 | 105 | void upload() { 106 | AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); 107 | 108 | File dir = new File(config.localDir); 109 | File[] files = dir.listFiles(new FilenameFilter() { 110 | public boolean accept(File file, String fileName) { 111 | return fileName.startsWith(dbName); 112 | } 113 | }); 114 | for (File file: files) 115 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + file.getName(), file); 116 | 117 | for (File file: files) 118 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "copy" + file.getName(), file); 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/TagGroupManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.netflix.ice.tag.*; 21 | import org.joda.time.Interval; 22 | 23 | import java.util.Collection; 24 | import java.util.Map; 25 | 26 | /** 27 | * Interface to manager tag groups. 28 | */ 29 | public interface TagGroupManager { 30 | 31 | /** 32 | * Get all accounts that meet query in tagLists. 33 | * @param tagLists 34 | * @return collection of accounts 35 | */ 36 | Collection getAccounts(TagLists tagLists); 37 | 38 | /** 39 | * Get all regions that meet query in tagLists. 40 | * @param tagLists 41 | * @return collection of regions 42 | */ 43 | Collection getRegions(TagLists tagLists); 44 | 45 | /** 46 | * Get all zones that meet query in tagLists. 47 | * @param tagLists 48 | * @return collection of zones 49 | */ 50 | Collection getZones(TagLists tagLists); 51 | 52 | /** 53 | * Get all products that meet query in tagLists. 54 | * @param tagLists 55 | * @return collection of products 56 | */ 57 | Collection getProducts(TagLists tagLists); 58 | 59 | /** 60 | * Get all operations that meet query in tagLists. 61 | * @param tagLists 62 | * @return collection of operations 63 | */ 64 | Collection getOperations(TagLists tagLists); 65 | 66 | /** 67 | * Get all usage types that meet query in tagLists. 68 | * @param tagLists 69 | * @return collection of usage types 70 | */ 71 | Collection getUsageTypes(TagLists tagLists); 72 | 73 | /** 74 | * Get all resource groups that meet query in tagLists. 75 | * @param tagLists 76 | * @return collection of resource groups 77 | */ 78 | Collection getResourceGroups(TagLists tagLists); 79 | 80 | /** 81 | * Get all accounts that meet query in tagLists and in specifed interval. 82 | * @param interval 83 | * @param tagLists 84 | * @return collection of accounts 85 | */ 86 | Collection getAccounts(Interval interval, TagLists tagLists); 87 | 88 | /** 89 | * Get all regions that meet query in tagLists and in specifed interval. 90 | * @param interval 91 | * @param tagLists 92 | * @return collection of regions 93 | */ 94 | Collection getRegions(Interval interval, TagLists tagLists); 95 | 96 | /** 97 | * Get all zones that meet query in tagLists and in specifed interval. 98 | * @param interval 99 | * @param tagLists 100 | * @return collection of zones 101 | */ 102 | Collection getZones(Interval interval, TagLists tagLists); 103 | 104 | /** 105 | * Get all products that meet query in tagLists and in specifed interval. 106 | * @param interval 107 | * @param tagLists 108 | * @return collection of products 109 | */ 110 | Collection getProducts(Interval interval, TagLists tagLists); 111 | 112 | /** 113 | * Get all operations that meet query in tagLists and in specifed interval. 114 | * @param interval 115 | * @param tagLists 116 | * @return collection of operations 117 | */ 118 | Collection getOperations(Interval interval, TagLists tagLists); 119 | 120 | /** 121 | * Get all usage types that meet query in tagLists and in specifed interval. 122 | * @param interval 123 | * @param tagLists 124 | * @return collection of usage types 125 | */ 126 | Collection getUsageTypes(Interval interval, TagLists tagLists); 127 | 128 | /** 129 | * Get all resource groups that meet query in tagLists and in specifed interval. 130 | * @param interval 131 | * @param tagLists 132 | * @return collection of resource groups 133 | */ 134 | Collection getResourceGroups(Interval interval, TagLists tagLists); 135 | 136 | /** 137 | * Get overlapping interval 138 | * @param interval 139 | * @return overlapping interval 140 | */ 141 | Interval getOverlapInterval(Interval interval); 142 | 143 | /** 144 | * Get map of tag lists based on group by. 145 | * @param interval 146 | * @param tagLists 147 | * @param groupBy 148 | * @param forReservation 149 | * @return 150 | */ 151 | Map getTagListsMap(Interval interval, TagLists tagLists, TagType groupBy, boolean forReservation); 152 | } 153 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/ReadWriteData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | import com.google.common.collect.Sets; 23 | import com.netflix.ice.common.TagGroup; 24 | 25 | import java.io.DataInput; 26 | import java.io.DataOutput; 27 | import java.io.IOException; 28 | import java.util.Collection; 29 | import java.util.List; 30 | import java.util.Map; 31 | import java.util.Set; 32 | 33 | public class ReadWriteData { 34 | private List> data; 35 | 36 | public ReadWriteData() { 37 | data = Lists.newArrayList(); 38 | } 39 | 40 | private ReadWriteData(List> data) { 41 | this.data = data; 42 | } 43 | 44 | public int getNum() { 45 | return data.size(); 46 | } 47 | 48 | void cutData(int num) { 49 | if (data.size() > num) 50 | data = data.subList(0, num); 51 | } 52 | 53 | public Map getData(int i) { 54 | return getCreateData(data, i); 55 | } 56 | 57 | void setData(List> newData, int startIndex, boolean merge) { 58 | for (int i = 0; i < newData.size(); i++) { 59 | int index = startIndex + i; 60 | 61 | if (index > data.size()) { 62 | getCreateData(data, index-1); 63 | } 64 | if (index >= data.size()) { 65 | data.add(newData.get(i)); 66 | } 67 | else { 68 | if (merge) { 69 | Map existed = data.get(index); 70 | for (Map.Entry entry: newData.get(i).entrySet()) { 71 | existed.put(entry.getKey(), entry.getValue()); 72 | } 73 | } 74 | else { 75 | data.set(index, newData.get(i)); 76 | } 77 | } 78 | } 79 | } 80 | 81 | static Map getCreateData(List> data, int i) { 82 | if (i >= data.size()) { 83 | for (int j = data.size(); j <= i; j++) { 84 | data.add(Maps.newHashMap()); 85 | } 86 | } 87 | return data.get(i); 88 | } 89 | 90 | public Collection getTagGroups() { 91 | Set keys = Sets.newTreeSet(); 92 | 93 | for (Map map: data) { 94 | keys.addAll(map.keySet()); 95 | } 96 | 97 | return keys; 98 | } 99 | 100 | public static class Serializer { 101 | public static void serialize(DataOutput out, ReadWriteData data) throws IOException { 102 | 103 | Collection keys = data.getTagGroups(); 104 | out.writeInt(keys.size()); 105 | for (TagGroup tagGroup: keys) { 106 | TagGroup.Serializer.serialize(out, tagGroup); 107 | } 108 | 109 | out.writeInt(data.data.size()); 110 | for (int i = 0; i < data.data.size(); i++) { 111 | Map map = data.getData(i); 112 | out.writeBoolean(map.size() > 0); 113 | if (map.size() > 0) { 114 | for (TagGroup tagGroup: keys) { 115 | Double v = map.get(tagGroup); 116 | out.writeDouble(v == null ? 0 : v); 117 | } 118 | } 119 | } 120 | } 121 | 122 | public static ReadWriteData deserialize(DataInput in) throws IOException { 123 | 124 | int numKeys = in.readInt(); 125 | List keys = Lists.newArrayList(); 126 | for (int j = 0; j < numKeys; j++) { 127 | keys.add(TagGroup.Serializer.deserialize(ProcessorConfig.getInstance(), in)); 128 | } 129 | 130 | List> data = Lists.newArrayList(); 131 | int num = in.readInt(); 132 | for (int i = 0; i < num; i++) { 133 | Map map = Maps.newHashMap(); 134 | boolean hasData = in.readBoolean(); 135 | if (hasData) { 136 | for (int j = 0; j < keys.size(); j++) { 137 | double v = in.readDouble(); 138 | if (v != 0) { 139 | map.put(keys.get(j), v); 140 | } 141 | } 142 | } 143 | data.add(map); 144 | } 145 | 146 | return new ReadWriteData(data); 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/tag/Region.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.tag; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | 23 | import java.util.List; 24 | import java.util.concurrent.ConcurrentMap; 25 | 26 | public class Region extends Tag { 27 | 28 | public static final Region US_EAST_1 = new Region("us-east-1", "USE1"); 29 | public static final Region US_EAST_2 = new Region("us-east-2", "USE2"); 30 | public static final Region US_WEST_1 = new Region("us-west-1", "USW1"); 31 | public static final Region US_WEST_2 = new Region("us-west-2", "USW2"); 32 | public static final Region US_GOV_WEST_1 = new Region("us-gov-west-1", "UGW1"); 33 | public static final Region EU_WEST_1 = new Region("eu-west-1", "EU"); 34 | public static final Region EU_WEST_2 = new Region("eu-west-2", "EUW2"); 35 | public static final Region EU_CENTRAL_1 = new Region("eu-central-1", "EUC1"); 36 | public static final Region AP_SOUTH_1 = new Region("ap-south-1", "APS3"); 37 | public static final Region AP_SOUTHEAST_1 = new Region("ap-southeast-1", "APS1"); 38 | public static final Region AP_SOUTHEAST_2 = new Region("ap-southeast-2", "APS2"); 39 | public static final Region AP_NORTHEAST_1 = new Region("ap-northeast-1","APN1"); 40 | public static final Region AP_NORTHEAST_2 = new Region("ap-northeast-2","APN2"); 41 | public static final Region SA_EAST_1 = new Region("sa-east-1", "SAE1"); 42 | public static final Region CA_CENTRAL_1 = new Region("ca-central-1", "CAC1"); 43 | 44 | private static ConcurrentMap regionsByName = Maps.newConcurrentMap(); 45 | private static ConcurrentMap regionsByShortName = Maps.newConcurrentMap(); 46 | 47 | static { 48 | regionsByShortName.put(US_EAST_1.shortName, US_EAST_1); 49 | regionsByShortName.put(US_EAST_2.shortName, US_EAST_2); 50 | regionsByShortName.put(US_WEST_1.shortName, US_WEST_1); 51 | regionsByShortName.put(US_WEST_2.shortName, US_WEST_2); 52 | regionsByShortName.put(US_GOV_WEST_1.shortName, US_GOV_WEST_1); 53 | regionsByShortName.put(EU_WEST_1.shortName, EU_WEST_1); 54 | regionsByShortName.put(EU_WEST_2.shortName, EU_WEST_2); 55 | regionsByShortName.put(EU_CENTRAL_1.shortName, EU_CENTRAL_1); 56 | regionsByShortName.put(AP_SOUTH_1.shortName, AP_SOUTH_1); 57 | regionsByShortName.put(AP_SOUTHEAST_1.shortName, AP_SOUTHEAST_1); 58 | regionsByShortName.put(AP_SOUTHEAST_2.shortName, AP_SOUTHEAST_2); 59 | regionsByShortName.put(AP_NORTHEAST_1.shortName, AP_NORTHEAST_1); 60 | regionsByShortName.put(AP_NORTHEAST_2.shortName, AP_NORTHEAST_2); 61 | regionsByShortName.put(SA_EAST_1.shortName, SA_EAST_1); 62 | regionsByShortName.put(CA_CENTRAL_1.shortName, CA_CENTRAL_1); 63 | 64 | regionsByName.put(US_EAST_1.name, US_EAST_1); 65 | regionsByName.put(US_EAST_2.name, US_EAST_2); 66 | regionsByName.put(US_WEST_1.name, US_WEST_1); 67 | regionsByName.put(US_WEST_2.name, US_WEST_2); 68 | regionsByName.put(US_GOV_WEST_1.name, US_GOV_WEST_1); 69 | regionsByName.put(EU_WEST_1.name, EU_WEST_1); 70 | regionsByName.put(EU_WEST_2.name, EU_WEST_2); 71 | regionsByName.put(EU_CENTRAL_1.name, EU_CENTRAL_1); 72 | regionsByName.put(AP_SOUTH_1.name, AP_SOUTH_1); 73 | regionsByName.put(AP_SOUTHEAST_1.name, AP_SOUTHEAST_1); 74 | regionsByName.put(AP_SOUTHEAST_2.name, AP_SOUTHEAST_2); 75 | regionsByName.put(AP_NORTHEAST_1.name, AP_NORTHEAST_1); 76 | regionsByName.put(AP_NORTHEAST_2.name, AP_NORTHEAST_2); 77 | regionsByName.put(SA_EAST_1.name, SA_EAST_1); 78 | regionsByName.put(CA_CENTRAL_1.name, CA_CENTRAL_1); 79 | } 80 | 81 | public final String shortName; 82 | List zones = Lists.newArrayList(); 83 | 84 | private Region(String name, String shortName) { 85 | super(name); 86 | this.shortName = shortName; 87 | } 88 | 89 | public List getZones() { 90 | return Lists.newArrayList(zones); 91 | } 92 | 93 | void addZone(Zone zone) { 94 | zones.add(zone); 95 | } 96 | 97 | public static Region getRegionByShortName(String shortName) { 98 | return regionsByShortName.get(shortName); 99 | } 100 | 101 | public static Region getRegionByName(String name) { 102 | return regionsByName.get(name); 103 | } 104 | 105 | public static List getRegions(List names) { 106 | List result = Lists.newArrayList(); 107 | for (String name: names) 108 | result.add(regionsByName.get(name)); 109 | return result; 110 | } 111 | 112 | public static List getAllRegions() { 113 | return Lists.newArrayList(regionsByName.values()); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /grails-app/conf/BuildConfig.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import org.apache.ivy.plugins.resolver.FileSystemResolver 17 | import org.apache.ivy.plugins.resolver.URLResolver 18 | 19 | grails.project.work.dir = 'work' 20 | grails.project.class.dir = 'target/classes' 21 | grails.project.test.class.dir = 'target/test-classes' 22 | grails.project.test.reports.dir = 'target/test-reports' 23 | grails.project.war.file = "target/${appName}.war" 24 | 25 | grails.project.dependency.resolver = "maven" 26 | 27 | grails.project.dependency.resolution = { 28 | // inherit Grails' default dependencies 29 | inherits("global") { 30 | } 31 | log "warn" // log level of Ivy resolver, either 'error', 'warn', 'info', 'debug' or 'verbose' 32 | repositories { 33 | grailsPlugins() 34 | grailsHome() 35 | grailsCentral() 36 | mavenCentral() 37 | 38 | // Optional custom repository for dependencies. 39 | Closure internalRepo = { 40 | String repoUrl = 'http://artifacts/ext-releases-local' 41 | String artifactPattern = '[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier]).[ext]' 42 | String ivyPattern = '[organisation]/[module]/[revision]/[module]-[revision]-ivy.[ext]' 43 | URLResolver urlLibResolver = new URLResolver() 44 | urlLibResolver.with { 45 | name = repoUrl 46 | addArtifactPattern("${repoUrl}/${artifactPattern}") 47 | addIvyPattern("${repoUrl}/${ivyPattern}") 48 | m2compatible = true 49 | } 50 | resolver urlLibResolver 51 | 52 | String localDir = System.getenv('IVY_LOCAL_REPO') ?: "${System.getProperty('user.home')}/ivy2-local" 53 | FileSystemResolver localLibResolver = new FileSystemResolver() 54 | localLibResolver.with { 55 | name = localDir 56 | addArtifactPattern("${localDir}/${artifactPattern}") 57 | addIvyPattern("${localDir}/${ivyPattern}") 58 | } 59 | resolver localLibResolver 60 | } 61 | // Comment or uncomment the next line to toggle the use of an internal artifacts repository. 62 | //internalRepo() 63 | } 64 | 65 | dependencies { 66 | 67 | compile( 68 | // Amazon Web Services programmatic interface 69 | 'com.amazonaws:aws-java-sdk:1.11.136', 70 | // Transitive dependencies of aws-java-sdk, but also used directly. 71 | // It would be great if we could upgrade httpcore and httpclient, but we can't until the AWS Java SDK 72 | // upgrades its dependencies. If we simply upgrade these, then some Amazon calls fail. 73 | 'org.apache.httpcomponents:httpcore:4.4.4', 74 | 'org.apache.httpcomponents:httpclient:4.5.2', 75 | 76 | // Explicitly including aws-java-sdk transitive dependencies 77 | 'org.codehaus.jackson:jackson-core-asl:1.8.9', 78 | 'org.codehaus.jackson:jackson-mapper-asl:1.8.9', 79 | 80 | // Extra collection types and utilities 81 | 'commons-collections:commons-collections:3.2.1', 82 | 83 | // Easier Java from of the Apache Foundation 84 | 'commons-lang:commons-lang:2.4', 85 | 86 | // Better Zip Support 87 | 'org.apache.commons:commons-compress:1.8', 88 | 89 | // Better IO Support 90 | 'commons-io:commons-io:2.4', 91 | 92 | // Easier Java from Joshua Bloch and Google 93 | 'com.google.guava:guava:14.0', 94 | 95 | // Send emails about system errors and task completions 96 | 'javax.mail:mail:1.4.1', 97 | 98 | // Better date API 99 | 'joda-time:joda-time:2.0', 100 | 101 | 'net.sourceforge.javacsv:javacsv:2.0', 102 | 103 | 'org.apache.poi:poi-ooxml:3.7', 104 | 'org.codehaus.woodstox:wstx-asl:3.2.9', 105 | 'jfree:jfreechart:1.0.13', 106 | 'org.json:json:20090211', 107 | 'org.mapdb:mapdb:0.9.1', 108 | 109 | // Since the AWS SDK has removed its package "com.amazonaws.util.json", we need to include it as a separate library 110 | 'org.json:json:20090211' 111 | 112 | ) { // Exclude superfluous and dangerous transitive dependencies 113 | excludes( 114 | // Some libraries bring older versions of JUnit as a transitive dependency and that can interfere 115 | // with Grails' built in JUnit 116 | 'junit', 117 | 118 | 'mockito-core', 119 | ) 120 | } 121 | } 122 | 123 | plugins { 124 | //runtime ":hibernate4:4.3.6.1" 125 | build ":tomcat:8.0.20" 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /grails-app/views/dashboard/summary.gsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | 3 | Copyright 2013 Netflix, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | --%> 18 | 19 | <%@ page contentType="text/html;charset=UTF-8" %> 20 | 21 | 22 | 23 | AWS Usage Summary 24 | 25 | 26 |
27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 41 | 45 | 49 | 53 | 57 | 61 | 62 |
Group ByAccountRegionProductOperationUsageType
39 | 40 | 42 | 43 |
44 | 		46 | 47 |
48 | 		50 | 51 |
52 | 		54 | 55 |
56 | 		58 | 59 |
60 |
63 |
64 | 65 | Submit 68 |
69 | 70 |
71 |
72 |
73 |
74 | SHOW ALL 75 | HIDE ALL 76 | 77 |
78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 95 | 96 | 97 |
{{legendName}}{{monthFormat(month)}}
{{header.name}}
{{legend.name}}		 92 | {{currencySign}} {{data[legend.name][header.index] | number:2}} 93 | {{currencySign}} {{data[legend.name][header.index] | number:2}} 94 |
98 |
99 | 100 |
101 | 102 | 103 | -------------------------------------------------------------------------------- /grails-app/views/layouts/main.gsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | 3 | Copyright 2013 Netflix, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | --%> 18 | 19 | <%@ page import="com.netflix.ice.reader.ReaderConfig" %> 20 | 21 | 22 | 23 | 24 | <g:layoutTitle default="${ReaderConfig.getInstance().companyName} AWS Usage Dashboard"/> 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |
33 |
34 | 35 | 36 | ${ReaderConfig.getInstance().companyName} AWS Usage Dashboard 37 | 38 | 39 |
40 |
41 | 72 |
73 | 74 | 76 | 77 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicS3ApplicationGroupService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.amazonaws.services.s3.AmazonS3Client; 21 | import com.amazonaws.services.s3.model.ObjectMetadata; 22 | import com.google.common.collect.Maps; 23 | import com.netflix.ice.common.AwsUtils; 24 | import com.netflix.ice.reader.ApplicationGroup; 25 | import com.netflix.ice.reader.ApplicationGroupService; 26 | import com.netflix.ice.reader.ReaderConfig; 27 | import org.apache.commons.io.IOUtils; 28 | import org.json.JSONException; 29 | import org.json.JSONObject; 30 | import org.json.JSONTokener; 31 | import org.slf4j.Logger; 32 | import org.slf4j.LoggerFactory; 33 | 34 | import java.io.ByteArrayInputStream; 35 | import java.io.InputStream; 36 | import java.util.Iterator; 37 | import java.util.Map; 38 | 39 | public class BasicS3ApplicationGroupService implements ApplicationGroupService { 40 | private final static Logger logger = LoggerFactory.getLogger(BasicS3ApplicationGroupService.class); 41 | private AmazonS3Client s3Client; 42 | private ReaderConfig config; 43 | 44 | public void init() { 45 | this.config = ReaderConfig.getInstance(); 46 | s3Client = AwsUtils.getAmazonS3Client(); 47 | } 48 | 49 | private String getJson(Map appgroups) throws JSONException { 50 | JSONObject json = new JSONObject(); 51 | for (String name: appgroups.keySet()) { 52 | ApplicationGroup appgroup = appgroups.get(name); 53 | json.put(name, appgroup.getJSON()); 54 | } 55 | 56 | return json.toString(); 57 | } 58 | 59 | public Map getApplicationGroups() { 60 | String jsonStr; 61 | try { 62 | InputStream in = s3Client.getObject(config.workS3BucketName, config.workS3BucketPrefix + "appgroups").getObjectContent(); 63 | jsonStr = IOUtils.toString(in); 64 | in.close(); 65 | } 66 | catch (Exception e) { 67 | logger.error("Error reading from appgroups file", e); 68 | try { 69 | InputStream in = s3Client.getObject(config.workS3BucketName, config.workS3BucketPrefix + "copy_appgroups").getObjectContent(); 70 | jsonStr = IOUtils.toString(in); 71 | in.close(); 72 | } 73 | catch (Exception r) { 74 | logger.error("Error reading from copy_appgroups file", r); 75 | return Maps.newHashMap(); 76 | } 77 | } 78 | 79 | try { 80 | JSONObject json = new JSONObject(new JSONTokener(jsonStr)); 81 | Map appgroups = Maps.newHashMap(); 82 | Iterator keys = json.keys(); 83 | while (keys.hasNext()) { 84 | String key = keys.next(); 85 | String str = json.getString(key); 86 | appgroups.put(key, new ApplicationGroup(str)); 87 | } 88 | 89 | return appgroups; 90 | } 91 | catch (JSONException e) { 92 | logger.error("Error reading appgroups from json...", e); 93 | return Maps.newHashMap(); 94 | } 95 | } 96 | 97 | public ApplicationGroup getApplicationGroup(String name) { 98 | Map appgroups = getApplicationGroups(); 99 | return appgroups.get(name); 100 | } 101 | 102 | public boolean saveApplicationGroup(ApplicationGroup appgroup) { 103 | Map appgroups = getApplicationGroups(); 104 | appgroups.put(appgroup.name, appgroup); 105 | 106 | try { 107 | String json = getJson(appgroups); 108 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "appgroups", IOUtils.toInputStream(json), new ObjectMetadata()); 109 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "copy_appgroups", IOUtils.toInputStream(json), new ObjectMetadata()); 110 | 111 | BasicS3ApplicationGroupService.logger.info("saved appgroup " + appgroup); 112 | return true; 113 | } 114 | catch (JSONException e) { 115 | logger.error("Error saving appgroup " + appgroup, e); 116 | return false; 117 | } 118 | } 119 | 120 | public boolean deleteApplicationGroup(String name) { 121 | Map appgroups = getApplicationGroups(); 122 | ApplicationGroup appgroup = appgroups.remove(name); 123 | 124 | try { 125 | String json = getJson(appgroups); 126 | s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + "appgroups", new ByteArrayInputStream(json.getBytes()), new ObjectMetadata()); 127 | 128 | BasicS3ApplicationGroupService.logger.info("delete appgroup " + name + " " + appgroup); 129 | return true; 130 | } 131 | catch (JSONException e) { 132 | logger.error("Error deleting appgroup " + appgroup, e); 133 | return false; 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicProductService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.google.common.collect.Lists; 21 | import com.google.common.collect.Maps; 22 | import com.netflix.ice.common.ProductService; 23 | import com.netflix.ice.tag.Product; 24 | 25 | import java.util.Collection; 26 | import java.util.List; 27 | import java.util.concurrent.ConcurrentMap; 28 | 29 | public class BasicProductService implements ProductService { 30 | 31 | private static ConcurrentMap productsByAwsName = Maps.newConcurrentMap(); 32 | private static ConcurrentMap productsByName = Maps.newConcurrentMap(); 33 | 34 | public static final BasicProduct cloudfront = new BasicProduct(Product.cloudfront, "CloudFront"); 35 | public static final BasicProduct cloudhsm = new BasicProduct(Product.cloudhsm, "CloudHSM"); 36 | public static final BasicProduct cloudwatch = new BasicProduct(Product.cloudwatch, "CloudWatch"); 37 | public static final BasicProduct data_pipeline = new BasicProduct(Product.data_pipeline, "Data Pipeline"); 38 | public static final BasicProduct data_transfer = new BasicProduct(Product.data_transfer, "Data Transfer"); 39 | public static final BasicProduct direct_connect = new BasicProduct(Product.direct_connect, "Direct Connect"); 40 | public static final BasicProduct dynamodb = new BasicProduct(Product.dynamodb, "DynamoDB"); 41 | public static final BasicProduct ebs = new BasicProduct(Product.ebs, "ebs"); 42 | public static final BasicProduct ec2 = new BasicProduct(Product.ec2, "Elastic Compute Cloud"); 43 | public static final BasicProduct ec2_instance = new BasicProduct(Product.ec2_instance, "ec2_instance"); 44 | public static final BasicProduct eip = new BasicProduct(Product.eip, "eip"); 45 | public static final BasicProduct elasticache = new BasicProduct(Product.elasticache, "ElastiCache"); 46 | public static final BasicProduct emr = new BasicProduct(Product.emr, "Elastic MapReduce"); 47 | public static final BasicProduct glacier = new BasicProduct(Product.glacier, "Glacier"); 48 | public static final BasicProduct rds = new BasicProduct(Product.rds, "RDS Service"); 49 | public static final BasicProduct redshift = new BasicProduct(Product.redshift, "Redshift"); 50 | public static final BasicProduct route53 = new BasicProduct(Product.route53, "Route 53"); 51 | public static final BasicProduct s3 = new BasicProduct(Product.s3, "Simple Storage Service"); 52 | public static final BasicProduct simpledb = new BasicProduct(Product.simpledb, "SimpleDB"); 53 | public static final BasicProduct ses = new BasicProduct(Product.ses, "Simple Email Service"); 54 | public static final BasicProduct sns = new BasicProduct(Product.sns, "Simple Notification Service"); 55 | public static final BasicProduct sqs = new BasicProduct(Product.sqs, "Simple Queue Service"); 56 | public static final BasicProduct storage_gateway = new BasicProduct(Product.storage_gateway, "Storage Gateway"); 57 | public static final BasicProduct sws = new BasicProduct(Product.sws, "Simple Workflow Service"); 58 | public static final BasicProduct vpc = new BasicProduct(Product.vpc, "Virtual Private Cloud"); 59 | public static final BasicProduct monitor = new BasicProduct(Product.monitor, "monitor"); 60 | 61 | private static BasicProduct[] products = new BasicProduct[]{cloudfront, cloudhsm, cloudwatch, data_pipeline, data_transfer, direct_connect, dynamodb, ebs, ec2, ec2_instance, eip, elasticache, emr, glacier, rds, redshift, route53, s3, simpledb, ses, sns, sqs, storage_gateway, sws, vpc}; 62 | 63 | static { 64 | for (BasicProduct product: products) { 65 | productsByAwsName.put("AWS " + product.awsName, product.product); 66 | productsByAwsName.put("Amazon " + product.awsName, product.product); 67 | productsByName.put(product.product.name, product.product); 68 | } 69 | productsByAwsName.put(monitor.awsName, monitor.product); 70 | productsByName.put(monitor.product.name, monitor.product); 71 | } 72 | 73 | public Product getProductByAwsName(String awsName) { 74 | Product product = productsByAwsName.get(awsName); 75 | if (product == null) { 76 | product = new Product(awsName); 77 | productsByAwsName.put(awsName, product); 78 | productsByName.put(awsName, product); 79 | } 80 | return product; 81 | } 82 | 83 | public Product getProductByName(String name) { 84 | Product product = productsByName.get(name); 85 | if (product == null) { 86 | product = new Product(name); 87 | productsByAwsName.put(name, product); 88 | productsByName.put(name, product); 89 | } 90 | return product; 91 | } 92 | 93 | public List getProducts(List names) { 94 | List result = Lists.newArrayList(); 95 | for (String name: names) 96 | result.add(productsByName.get(name)); 97 | return result; 98 | } 99 | 100 | public Collection getProducts() { 101 | return productsByName.values(); 102 | } 103 | 104 | private static class BasicProduct { 105 | private final String awsName; 106 | private final Product product; 107 | BasicProduct(Product product, String awsName) { 108 | this.product = product; 109 | this.awsName = awsName; 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/processor/ProcessorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.processor; 19 | 20 | import com.amazonaws.auth.AWSCredentialsProvider; 21 | import com.netflix.ice.common.*; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | import java.util.Properties; 26 | 27 | public class ProcessorConfig extends Config { 28 | private static final Logger logger = LoggerFactory.getLogger(ProcessorConfig.class); 29 | private static ProcessorConfig instance; 30 | private static ReservationCapacityPoller reservationCapacityPoller; 31 | private static BillingFileProcessor billingFileProcessor; 32 | 33 | public final String[] billingAccountIds; 34 | public final String[] billingS3BucketNames; 35 | public final String[] billingS3BucketRegions; 36 | public final String[] billingS3BucketPrefixes; 37 | public final String[] billingAccessRoleNames; 38 | public final String[] billingAccessExternalIds; 39 | 40 | public final String[] customTags; 41 | public final ReservationService reservationService; 42 | public final LineItemProcessor lineItemProcessor; 43 | public final Randomizer randomizer; 44 | public final double costPerMonitorMetricPerHour; 45 | public final boolean useBlended; 46 | 47 | public final String useCostForResourceGroup; 48 | 49 | /** 50 | * 51 | * @param properties (required) 52 | * @param accountService (required) 53 | * @param productService (required) 54 | * @param reservationService (required) 55 | * @param resourceService (optional) 56 | * @param lineItemProcessor (required) 57 | * @param randomizer (optional) 58 | */ 59 | public ProcessorConfig( 60 | Properties properties, 61 | AWSCredentialsProvider credentialsProvider, 62 | AccountService accountService, 63 | ProductService productService, 64 | ReservationService reservationService, 65 | ResourceService resourceService, 66 | LineItemProcessor lineItemProcessor, 67 | Randomizer randomizer) { 68 | 69 | super(properties, credentialsProvider, accountService, productService, resourceService); 70 | 71 | if (reservationService == null) throw new IllegalArgumentException("reservationService must be specified"); 72 | if (lineItemProcessor == null) throw new IllegalArgumentException("lineItemProcessor must be specified"); 73 | 74 | this.reservationService = reservationService; 75 | this.lineItemProcessor = lineItemProcessor; 76 | this.randomizer = randomizer; 77 | 78 | if (properties.getProperty(IceOptions.COST_PER_MONITORMETRIC_PER_HOUR) != null) 79 | this.costPerMonitorMetricPerHour = Double.parseDouble(properties.getProperty(IceOptions.COST_PER_MONITORMETRIC_PER_HOUR)); 80 | else 81 | this.costPerMonitorMetricPerHour = 0; 82 | 83 | billingS3BucketNames = properties.getProperty(IceOptions.BILLING_S3_BUCKET_NAME).split(","); 84 | billingS3BucketRegions = properties.getProperty(IceOptions.BILLING_S3_BUCKET_REGION).split(","); 85 | billingS3BucketPrefixes = properties.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, "").split(","); 86 | billingAccountIds = properties.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, "").split(","); 87 | billingAccessRoleNames = properties.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, "").split(","); 88 | billingAccessExternalIds = properties.getProperty(IceOptions.BILLING_ACCESS_EXTERNALID, "").split(","); 89 | 90 | customTags = properties.getProperty(IceOptions.CUSTOM_TAGS, "").split(","); 91 | useBlended = properties.getProperty(IceOptions.USE_BLENDED) == null ? false : Boolean.parseBoolean(properties.getProperty(IceOptions.USE_BLENDED)); 92 | 93 | useCostForResourceGroup = properties.getProperty(IceOptions.RESOURCE_GROUP_COST, "modeled"); 94 | 95 | ProcessorConfig.instance = this; 96 | 97 | reservationService.init(); 98 | if (resourceService != null) 99 | resourceService.init(); 100 | 101 | billingFileProcessor = new BillingFileProcessor( 102 | properties.getProperty(IceOptions.URL_PREFIX), 103 | properties.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD) == null ? null : Double.parseDouble(properties.getProperty(IceOptions.ONDEMAND_COST_ALERT_THRESHOLD)), 104 | properties.getProperty(IceOptions.FROM_EMAIL), 105 | properties.getProperty(IceOptions.ONDEMAND_COST_ALERT_EMAILS)); 106 | } 107 | 108 | public void start (ReservationCapacityPoller reservationCapacityPoller) { 109 | logger.info("starting up..."); 110 | 111 | ProcessorConfig.reservationCapacityPoller = reservationCapacityPoller; 112 | if (reservationCapacityPoller != null) 113 | reservationCapacityPoller.start(); 114 | 115 | while (reservationCapacityPoller != null && !reservationCapacityPoller.updatedConfig()) { 116 | try { 117 | Thread.sleep(10000L); 118 | } 119 | catch (InterruptedException e) { 120 | } 121 | } 122 | 123 | billingFileProcessor.start(300); 124 | } 125 | 126 | public void shutdown() { 127 | logger.info("Shutting down..."); 128 | 129 | billingFileProcessor.shutdown(); 130 | reservationCapacityPoller.shutdown(); 131 | } 132 | 133 | /** 134 | * 135 | * @return singlton instance 136 | */ 137 | public static ProcessorConfig getInstance() { 138 | return instance; 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/EddaResourceService.java: -------------------------------------------------------------------------------- 1 | package com.netflix.ice.basic; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.net.MalformedURLException; 6 | import java.net.URL; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Properties; 10 | 11 | import org.apache.commons.io.IOUtils; 12 | import org.apache.commons.lang.StringUtils; 13 | import org.json.JSONArray; 14 | import org.json.JSONException; 15 | import org.json.JSONObject; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | import com.google.common.collect.Lists; 20 | import com.netflix.ice.common.ResourceService; 21 | import com.netflix.ice.tag.Account; 22 | import com.netflix.ice.tag.Product; 23 | import com.netflix.ice.tag.Region; 24 | 25 | 26 | /** 27 | * A ResourceService which queries an https://github.com/Netflix/edda instance for the 'Usage' tag of instances for breaking down 28 | * costs based on this tag. 29 | * 30 | * Recognizes configuration values "ice.eddaresourceservice.url" and "ice.eddaresourceservice.tag", i.e. 31 | * 32 | # Settings for our own Resource-Service 33 | ice.eddaresourceservice.url=http://172.16.110.80:8080 34 | ice.eddaresourceservice.tag=Usage 35 | 36 | * Note: You will need to register the service in Bootstrap.groovy when ProcessorConfig and ReaderConfig are instantiated. 37 | * 38 | * TODO: There is currently no caching done, so there might be a lot of requests fired off to Edda! 39 | */ 40 | public class EddaResourceService extends ResourceService { 41 | private static final ArrayList EC2_PRODUCTS = Lists.newArrayList(Product.ec2, Product.ec2_instance, Product.ebs); 42 | //private static final ArrayList RDS_PRODUCTS = Lists.newArrayList(Product.rds); 43 | //private static final ArrayList S3_PRODUCTS = Lists.newArrayList(Product.s3); 44 | 45 | private final static Logger logger = LoggerFactory.getLogger(EddaResourceService.class); 46 | 47 | @SuppressWarnings("unchecked") 48 | private static List> productsWithResources = 49 | Lists.>newArrayList(EC2_PRODUCTS/*, RDS_PRODUCTS*//*, S3_PRODUCTS*/); 50 | 51 | // read from properties 52 | protected String EDDA_ROOT_URL; 53 | protected String EDDA_TAG_NAME; 54 | 55 | //private final Properties prop; 56 | 57 | public EddaResourceService(Properties prop) { 58 | super(); 59 | //this.prop = prop; 60 | 61 | EDDA_ROOT_URL = prop.getProperty("ice.eddaresourceservice.url", "http://localhost:18081/edda/api/v2/"); 62 | EDDA_TAG_NAME = prop.getProperty("ice.eddaresourceservice.tag", "Usage"); 63 | } 64 | 65 | /* (non-Javadoc) 66 | * @see com.netflix.ice.common.ResourceService#init() 67 | */ 68 | @Override 69 | public void init() { 70 | logger.info("Initializing..."); 71 | } 72 | 73 | 74 | @Override 75 | public String getResource(Account account, Region region, Product product, String resourceId, String[] lineItem, 76 | long millisStart) { 77 | // currently we support ec2 78 | if(Product.ec2.equals(product) || Product.ec2_instance.equals(product)) { 79 | if(StringUtils.isEmpty(resourceId)) { 80 | logger.warn("Had empty resourceId"); 81 | return "Error"; 82 | } 83 | 84 | try { 85 | JSONArray instances = readInstanceArray(); 86 | boolean found = false; 87 | for(int i = 0;i < instances.length();i++) { 88 | String instance = instances.getString(i); 89 | if(resourceId.equals(instance)) { 90 | found = true; 91 | break; 92 | } 93 | } 94 | if(!found) { 95 | logger.warn("Did not find resourceId in edda: " + resourceId); 96 | return "Unknown"; 97 | } 98 | 99 | InputStream stream = new URL(EDDA_ROOT_URL + "view/instances/" + resourceId).openStream(); 100 | final String json; 101 | try { 102 | json = IOUtils.toString(stream); 103 | } finally { 104 | stream.close(); 105 | } 106 | 107 | JSONObject object = new JSONObject(json); 108 | JSONArray tags = object.getJSONArray("tags"); 109 | for(int i = 0;i < tags.length();i++) { 110 | JSONObject tag = tags.getJSONObject(i); 111 | String key = tag.getString("key"); 112 | if(key.equals(EDDA_TAG_NAME)) { 113 | String usage = tag.getString("value"); 114 | logger.debug("Found usage: " + usage + " for resource " + resourceId); 115 | return usage; 116 | } 117 | } 118 | 119 | logger.debug("Did not find tag 'Usage' for resource " + resourceId); 120 | return "Unknown"; 121 | } catch (JSONException e) { 122 | logger.warn("error parsing json", e); 123 | return "Error"; 124 | } catch (MalformedURLException e) { 125 | logger.warn("error parsing url", e); 126 | return "Error"; 127 | } catch (IOException e) { 128 | logger.warn("error fetching data from edda at " + EDDA_ROOT_URL, e); 129 | return "Error"; 130 | } 131 | } 132 | 133 | logger.debug("Product: " + product + " not handled, resourceId: " + resourceId); 134 | //logger.info("get resource for account " + account + " region " + region + " product " + product + " resource: " + resourceId + " lineItem: " + Arrays.toString(lineItem)); 135 | return super.getResource(account, region, product, resourceId, lineItem, millisStart); 136 | } 137 | 138 | 139 | /* (non-Javadoc) 140 | * @see com.netflix.ice.common.ResourceService#getProductsWithResources() 141 | */ 142 | @Override 143 | public List> getProductsWithResources() { 144 | logger.info("Register for products: " + productsWithResources + "..."); 145 | return productsWithResources; 146 | } 147 | 148 | /* (non-Javadoc) 149 | * @see com.netflix.ice.common.ResourceService#commit() 150 | */ 151 | @Override 152 | public void commit() { 153 | logger.info("Commit..."); 154 | } 155 | 156 | protected JSONArray readInstanceArray() throws IOException, MalformedURLException, JSONException { 157 | InputStream stream = new URL(EDDA_ROOT_URL + "view/instances").openStream(); 158 | final String json; 159 | try { 160 | json = IOUtils.toString(stream); 161 | } finally { 162 | stream.close(); 163 | } 164 | JSONArray instances = new JSONArray(json); 165 | return instances; 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /grails-app/views/dashboard/breakdown.gsp: -------------------------------------------------------------------------------- 1 | <%-- 2 | 3 | Copyright 2013 Netflix, Inc. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | --%> 18 | 19 | <%@ page contentType="text/html;charset=UTF-8" %> 20 | 21 | 22 | 23 | Aws Usage Breakdown 24 | 25 | 26 |
27 |
{{message}}
28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 50 | 54 | 58 | 62 | 66 | 70 | 74 | 75 | 76 |
ShowAccountRegionProductOperationUsageType
41 |
End:
42 |
# of Spans:
43 |
Aggregate:
49 | 		51 |
52 | 53 | 		55 | 56 |
57 | 		59 | 60 |
61 | 		63 | 64 |
65 | 		67 | 68 |
69 | 		71 | 72 |
73 |
77 | 78 |
79 | 80 | Submit 83 |
84 | 85 |
86 |
87 | 88 |
89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 106 | 109 | 113 | 116 | 117 | 118 |
EditDelete{{groupBy.name}}{{dayFormat(period)}} {{getConsolidateName(data_consolidate)}}
{{header.name}}
104 | Edit 105 | 107 | Delete 108 | 110 | {{row.name}} 111 | {{row.name}} 112 | 114 | {{currencySign}} {{row[header.index] | number:2}} 115 |
119 |
120 | 121 |
122 | 123 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/common/IceOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.common; 19 | 20 | public class IceOptions { 21 | 22 | /** 23 | * Data start date in millis. 24 | */ 25 | public static final String START_MILLIS = "ice.startmillis"; 26 | 27 | /** 28 | * Property for company name. It must be specified in ReaderConfig. 29 | */ 30 | public static final String COMPANY_NAME = "ice.companyName"; 31 | 32 | /** 33 | * Property for currency sign. The default value is "$". 34 | */ 35 | public static final String CURRENCY_SIGN = "ice.currencySign"; 36 | 37 | /** 38 | * Property for currency rate. The default value is "1". 39 | */ 40 | public static final String CURRENCY_RATE = "ice.currencyRate"; 41 | 42 | /** 43 | * The URL of highstock.js. The default value is the Highcharts CDN (HTTPS) 44 | */ 45 | public static final String HIGHSTOCK_URL = "ice.highstockUrl"; 46 | 47 | /** 48 | * s3 bucket name where billing files are located. For multiple payer accounts, multiple bucket names can be specified delimited by comma ",". 49 | * Only read permission is needed. It must be specified in Config. 50 | */ 51 | public static final String BILLING_S3_BUCKET_NAME = "ice.billing_s3bucketname"; 52 | 53 | /** 54 | * Region for billing s3 bucket. It should be specified for buckets using v4 validation ",". 55 | * It must be specified in Config. 56 | */ 57 | public static final String BILLING_S3_BUCKET_REGION = "ice.billing_s3bucketregion"; 58 | 59 | /** 60 | * Prefix of billing files in billing s3 bucket. For multiple payer accounts, multiple bucket prefixes can be specified delimited by comma ",". 61 | * It must be specified in Config. 62 | */ 63 | public static final String BILLING_S3_BUCKET_PREFIX = "ice.billing_s3bucketprefix"; 64 | 65 | /** 66 | * Payer account id. Must be specified if across-accounts role is used to access billing files. For multiple payer accounts, acocunt ids can 67 | * be specified delimited by comma ",". 68 | */ 69 | public static final String BILLING_PAYER_ACCOUNT_ID = "ice.billing_payerAccountId"; 70 | 71 | /** 72 | * Billing file access role name to assume. Must be specified if across-accounts role is used to access billing files. For multiple payer accounts, 73 | * role names can be specified delimited by comma ",". 74 | */ 75 | public static final String BILLING_ACCESS_ROLENAME = "ice.billing_accessRoleName"; 76 | 77 | /** 78 | * Billing file access external ID. It is optional. Specify it if cross-accounts role is used to access billing files and external id is needed. 79 | * For multiple payer accounts, external ids can be specified delimited by comma ",". 80 | */ 81 | public static final String BILLING_ACCESS_EXTERNALID = "ice.billing_accessExternalId"; 82 | 83 | /** 84 | * User can configure their custom tags. 85 | */ 86 | public static final String CUSTOM_TAGS = "ice.customTags"; 87 | 88 | /** 89 | * Boolean Flag whether to use blended or Unblended Costs. Default is UnBlended Cost(false) 90 | */ 91 | public static final String USE_BLENDED = "ice.use_blended"; 92 | 93 | /** 94 | * s3 bucket name where output files are to be store. Both read and write permissions are needed. It must be specified in Config. 95 | */ 96 | public static final String WORK_S3_BUCKET_NAME = "ice.work_s3bucketname"; 97 | 98 | /** 99 | * Prefix of output files in output s3 bucket. It must be specified in Config. 100 | */ 101 | public static final String WORK_S3_BUCKET_PREFIX = "ice.work_s3bucketprefix"; 102 | 103 | /** 104 | * Local directory. It must be specified in Config. 105 | */ 106 | public static final String LOCAL_DIR = "ice.localDir"; 107 | 108 | /** 109 | * Monthly data cache size for reader. Default is 12. 110 | */ 111 | public static final String MONTHLY_CACHE_SIZE = "ice.monthlycachesize"; 112 | 113 | /** 114 | * Cost per monitor metric per hour, It's optional. 115 | */ 116 | public static final String COST_PER_MONITORMETRIC_PER_HOUR = "ice.cost_per_monitormetric_per_hour"; 117 | 118 | /** 119 | * url prefix, e.g. http://ice.netflix.com/ 120 | */ 121 | public static final String URL_PREFIX = "ice.urlPrefix"; 122 | 123 | /** 124 | * from email address. It must be registered in aws ses. 125 | */ 126 | public static final String FROM_EMAIL = "ice.fromEmail"; 127 | 128 | /** 129 | * ec2 ondemand hourly cost threshold to send alert email. The alert email will be sent at most once per day. 130 | */ 131 | public static final String ONDEMAND_COST_ALERT_THRESHOLD = "ice.ondemandCostAlertThreshold"; 132 | 133 | /** 134 | * ec2 ondemand hourly cost alert emails, separated by "," 135 | */ 136 | public static final String ONDEMAND_COST_ALERT_EMAILS = "ice.ondemandCostAlertEmails"; 137 | 138 | /** 139 | * What pricing data ice should use when calculating usage costs for resource groups 140 | */ 141 | public static final String RESOURCE_GROUP_COST = "ice.resourceGroupCost"; 142 | 143 | /** 144 | * Enable weekly cost email per application groups 145 | */ 146 | public static final String WEEKLYEMAILS = "ice.weeklyCostEmails"; 147 | 148 | /** 149 | * from email address for weekly cost emails. Must be registered in aws ses. 150 | */ 151 | public static final String WEEKLYFROM = "ice.weeklyCostEmails_fromEmail"; 152 | 153 | /** 154 | * bcc email address for weekly cost emails. 155 | */ 156 | public static final String WEEKLYBCC = "ice.weeklyCostEmails_bccEmail"; 157 | 158 | /** 159 | * from email to use when test flag is enabled. 160 | */ 161 | public static final String WEEKLYTEST = "ice.weeklyCostEmails_testEmail"; 162 | 163 | /** 164 | * from email to use when test flag is enabled. 165 | */ 166 | public static final String NUM_WEEKS_FOR_WEEKLYEMAILS = "ice.weeklyCostEmails_numWeeks"; 167 | } 168 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/basic/BasicManagers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.basic; 19 | 20 | import com.amazonaws.services.s3.AmazonS3Client; 21 | import com.amazonaws.services.s3.model.S3ObjectSummary; 22 | import com.google.common.collect.Maps; 23 | import com.google.common.collect.Sets; 24 | import com.netflix.ice.common.*; 25 | import com.netflix.ice.processor.TagGroupWriter; 26 | import com.netflix.ice.reader.*; 27 | import com.netflix.ice.tag.Product; 28 | import com.netflix.ice.tag.Tag; 29 | 30 | import java.util.Collection; 31 | import java.util.Map; 32 | import java.util.Set; 33 | import java.util.TreeMap; 34 | 35 | /** 36 | * This class manages all BasicTagGroupManager and BasicDataManager instances. 37 | */ 38 | public class BasicManagers extends Poller implements Managers { 39 | private ReaderConfig config; 40 | 41 | private Set products = Sets.newHashSet(); 42 | private Map tagGroupManagers = Maps.newHashMap(); 43 | private TreeMap costManagers = Maps.newTreeMap(); 44 | private TreeMap usageManagers = Maps.newTreeMap(); 45 | 46 | public void shutdown() { 47 | for (BasicTagGroupManager tagGroupManager: tagGroupManagers.values()) { 48 | tagGroupManager.shutdown(); 49 | } 50 | for (BasicDataManager dataManager: costManagers.values()) { 51 | dataManager.shutdown(); 52 | } 53 | for (BasicDataManager dataManager: usageManagers.values()) { 54 | dataManager.shutdown(); 55 | } 56 | } 57 | 58 | public void init() { 59 | config = ReaderConfig.getInstance(); 60 | doWork(); 61 | start(300); 62 | } 63 | 64 | public Collection getProducts() { 65 | return products; 66 | } 67 | 68 | public TagGroupManager getTagGroupManager(Product product) { 69 | return tagGroupManagers.get(product); 70 | } 71 | 72 | public DataManager getCostManager(Product product, ConsolidateType consolidateType) { 73 | return costManagers.get(new Key(product, consolidateType)); 74 | } 75 | 76 | public DataManager getUsageManager(Product product, ConsolidateType consolidateType) { 77 | return usageManagers.get(new Key(product, consolidateType)); 78 | } 79 | 80 | @Override 81 | protected void poll() throws Exception { 82 | doWork(); 83 | } 84 | 85 | private void doWork() { 86 | 87 | logger.info("trying to find new tag group and data managers..."); 88 | Set products = Sets.newHashSet(this.products); 89 | Map tagGroupManagers = Maps.newHashMap(this.tagGroupManagers); 90 | TreeMap costManagers = Maps.newTreeMap(this.costManagers); 91 | TreeMap usageManagers = Maps.newTreeMap(this.usageManagers); 92 | 93 | Set newProducts = Sets.newHashSet(); 94 | AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); 95 | for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).getObjectSummaries()) { 96 | String key = s3ObjectSummary.getKey(); 97 | Product product; 98 | if (key.endsWith("_all")) { 99 | product = null; 100 | } 101 | else { 102 | String name = key.substring((config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).length()); 103 | name = Tag.fromS3(name); 104 | product = config.productService.getProductByName(name); 105 | } 106 | if (!products.contains(product)) { 107 | products.add(product); 108 | newProducts.add(product); 109 | } 110 | } 111 | 112 | for (Product product: newProducts) { 113 | tagGroupManagers.put(product, new BasicTagGroupManager(product)); 114 | for (ConsolidateType consolidateType: ConsolidateType.values()) { 115 | Key key = new Key(product, consolidateType); 116 | costManagers.put(key, new BasicDataManager(product, consolidateType, true)); 117 | usageManagers.put(key, new BasicDataManager(product, consolidateType, false)); 118 | } 119 | } 120 | 121 | if (newProducts.size() > 0) { 122 | this.costManagers = costManagers; 123 | this.usageManagers = usageManagers; 124 | this.tagGroupManagers = tagGroupManagers; 125 | this.products = products; 126 | } 127 | } 128 | 129 | private static class Key implements Comparable { 130 | Product product; 131 | ConsolidateType consolidateType; 132 | Key(Product product, ConsolidateType consolidateType) { 133 | this.product = product; 134 | this.consolidateType = consolidateType; 135 | } 136 | 137 | public int compareTo(Key t) { 138 | int result = this.product == t.product ? 0 : (this.product == null ? 1 : (t.product == null ? -1 : t.product.compareTo(this.product))); 139 | if (result != 0) 140 | return result; 141 | return consolidateType.compareTo(t.consolidateType); 142 | } 143 | 144 | @Override 145 | public boolean equals(Object o) { 146 | if (o == null) 147 | return false; 148 | Key other = (Key)o; 149 | return 150 | this.product == other.product && 151 | this.consolidateType == other.consolidateType; 152 | } 153 | 154 | @Override 155 | public int hashCode() { 156 | final int prime = 31; 157 | int result = 1; 158 | if (product != null) 159 | result = prime * result + this.product.hashCode(); 160 | result = prime * result + this.consolidateType.hashCode(); 161 | 162 | return result; 163 | } 164 | } 165 | 166 | 167 | } 168 | -------------------------------------------------------------------------------- /grailsw.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem ## 4 | @rem Grails JVM Bootstrap for Windows ## 5 | @rem ## 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set CLASS=org.grails.wrapper.GrailsWrapper 12 | 13 | if exist "%USERPROFILE%/.groovy/preinit.bat" call "%USERPROFILE%/.groovy/preinit.bat" 14 | 15 | @rem Determine the command interpreter to execute the "CD" later 16 | set COMMAND_COM="cmd.exe" 17 | if exist "%SystemRoot%\system32\cmd.exe" set COMMAND_COM="%SystemRoot%\system32\cmd.exe" 18 | if exist "%SystemRoot%\command.com" set COMMAND_COM="%SystemRoot%\command.com" 19 | 20 | @rem Use explicit find.exe to prevent cygwin and others find.exe from being used 21 | set FIND_EXE="find.exe" 22 | if exist "%SystemRoot%\system32\find.exe" set FIND_EXE="%SystemRoot%\system32\find.exe" 23 | if exist "%SystemRoot%\command\find.exe" set FIND_EXE="%SystemRoot%\command\find.exe" 24 | 25 | :check_JAVA_HOME 26 | @rem Make sure we have a valid JAVA_HOME 27 | if not "%JAVA_HOME%" == "" goto have_JAVA_HOME 28 | 29 | echo. 30 | echo ERROR: Environment variable JAVA_HOME has not been set. 31 | echo. 32 | echo Please set the JAVA_HOME variable in your environment to match the 33 | echo location of your Java installation. 34 | echo. 35 | goto end 36 | 37 | :have_JAVA_HOME 38 | @rem Remove trailing slash from JAVA_HOME if found 39 | if "%JAVA_HOME:~-1%"=="\" SET JAVA_HOME=%JAVA_HOME:~0,-1% 40 | 41 | @rem Validate JAVA_HOME 42 | %COMMAND_COM% /C DIR "%JAVA_HOME%" 2>&1 | %FIND_EXE% /I /C "%JAVA_HOME%" >nul 43 | if not errorlevel 1 goto check_GRAILS_HOME 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | echo. 51 | goto end 52 | 53 | :check_GRAILS_HOME 54 | @rem Define GRAILS_HOME if not set 55 | if "%GRAILS_HOME%" == "" set GRAILS_HOME=%DIRNAME%.. 56 | 57 | @rem Remove trailing slash from GRAILS_HOME if found 58 | if "%GRAILS_HOME:~-1%"=="\" SET GRAILS_HOME=%GRAILS_HOME:~0,-1% 59 | 60 | :init 61 | 62 | for %%x in ("%USERPROFILE%") do set SHORTHOME=%%~fsx 63 | if "x%GRAILS_AGENT_CACHE_DIR%" == "x" set GRAILS_AGENT_CACHE_DIR=%SHORTHOME%/.grails/2.4.4/ 64 | set SPRINGLOADED_PARAMS="profile=grails;cacheDir=%GRAILS_AGENT_CACHE_DIR%" 65 | if not exist "%GRAILS_AGENT_CACHE_DIR%" mkdir "%GRAILS_AGENT_CACHE_DIR%" 66 | 67 | if "%GRAILS_NO_PERMGEN%" == "" ( 68 | type "%JAVA_HOME%\include\classfile_constants.h" 2>nul | findstr /R /C:"#define JVM_CLASSFILE_MAJOR_VERSION 5[23]" >nul 69 | if not errorlevel 1 set GRAILS_NO_PERMGEN=1 70 | ) 71 | 72 | set AGENT_STRING=-javaagent:wrapper/springloaded-1.2.1.RELEASE.jar -Xverify:none -Dspringloaded.synchronize=true -Djdk.reflect.allowGetCallerClass=true -Dspringloaded=\"%SPRINGLOADED_PARAMS%\" 73 | set DISABLE_RELOADING= 74 | if "%GRAILS_OPTS%" == "" ( 75 | set GRAILS_OPTS=-server -Xmx768M -Xms64M -Dfile.encoding=UTF-8 76 | if not "%GRAILS_NO_PERMGEN%" == "1" ( 77 | set GRAILS_OPTS=-server -Xmx768M -Xms64M -XX:PermSize=32m -XX:MaxPermSize=256m -Dfile.encoding=UTF-8 78 | ) 79 | ) 80 | 81 | @rem Get command-line arguments, handling Windows variants 82 | if "%@eval[2+2]" == "4" goto 4NT_args 83 | 84 | @rem Slurp the command line arguments. 85 | set CMD_LINE_ARGS= 86 | set CP= 87 | set INTERACTIVE=true 88 | 89 | :win9xME_args_slurp 90 | if "x%~1" == "x" goto execute 91 | set CURR_ARG=%~1 92 | if "%CURR_ARG:~0,2%" == "-D" ( 93 | set CMD_LINE_ARGS=%CMD_LINE_ARGS% %~1=%~2 94 | shift 95 | shift 96 | goto win9xME_args_slurp 97 | ) 98 | if "x%~1" == "x-cp" ( 99 | set CP=%~2 100 | shift 101 | shift 102 | goto win9xME_args_slurp 103 | ) 104 | if "x%~1" == "x-debug" ( 105 | set JAVA_OPTS=%JAVA_OPTS% -Xdebug -Xnoagent -Dgrails.full.stacktrace=true -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005 106 | shift 107 | goto win9xME_args_slurp 108 | ) 109 | if "x%~1" == "x-classpath" ( 110 | set CP=%~2 111 | shift 112 | shift 113 | goto win9xME_args_slurp 114 | ) 115 | if "x%~1" == "x-reloading" ( 116 | set AGENT=%AGENT_STRING% 117 | shift 118 | goto win9xME_args_slurp 119 | ) 120 | if "x%~1" == "xrun-app" ( 121 | set AGENT=%AGENT_STRING% 122 | set INTERACTIVE= 123 | set CMD_LINE_ARGS=%CMD_LINE_ARGS% %1 124 | shift 125 | goto win9xME_args_slurp 126 | ) 127 | if "x%~1" == "x-noreloading" ( 128 | set DISABLE_RELOADING=true 129 | shift 130 | goto win9xME_args_slurp 131 | ) 132 | set INTERACTIVE= 133 | set CMD_LINE_ARGS=%CMD_LINE_ARGS% %1 134 | shift 135 | goto win9xME_args_slurp 136 | 137 | :4NT_args 138 | @rem Get arguments from the 4NT Shell from JP Software 139 | set CMD_LINE_ARGS=%$ 140 | 141 | :execute 142 | @rem Setup the command line 143 | set STARTER_CLASSPATH=wrapper/grails-wrapper-runtime-2.4.4.jar;wrapper;. 144 | 145 | if exist "%USERPROFILE%/.groovy/init.bat" call "%USERPROFILE%/.groovy/init.bat" 146 | 147 | @rem Setting a classpath using the -cp or -classpath option means not to use 148 | @rem the global classpath. Groovy behaves then the same as the java interpreter 149 | 150 | if "x" == "x%CLASSPATH%" goto after_classpath 151 | set CP=%CP%;%CLASSPATH% 152 | :after_classpath 153 | 154 | if "x%DISABLE_RELOADING%" == "xtrue" ( 155 | set AGENT= 156 | ) else ( 157 | if "x%INTERACTIVE%" == "xtrue" ( 158 | set AGENT=%AGENT_STRING% 159 | ) 160 | ) 161 | 162 | set STARTER_MAIN_CLASS=org.grails.wrapper.GrailsWrapper 163 | set STARTER_CONF=%GRAILS_HOME%\conf\groovy-starter.conf 164 | 165 | set JAVA_EXE=%JAVA_HOME%\bin\java.exe 166 | set TOOLS_JAR=%JAVA_HOME%\lib\tools.jar 167 | 168 | set JAVA_OPTS=%GRAILS_OPTS% %JAVA_OPTS% %AGENT% 169 | 170 | set JAVA_OPTS=%JAVA_OPTS% -Dprogram.name="%PROGNAME%" 171 | set JAVA_OPTS=%JAVA_OPTS% -Dgrails.home="%GRAILS_HOME%" 172 | set JAVA_OPTS=%JAVA_OPTS% -Dgrails.version=2.4.4 173 | set JAVA_OPTS=%JAVA_OPTS% -Dbase.dir=. 174 | set JAVA_OPTS=%JAVA_OPTS% -Dtools.jar="%TOOLS_JAR%" 175 | set JAVA_OPTS=%JAVA_OPTS% -Dgroovy.starter.conf="%STARTER_CONF%" 176 | 177 | if exist "%USERPROFILE%/.groovy/postinit.bat" call "%USERPROFILE%/.groovy/postinit.bat" 178 | 179 | @rem Execute Grails 180 | CALL "%JAVA_EXE%" %JAVA_OPTS% -classpath "%STARTER_CLASSPATH%" %STARTER_MAIN_CLASS% --main %CLASS% --conf "%STARTER_CONF%" --classpath "%CP%" "%CMD_LINE_ARGS%" 181 | :end 182 | @rem End local scope for the variables with windows NT shell 183 | if "%OS%"=="Windows_NT" endlocal 184 | 185 | @rem Optional pause the batch file 186 | if "%GROOVY_BATCH_PAUSE%" == "on" pause 187 | -------------------------------------------------------------------------------- /conf/system/server.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 35 | 36 | 39 | 44 | 45 | 46 | 51 | 52 | 53 | 54 | 58 | 59 | 60 | 67 | 70 | 71 | 77 | 81 | 86 | 87 | 88 | 89 | 90 | 91 | 96 | 97 | 100 | 101 | 102 | 105 | 108 | 109 | 112 | 115 | 116 | 120 | 122 | 123 | 126 | 129 | 130 | 132 | 135 | 136 | 138 | 140 | 141 | 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /src/java/com/netflix/ice/reader/ReaderConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Copyright 2013 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | package com.netflix.ice.reader; 19 | 20 | import com.amazonaws.auth.AWSCredentialsProvider; 21 | import com.netflix.ice.basic.BasicWeeklyCostEmailService; 22 | import com.netflix.ice.common.*; 23 | import com.netflix.ice.tag.Product; 24 | import com.netflix.ice.tag.TagType; 25 | import org.joda.time.DateTime; 26 | import org.joda.time.DateTimeZone; 27 | import org.joda.time.Interval; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import java.util.Collection; 32 | import java.util.Properties; 33 | 34 | /** 35 | * COnfiguration class for reader/UI. 36 | */ 37 | public class ReaderConfig extends Config { 38 | private static ReaderConfig instance; 39 | private static final Logger logger = LoggerFactory.getLogger(ReaderConfig.class); 40 | 41 | public final String companyName; 42 | public final String currencySign; 43 | public final double currencyRate; 44 | public final String highstockUrl; 45 | public final ApplicationGroupService applicationGroupService; 46 | public final ThroughputMetricService throughputMetricService; 47 | public final BasicWeeklyCostEmailService costEmailService; 48 | public final Managers managers; 49 | public final int monthlyCacheSize; 50 | 51 | /** 52 | * 53 | * @param properties (required) 54 | * @param managers (required) 55 | * @param accountService (required) 56 | * @param productService (required) 57 | * @param resourceService (optional) 58 | * @param applicationGroupService (optional) 59 | * @param throughputMetricService (optional) 60 | */ 61 | public ReaderConfig( 62 | Properties properties, 63 | AWSCredentialsProvider credentialsProvider, 64 | Managers managers, 65 | AccountService accountService, 66 | ProductService productService, 67 | ResourceService resourceService, 68 | ApplicationGroupService applicationGroupService, 69 | ThroughputMetricService throughputMetricService, 70 | BasicWeeklyCostEmailService costEmailService) { 71 | super(properties, credentialsProvider, accountService, productService, resourceService); 72 | 73 | companyName = properties.getProperty(IceOptions.COMPANY_NAME, ""); 74 | currencySign = properties.getProperty(IceOptions.CURRENCY_SIGN, "$"); 75 | currencyRate = Double.parseDouble(properties.getProperty(IceOptions.CURRENCY_RATE, "1")); 76 | highstockUrl = properties.getProperty(IceOptions.HIGHSTOCK_URL, "https://code.highcharts.com/stock/4.2.1/highstock.js"); 77 | 78 | this.managers = managers; 79 | this.applicationGroupService = applicationGroupService; 80 | this.throughputMetricService = throughputMetricService; 81 | this.costEmailService = costEmailService; 82 | this.monthlyCacheSize = Integer.parseInt(properties.getProperty(IceOptions.MONTHLY_CACHE_SIZE, "12")); 83 | 84 | ReaderConfig.instance = this; 85 | 86 | // AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); 87 | // logger.info("Deleting all files..."); 88 | // List objectSummariesToDelete = AwsUtils.listAllObjects(instance.workS3BucketName, instance.workS3BucketPrefix); 89 | // for (S3ObjectSummary objectSummary : objectSummariesToDelete) { 90 | // 91 | // String fileKey = objectSummary.getKey(); 92 | // 93 | // String name = fileKey.substring(fileKey.lastIndexOf("/") + 1); 94 | // if (name.startsWith("cost_") || name.startsWith("usage_") || name.startsWith("tagdb_")) { 95 | // s3Client.deleteObject(instance.workS3BucketName, fileKey); 96 | // continue; 97 | // } 98 | // } 99 | 100 | if (throughputMetricService != null) 101 | throughputMetricService.init(); 102 | managers.init(); 103 | applicationGroupService.init(); 104 | } 105 | 106 | /** 107 | * 108 | * @return singlton instance 109 | */ 110 | public static ReaderConfig getInstance() { 111 | return instance; 112 | } 113 | 114 | public void start() { 115 | 116 | Managers managers = ReaderConfig.getInstance().managers; 117 | Collection products = managers.getProducts(); 118 | for (Product product: products) { 119 | TagGroupManager tagGroupManager = managers.getTagGroupManager(product); 120 | Interval interval = tagGroupManager.getOverlapInterval(new Interval(new DateTime(DateTimeZone.UTC).minusMonths(monthlyCacheSize), new DateTime(DateTimeZone.UTC))); 121 | if (interval == null) 122 | continue; 123 | for (ConsolidateType consolidateType: ConsolidateType.values()) { 124 | readData(product, managers.getCostManager(product, consolidateType), interval, consolidateType); 125 | readData(product, managers.getUsageManager(product, consolidateType), interval, consolidateType); 126 | } 127 | } 128 | 129 | if (costEmailService != null) 130 | costEmailService.start(); 131 | } 132 | 133 | public void shutdown() { 134 | logger.info("Shutting down..."); 135 | 136 | instance.managers.shutdown(); 137 | if (instance.costEmailService != null) 138 | instance.costEmailService.shutdown(); 139 | } 140 | 141 | private void readData(Product product, DataManager dataManager, Interval interval, ConsolidateType consolidateType) { 142 | if (consolidateType == ConsolidateType.hourly) { 143 | DateTime start = interval.getStart().withDayOfMonth(1).withMillisOfDay(0); 144 | do { 145 | int hours = dataManager.getDataLength(start); 146 | logger.info("found " + hours + " hours data for " + product + " " + interval); 147 | start = start.plusMonths(1); 148 | } 149 | while (start.isBefore(interval.getEnd())); 150 | } 151 | else if (consolidateType == ConsolidateType.daily) { 152 | DateTime start = interval.getStart().withDayOfYear(1).withMillisOfDay(0); 153 | do { 154 | dataManager.getDataLength(start); 155 | start = start.plusYears(1); 156 | } 157 | while (start.isBefore(interval.getEnd())); 158 | } 159 | else { 160 | dataManager.getData(interval, new TagLists(), TagType.Account, AggregateType.both, false); 161 | } 162 | } 163 | } 164 | --------------------------------------------------------------------------------