├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .travis.yml ├── .zappr.yaml ├── Dockerfile ├── LICENSE ├── MAINTAINERS ├── README.rst ├── delivery.yaml ├── dummy_data ├── alerts.yaml ├── checks.yaml ├── cities.json └── entities.yaml ├── mvnw ├── pom.xml ├── scripts ├── pyfetch.py ├── redis-cleanup-downtimes.py ├── redis-cleanup.py ├── redis-keysizes.py └── redis-stats.py ├── src ├── main │ ├── java │ │ └── de │ │ │ └── zalando │ │ │ └── zmon │ │ │ └── scheduler │ │ │ └── ng │ │ │ ├── Alert.java │ │ │ ├── AlertOverlapGenerator.java │ │ │ ├── Application.java │ │ │ ├── BaseSource.java │ │ │ ├── CachedRepository.java │ │ │ ├── CeleryBody.java │ │ │ ├── CeleryWriter.java │ │ │ ├── Check.java │ │ │ ├── CommandSerializer.java │ │ │ ├── DataCenterSubscriber.java │ │ │ ├── DefinitionStatus.java │ │ │ ├── LocalDateFormatter.java │ │ │ ├── RedisResponseHolder.java │ │ │ ├── SchedulePersistType.java │ │ │ ├── Source.java │ │ │ ├── SourceRegistry.java │ │ │ ├── TaskSerializerType.java │ │ │ ├── TaskWriterType.java │ │ │ ├── TokenWrapper.java │ │ │ ├── alerts │ │ │ ├── AlertChangeListener.java │ │ │ ├── AlertDefinition.java │ │ │ ├── AlertDefinitions.java │ │ │ ├── AlertRepository.java │ │ │ ├── AlertSource.java │ │ │ ├── AlertSourceRegistry.java │ │ │ ├── DefaultAlertSource.java │ │ │ ├── Parameter.java │ │ │ └── YamlAlertSource.java │ │ │ ├── checks │ │ │ ├── CheckChangeListener.java │ │ │ ├── CheckChangedListener.java │ │ │ ├── CheckDefinition.java │ │ │ ├── CheckDefinitions.java │ │ │ ├── CheckRepository.java │ │ │ ├── CheckSource.java │ │ │ ├── CheckSourceRegistry.java │ │ │ ├── DefaultCheckSource.java │ │ │ ├── MinIntervalEntityFetcher.java │ │ │ └── YamlCheckSource.java │ │ │ ├── cleanup │ │ │ ├── AlertChangedCleaner.java │ │ │ ├── AllTrialRunCleanupTask.java │ │ │ ├── CheckChangedCleaner.java │ │ │ ├── DowntimeCleanup.java │ │ │ ├── EntityChangedCleaner.java │ │ │ ├── MetricsCleanup.java │ │ │ ├── SingleEntityCleanup.java │ │ │ └── TrialRunCleanupTask.java │ │ │ ├── config │ │ │ ├── CleanupConfiguration.java │ │ │ ├── Oauth2Config.java │ │ │ ├── QueueSelectorConfiguration.java │ │ │ ├── RestTemplateConfiguration.java │ │ │ ├── SchedulerConfig.java │ │ │ ├── SchedulerFactory.java │ │ │ ├── SingleEntityCleanupConfiguration.java │ │ │ └── StupsOAuthConfig.java │ │ │ ├── downtimes │ │ │ ├── DowntimeAlertRequest.java │ │ │ ├── DowntimeConfiguration.java │ │ │ ├── DowntimeData.java │ │ │ ├── DowntimeForwardTask.java │ │ │ ├── DowntimeForwarder.java │ │ │ ├── DowntimeHttpSubscriber.java │ │ │ ├── DowntimeRequest.java │ │ │ ├── DowntimeRequestResult.java │ │ │ ├── DowntimeService.java │ │ │ ├── DowntimeTaskType.java │ │ │ └── DowntimesAPI.java │ │ │ ├── entities │ │ │ ├── Entity.java │ │ │ ├── EntityAdapter.java │ │ │ ├── EntityAdapterRegistry.java │ │ │ ├── EntityChangeListener.java │ │ │ ├── EntityRepository.java │ │ │ ├── Environments.java │ │ │ ├── Teams.java │ │ │ └── adapters │ │ │ │ ├── EmptyAdapter.java │ │ │ │ ├── EntityServiceAdapter.java │ │ │ │ ├── GlobalAdapter.java │ │ │ │ └── YamlEntityAdapter.java │ │ │ ├── instantevaluations │ │ │ ├── InstantEvalForwarder.java │ │ │ ├── InstantEvalHttpSubscriber.java │ │ │ └── InstantEvaluationAPI.java │ │ │ ├── queue │ │ │ ├── ArrayQueueWriter.java │ │ │ ├── HardCodedSelector.java │ │ │ ├── JedisQueueWriter.java │ │ │ ├── LogQueueWriter.java │ │ │ ├── PropertyQueueSelector.java │ │ │ ├── QueueMetrics.java │ │ │ ├── QueueSelector.java │ │ │ ├── QueueWriter.java │ │ │ ├── RepoSelector.java │ │ │ └── Selector.java │ │ │ ├── scheduler │ │ │ ├── RedisMetricsUpdater.java │ │ │ ├── SchedulePersister.java │ │ │ ├── ScheduledCheck.java │ │ │ ├── Scheduler.java │ │ │ └── SchedulerMetrics.java │ │ │ └── trailruns │ │ │ ├── TrialRunAPI.java │ │ │ ├── TrialRunForwarder.java │ │ │ ├── TrialRunHttpSubscriber.java │ │ │ └── TrialRunRequest.java │ └── resources │ │ └── config │ │ └── application.yaml └── test │ └── java │ └── de │ └── zalando │ └── zmon │ └── scheduler │ └── ng │ ├── CeleryWriterTest.java │ ├── FilterTest.java │ ├── SchedulerTest.java │ ├── alerts │ ├── AlertPropertiesChangedTest.java │ └── AlertRepositoryTest.java │ ├── checks │ ├── CheckDefinitionTest.java │ └── CheckRepositoryTest.java │ ├── cleanup │ └── EntityPropertyChangeTest.java │ └── entities │ └── EntityRepositoryTest.java └── start.sh /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | config/application-zalando.yaml 3 | src/main/resources/application-zalando.properties 4 | src/main/resources/application-zalando.yaml 5 | .idea 6 | *.iml 7 | *.json 8 | 9 | .project 10 | .classpath 11 | .settings 12 | 13 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zalando-zmon/zmon-scheduler/6e7938023ecf1c6b8c2a94daa332ccb3ca26ecc2/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.3/apache-maven-3.3.3-bin.zip -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | dist: trusty 3 | 4 | jdk: 5 | - oraclejdk8 6 | 7 | services: 8 | - redis-server 9 | 10 | before_script: 11 | - pip install --user codecov 12 | 13 | script: 14 | - mvn clean verify -Pcoverage 15 | 16 | after_success: 17 | - codecov 18 | -------------------------------------------------------------------------------- /.zappr.yaml: -------------------------------------------------------------------------------- 1 | # for github.com 2 | approvals: 3 | groups: 4 | zalando: 5 | minimum: 2 6 | from: 7 | orgs: 8 | - "zalando" 9 | 10 | # Allow last committer / PR creator to approve as well. 11 | # This will probably become the default in zappr. 12 | ignore: none 13 | X-Zalando-Team: "zmon" 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM registry.opensource.zalan.do/stups/openjdk:latest 2 | 3 | EXPOSE 8085 4 | 5 | COPY target/zmon-scheduler-1.0-SNAPSHOT.jar /zmon-scheduler.jar 6 | COPY start.sh /start.sh 7 | RUN chmod 755 /start.sh 8 | CMD ["/start.sh"] 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Zalando SE 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /MAINTAINERS: -------------------------------------------------------------------------------- 1 | Jan Mußler 2 | Henning Jacobs 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ZMON source code on GitHub is no longer in active development. Zalando will no longer actively review issues or merge pull-requests. 2 | 3 | ZMON is still being used at Zalando and serves us well for many purposes. We are now deeper into our observability journey and understand better that we need other telemetry sources and tools to elevate our understanding of the systems we operate. We support the `OpenTelemetry `_ initiative and recommended others starting their journey to begin there. 4 | 5 | If members of the community are interested in continuing developing ZMON, consider forking it. Please review the licence before you do. 6 | 7 | ============== 8 | ZMON Scheduler 9 | ============== 10 | 11 | .. image:: https://travis-ci.org/zalando-zmon/zmon-scheduler.svg?branch=master 12 | :target: https://travis-ci.org/zalando-zmon/zmon-scheduler 13 | :alt: Build Status 14 | 15 | .. image:: https://codecov.io/github/zalando-zmon/zmon-scheduler/coverage.svg?branch=master 16 | :target: https://codecov.io/github/zalando-zmon/zmon-scheduler 17 | :alt: Code Coverage 18 | 19 | The ZMON Scheduler is responsible for keeping track of all existing entities, checks and alerts and scheduling checks in time for applicable entities, which are then executed by the `ZMON Worker`_. 20 | 21 | Running Unit Tests 22 | ================== 23 | 24 | .. code-block:: bash 25 | 26 | $ ./mvnw clean test 27 | 28 | Running Locally 29 | =============== 30 | 31 | .. code-block:: bash 32 | 33 | $ ./mvnw clean install 34 | $ java -jar target/zmon-scheduler-1.0-SNAPSHOT.jar 35 | 36 | Building the Docker Image 37 | ========================= 38 | 39 | .. code-block:: bash 40 | 41 | $ ./mvnw clean package 42 | $ docker build -t zmon-scheduler . 43 | 44 | 45 | See also the `ZMON Documentation`_. 46 | 47 | .. _main ZMON repository: https://github.com/zalando/zmon 48 | .. _ZMON Documentation: https://docs.zmon.io/ 49 | .. _ZMON Worker: https://github.com/zalando-zmon/zmon-worker 50 | -------------------------------------------------------------------------------- /delivery.yaml: -------------------------------------------------------------------------------- 1 | version: "2017-09-20" 2 | pipeline: 3 | - id: build 4 | type: script 5 | commands: 6 | - desc: "Install dependencies" 7 | cmd: | 8 | apt-get -y update 9 | apt-get -y install openjdk-8-* maven git 10 | curl -fLOsS https://delivery.cloud.zalando.com/utils/ensure-docker && sh ensure-docker && rm ensure-docker 11 | 12 | - desc: "Build maven package" 13 | cmd: | 14 | ./mvnw clean package 15 | 16 | - desc: "Push Docker Image" 17 | cmd: | 18 | IS_PR_BUILD=${CDP_PULL_REQUEST_NUMBER+"true"} 19 | if [[ ${IS_PR_BUILD} != "true" ]] 20 | then 21 | VERSION=$(git describe --tags --always --dirty) 22 | AGENT_IMAGE=registry-write.opensource.zalan.do/zmon/zmon-scheduler:${VERSION} 23 | else 24 | VERSION=${CDP_BUILD_VERSION} 25 | AGENT_IMAGE=registry-write.opensource.zalan.do/zmon/zmon-scheduler-unstable:${VERSION} 26 | fi 27 | 28 | docker build --tag "$AGENT_IMAGE" . 29 | docker push "$AGENT_IMAGE" 30 | -------------------------------------------------------------------------------- /dummy_data/alerts.yaml: -------------------------------------------------------------------------------- 1 | - id: 1 2 | check_definition_id: 1 3 | name: "Alert 1" 4 | entities: [] 5 | - id: 2 6 | check_definition_id: 2 7 | name: "Alert 2" 8 | entities: [] 9 | - id: 3 10 | check_definition_id: 3 11 | name: "Live ZMON Schedulers" 12 | entities: [] 13 | - id: 4 14 | check_definition_id: 4 15 | name: "Live Alert filter" 16 | entities: 17 | - environment: live -------------------------------------------------------------------------------- /dummy_data/checks.yaml: -------------------------------------------------------------------------------- 1 | - id: 1 2 | name: "Check 1" 3 | interval: 5 4 | entities: 5 | - type: "host" 6 | - id: 2 7 | name: "Zmon Scheduler Group" 8 | interval: 10 9 | entities: 10 | - type: instance 11 | application_id: zmon-scheduler 12 | - id: 3 13 | name: "Zmon Live Schedulers" 14 | interval: 10 15 | entities: 16 | - type: instance 17 | environment: live 18 | - id: 4 19 | name: "Zmon Base Scheduler Group" 20 | interval: 10 21 | entities: 22 | - type: instance 23 | application_id: zmon-scheduler -------------------------------------------------------------------------------- /dummy_data/cities.json: -------------------------------------------------------------------------------- 1 | [{"city": "tokyo", "country": "jp", "region": "40", "longitude": "139.751389", "latitude": "35.685", "accentcity": "Tokyo", "population": "31480498"}, {"city": "shanghai", "country": "cn", "region": "23", "longitude": "121.399722", "latitude": "31.045556", "accentcity": "Shanghai", "population": "14608512"}, {"city": "bombay", "country": "in", "region": "16", "longitude": "72.825833", "latitude": "18.975", "accentcity": "Bombay", "population": "12692717"}, {"city": "karachi", "country": "pk", "region": "05", "longitude": "67.0822", "latitude": "24.9056", "accentcity": "Karachi", "population": "11627378"}, {"city": "delhi", "country": "in", "region": "07", "longitude": "77.216667", "latitude": "28.666667", "accentcity": "Delhi", "population": "10928270"}, {"city": "new delhi", "country": "in", "region": "07", "longitude": "77.2", "latitude": "28.6", "accentcity": "New Delhi", "population": "10928270"}, {"city": "manila", "country": "ph", "region": "D9", "longitude": "120.9822", "latitude": "14.6042", "accentcity": "Manila", "population": "10443877"}, {"city": "moscow", "country": "ru", "region": "48", "longitude": "37.615556", "latitude": "55.752222", "accentcity": "Moscow", "population": "10381288"}, {"city": "seoul", "country": "kr", "region": "11", "longitude": "126.9783", "latitude": "37.5985", "accentcity": "Seoul", "population": "10323448"}, {"city": "sao paulo", "country": "br", "region": "27", "longitude": "-46.665803", "latitude": "-23.473293", "accentcity": "Sao Paulo", "population": "10021437"}, {"city": "istanbul", "country": "tr", "region": "34", "longitude": "28.964722", "latitude": "41.018611", "accentcity": "Istanbul", "population": "9797536"}, {"city": "lagos", "country": "ng", "region": "05", "longitude": "3.395833", "latitude": "6.453056", "accentcity": "Lagos", "population": "8789133"}, {"city": "mexico", "country": "mx", "region": "09", "longitude": "-99.138611", "latitude": "19.434167", "accentcity": "Mexico", "population": "8720916"}, {"city": "jakarta", "country": "id", "region": "04", "longitude": "106.829444", "latitude": "-6.174444", "accentcity": "Jakarta", "population": "8540306"}, {"city": "new york", "country": "us", "region": "NY", "longitude": "-74.0063889", "latitude": "40.7141667", "accentcity": "New York", "population": "8107916"}, {"city": "kinshasa", "country": "cd", "region": "06", "longitude": "15.3", "latitude": "-4.3", "accentcity": "Kinshasa", "population": "7787832"}, {"city": "cairo", "country": "eg", "region": "11", "longitude": "31.25", "latitude": "30.05", "accentcity": "Cairo", "population": "7734602"}, {"city": "lima", "country": "pe", "region": "15", "longitude": "-77.05", "latitude": "-12.05", "accentcity": "Lima", "population": "7646786"}, {"city": "peking", "country": "cn", "region": "22", "longitude": "116.388333", "latitude": "39.928889", "accentcity": "Peking", "population": "7480601"}, {"city": "london", "country": "gb", "region": "H9", "longitude": "-.093689", "latitude": "51.514125", "accentcity": "London", "population": "7421228"}] 2 | -------------------------------------------------------------------------------- /dummy_data/entities.yaml: -------------------------------------------------------------------------------- 1 | - id: "host1" 2 | type: "host" 3 | - id: "host2" 4 | type: "host" 5 | - id: "host01:3421" 6 | type: "instance" 7 | application_id: "zmon-scheduler" 8 | environment: "live" 9 | host: "host01" 10 | port: 3421 11 | - id: "host02:3421" 12 | type: "instance" 13 | application_id: "zmon-scheduler" 14 | environment: "live" 15 | host: "host02" 16 | port: 3421 17 | - id: "beta01:3411" 18 | type: "instance" 19 | environment: "beta" 20 | application_id: "zmon-scheduler" 21 | host: "beta01" 22 | port: "3411" -------------------------------------------------------------------------------- /scripts/pyfetch.py: -------------------------------------------------------------------------------- 1 | import redis 2 | import json 3 | import base64 4 | import snappy 5 | 6 | r = redis.StrictRedis('monitor03.zalando',6379) 7 | print r.llen('zmon:queue:default') 8 | 9 | val = r.blpop('zmon:queue:default', 30) 10 | 11 | print snappy.decompress(val[1]) 12 | -------------------------------------------------------------------------------- /scripts/redis-cleanup-downtimes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | Helper script to clean up some Redis keys 5 | 6 | This should probably become part of the regular scheduler code! 7 | ''' 8 | 9 | import click 10 | import clickclick 11 | import json 12 | import redis 13 | import collections 14 | import time 15 | 16 | 17 | @click.command() 18 | @click.argument('redis-host') 19 | @click.argument('redis-port') 20 | def main(redis_host, redis_port): 21 | r = redis.StrictRedis(redis_host, redis_port) 22 | 23 | keys = r.keys('zmon:downtimes:*:*') 24 | p = r.pipeline() 25 | for key in keys: 26 | print("Deleting ... {}".format(key)) 27 | p.delete(key) 28 | results = p.execute() 29 | 30 | if __name__ == '__main__': 31 | main() 32 | -------------------------------------------------------------------------------- /scripts/redis-cleanup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | Helper script to clean up some Redis keys 5 | 6 | This should probably become part of the regular scheduler code! 7 | ''' 8 | 9 | import click 10 | import json 11 | import redis 12 | import requests 13 | import time 14 | import zign.api 15 | 16 | 17 | def cleanup_outdated_results(r): 18 | now = time.time() 19 | keys = r.keys('zmon:checks:*:*') 20 | p = r.pipeline() 21 | for key in keys: 22 | p.lrange(key, 0, 0) 23 | results = p.execute() 24 | 25 | cutoff = now - (8 * 24 * 3600) # 8 days ago 26 | ages = {} 27 | for key, res in zip(keys, results): 28 | if res: 29 | data = json.loads(res[0].decode('utf-8')) 30 | if data['ts'] < cutoff: 31 | ages[key] = now - data['ts'] 32 | for key, age in ages.items(): 33 | _, _, check_id, entity_id = key.decode('utf-8').split(':', 3) 34 | print('Deleting outdated check results {}/{} ({}d ago)..'.format(check_id, entity_id, int(age/(3600*24)))) 35 | r.delete(key) 36 | 37 | 38 | @click.command() 39 | @click.argument('zmon_url') 40 | @click.argument('redis-host') 41 | @click.argument('redis-port') 42 | def main(zmon_url, redis_host, redis_port): 43 | r = redis.StrictRedis(redis_host, redis_port) 44 | 45 | # delete all Trial Run results 46 | keys = r.keys('zmon:trial_run:*') 47 | for key in keys: 48 | print('Deleting {}..'.format(key)) 49 | r.delete(key) 50 | 51 | headers = {'Authorization': 'Bearer {}'.format(zign.api.get_token('zmon', ['uid']))} 52 | 53 | # get all active checks 54 | response = requests.get(zmon_url + '/checks/all-active-alert-definitions', headers=headers) 55 | data = response.json() 56 | all_active_alert_ids = set() 57 | all_referenced_check_ids = set() 58 | for row in data['alert_definitions']: 59 | all_active_alert_ids.add(row['id']) 60 | all_referenced_check_ids.add(row['check_definition_id']) 61 | 62 | response = requests.get(zmon_url + '/checks/all-active-check-definitions', headers=headers) 63 | data = response.json() 64 | all_active_check_ids = set() 65 | for row in data['check_definitions']: 66 | all_active_check_ids.add(row['id']) 67 | 68 | all_active_check_ids = all_active_check_ids & all_referenced_check_ids 69 | 70 | keys = r.keys('zmon:alerts:*') 71 | keys_to_delete = set() 72 | for key in keys: 73 | parts = key.split(b':') 74 | alert_id = int(parts[2]) 75 | if alert_id not in all_active_alert_ids: 76 | keys_to_delete.add(key) 77 | for key in sorted(keys_to_delete): 78 | print('Deleting {}..'.format(key)) 79 | r.delete(key) 80 | 81 | keys = r.keys('zmon:metrics:*:alerts.*.*') 82 | keys_to_delete = set() 83 | for key in keys: 84 | parts = key.rsplit(b'.') 85 | alert_id = int(parts[-2]) 86 | if alert_id not in all_active_alert_ids: 87 | keys_to_delete.add(key) 88 | for key in sorted(keys_to_delete): 89 | print('Deleting {}..'.format(key)) 90 | r.delete(key) 91 | 92 | keys = r.keys('zmon:downtimes:*:*') 93 | keys_to_delete = set() 94 | for key in keys: 95 | parts = key.split(b':', 3) 96 | alert_id = int(parts[2]) 97 | if alert_id not in all_active_alert_ids: 98 | keys_to_delete.add(key) 99 | for key in sorted(keys_to_delete): 100 | print('Deleting {}..'.format(key)) 101 | r.delete(key) 102 | 103 | # delete all non-active checks 104 | keys = r.keys('zmon:checks:*') 105 | keys_to_delete = set() 106 | for key in keys: 107 | parts = key.split(b':') 108 | check_id = int(parts[2]) 109 | if check_id not in all_active_check_ids: 110 | keys_to_delete.add(key) 111 | for key in sorted(keys_to_delete): 112 | print('Deleting {}..'.format(key)) 113 | r.delete(key) 114 | 115 | keys = r.keys('zmon:metrics:*:check.*.*') 116 | keys_to_delete = set() 117 | for key in keys: 118 | parts = key.rsplit(b'.') 119 | check_id = int(parts[-2]) 120 | if check_id not in all_active_check_ids: 121 | keys_to_delete.add(key) 122 | for key in sorted(keys_to_delete): 123 | print('Deleting {}..'.format(key)) 124 | r.delete(key) 125 | 126 | cleanup_outdated_results(r) 127 | 128 | # delete ALL metrics 129 | # keys = r.keys('zmon:metrics:*') 130 | # for key in keys: 131 | # print('Deleting {}..'.format(key)) 132 | # r.delete(key) 133 | 134 | if __name__ == '__main__': 135 | main() 136 | -------------------------------------------------------------------------------- /scripts/redis-keysizes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | Helper script to clean up some Redis keys 5 | 6 | This should probably become part of the regular scheduler code! 7 | ''' 8 | 9 | import click 10 | import random 11 | import redis 12 | import collections 13 | 14 | 15 | @click.command() 16 | @click.argument('redis-host') 17 | @click.argument('redis-port') 18 | def main(redis_host, redis_port): 19 | r = redis.StrictRedis(redis_host, redis_port) 20 | 21 | keys = r.keys('zmon:checks:*') 22 | sizes = collections.Counter() 23 | for key in random.sample(keys, 10000): 24 | print('.', end='') 25 | try: 26 | res = r.debug_object(key) 27 | except: 28 | pass 29 | else: 30 | sizes[key] = res['serializedlength'] 31 | print(sizes.most_common(20)) 32 | 33 | if __name__ == '__main__': 34 | main() 35 | -------------------------------------------------------------------------------- /scripts/redis-stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ''' 4 | Helper script to clean up some Redis keys 5 | 6 | This should probably become part of the regular scheduler code! 7 | ''' 8 | 9 | import click 10 | import clickclick 11 | import json 12 | import redis 13 | import collections 14 | import time 15 | 16 | 17 | @click.command() 18 | @click.argument('redis-host') 19 | @click.argument('redis-port') 20 | def main(redis_host, redis_port): 21 | r = redis.StrictRedis(redis_host, redis_port) 22 | 23 | now = time.time() 24 | keys = r.keys('zmon:checks:*:*') 25 | p = r.pipeline() 26 | for key in keys: 27 | p.lrange(key, 0, 1) 28 | results = p.execute() 29 | 30 | cutoff = now - (2 * 3600) # two hours ago 31 | durations = collections.Counter() 32 | for key, res in zip(keys, results): 33 | if len(res) >= 2: 34 | data = json.loads(res[0].decode('utf-8')) 35 | # only consider "recent" results 36 | if data['ts'] > cutoff: 37 | _, _, check_id, entity_id = key.decode('utf-8').split(':', 3) 38 | check_id = int(check_id) 39 | data_before = json.loads(res[1].decode('utf-8')) 40 | interval_seconds = data['ts'] - data_before['ts'] 41 | checks_per_second = 1. / interval_seconds 42 | durations.update({check_id: data['td'] * checks_per_second}) 43 | 44 | # print "worst" checks 45 | # i.e. checks using the most worker time 46 | rows = [] 47 | for key, duration in durations.most_common(20): 48 | rows.append({'check_id': key, 'duration': round(duration, 1)}) 49 | clickclick.print_table(['check_id', 'duration'], rows) 50 | 51 | total_duration = sum(durations.values()) 52 | print('Total duration: {:.2f}s'.format(total_duration)) 53 | 54 | if __name__ == '__main__': 55 | main() 56 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/Alert.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 4 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 5 | import de.zalando.zmon.scheduler.ng.entities.Entity; 6 | 7 | /** 8 | * Created by jmussler on 30.06.16. 9 | */ 10 | public class Alert { 11 | 12 | private final int id; 13 | private final AlertRepository alertRepo; 14 | 15 | public Alert(int id, AlertRepository alertRepo) { 16 | this.alertRepo = alertRepo; 17 | this.id = id; 18 | } 19 | 20 | public AlertDefinition getAlertDefinition() { 21 | return alertRepo.get(id); 22 | } 23 | 24 | public boolean matchEntity(Entity entity) { 25 | return AlertOverlapGenerator.matchAlertFilter(getAlertDefinition(), entity); 26 | } 27 | 28 | public int getId() { 29 | return id; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/BaseSource.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import java.text.DateFormat; 4 | import java.text.ParseException; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Collection; 7 | import java.util.TimeZone; 8 | 9 | /** 10 | * Created by jmussler on 4/7/15. 11 | */ 12 | public abstract class BaseSource implements Source { 13 | private final String name; 14 | 15 | public BaseSource(String n) { 16 | name = n; 17 | } 18 | 19 | @Override 20 | public String getName() { 21 | return name; 22 | } 23 | 24 | protected static boolean doRefresh(String headerValue, long currentMaxLastModified, Collection lastData) { 25 | if (lastData == null) return true; 26 | 27 | TimeZone tz = TimeZone.getTimeZone("UTC"); 28 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); 29 | df.setTimeZone(tz); 30 | 31 | try { 32 | long lastModified = df.parse(headerValue).getTime(); 33 | return currentMaxLastModified != lastModified; 34 | } catch (ParseException e) { 35 | return true; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/CachedRepository.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import io.opentracing.Scope; 4 | import io.opentracing.Tracer; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.Collection; 9 | import java.util.Map; 10 | import java.util.concurrent.ScheduledExecutorService; 11 | import java.util.concurrent.ScheduledThreadPoolExecutor; 12 | import java.util.concurrent.TimeUnit; 13 | 14 | /** 15 | * Created by jmussler on 4/7/15. 16 | */ 17 | public abstract class CachedRepository implements Runnable { 18 | private static final Logger LOG = LoggerFactory.getLogger(CachedRepository.class); 19 | 20 | protected Map currentMap; 21 | protected final S registry; 22 | protected final Tracer tracer; 23 | protected static final ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1); 24 | 25 | private long lastUpdated = 0; 26 | 27 | public CachedRepository(S r, Tracer tracer) { 28 | assert (null != r); 29 | this.registry = r; 30 | this.tracer = tracer; 31 | 32 | LOG.info("starting scheduler for {}", registry.getClass().getName()); 33 | executor.scheduleAtFixedRate(this, 20, 60, TimeUnit.SECONDS); 34 | } 35 | 36 | protected abstract T getNullObject(); 37 | 38 | abstract public void fill(); 39 | 40 | public long getLastUpdated() { 41 | return lastUpdated; 42 | } 43 | 44 | public void run() { 45 | String operationName = String.format("repository_update_%s", registry.getClass().getSimpleName()); 46 | try (Scope scope = tracer.buildSpan(operationName).startActive(true)) { 47 | LOG.info("scheduling update of: {}", registry.getClass()); 48 | fill(); 49 | lastUpdated = System.currentTimeMillis(); 50 | } catch (Throwable e) { 51 | LOG.error("Error during refresh of {}", registry.getClass(), e); 52 | } 53 | } 54 | 55 | public T get(I id) { 56 | T v = currentMap.get(id); 57 | if (null == v) { 58 | return getNullObject(); 59 | } 60 | return v; 61 | } 62 | 63 | public Collection get() { 64 | return currentMap.values(); 65 | } 66 | 67 | public Map getCurrentMap() { 68 | return currentMap; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/CeleryBody.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.Parameter; 4 | 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | /** 11 | * Created by jmussler on 3/31/15. 12 | */ 13 | public class CeleryBody { 14 | 15 | public String expires; 16 | public final List args = new ArrayList<>(); // Order: CeleryCommandArg, CeleryAlertArg* 17 | public String id; 18 | public String task = "check_and_notify"; 19 | public final List timelimit = new ArrayList<>(2); 20 | public final Map kwargs = new HashMap<>(); 21 | 22 | public static class CeleryCommandArg { 23 | public int check_id; 24 | public Long interval; 25 | public Map entity; 26 | public String check_name; 27 | public String command; 28 | public double schedule_time; 29 | } 30 | 31 | public static class CeleryAlertArg { 32 | public String period; 33 | public List notifications = new ArrayList<>(); 34 | public int id; 35 | public String condition; 36 | public String name; 37 | public Map parameters; 38 | public int check_id; 39 | public List> entities_map; 40 | public String responsible_team; 41 | public int priority = 1; 42 | public String team; 43 | public List tags = new ArrayList<>(); 44 | } 45 | 46 | public static class TrialRunCeleryCommand { 47 | public String check_id; 48 | public Long interval; 49 | public Map entity; 50 | public String check_name; 51 | public String command; 52 | public double schedule_time; 53 | } 54 | 55 | public static class TrialRunCeleryAlertArg { 56 | public String period; 57 | public List notifications = new ArrayList<>(); 58 | public String id; 59 | public String condition; 60 | public String name; 61 | public Map parameters; 62 | public String check_id; 63 | public List> entities_map; 64 | public String responsible_team; 65 | public int priority = 1; 66 | public String team; 67 | } 68 | } 69 | 70 | /* 71 | { 72 | "expires": "2014-08-05T13:26:39.873547+00:00", 73 | "utc": true, 74 | "args": [ 75 | { 76 | "check_id": 510, 77 | "interval": 30, 78 | "entity": { 79 | "data_center_code": "GTH", 80 | "host": "1.1.1.1", 81 | "type": "loadbalancers", 82 | "id": "gth-lv1-zal-de02" 83 | }, 84 | "check_name": "LB Load", 85 | "command": "snmp(community='ZAL_readonly').load()", 86 | "schedule_time": 1407245139.8733 87 | }, 88 | [ 89 | { 90 | "period": "", 91 | "notifications": [ 92 | 93 | ], 94 | "id": 1189, 95 | "condition": "capture(value['load1']) > 10 or capture(value['load5']) > 8 or capture(value['load15']) > 6", 96 | "name": "LB - load", 97 | "parameters": { 98 | 99 | }, 100 | "check_id": 510, 101 | "entities_map": [ 102 | { 103 | "type": "loadbalancers" 104 | } 105 | ], 106 | "responsible_team": "Platform\/System", 107 | "priority": 1, 108 | "team": "Platform\/System" 109 | }, 110 | { 111 | "period": "", 112 | "notifications": [ 113 | 114 | ], 115 | "id": 1190, 116 | "condition": "capture(value['load1']) > 5 or capture(value['load5']) > 4 or capture(value['load15']) > 4", 117 | "name": "LB - load warn", 118 | "parameters": { 119 | 120 | }, 121 | "check_id": 510, 122 | "entities_map": [ 123 | { 124 | "type": "loadbalancers" 125 | } 126 | ], 127 | "responsible_team": "Platform\/System", 128 | "priority": 3, 129 | "team": "Platform\/System" 130 | }, 131 | { 132 | "period": "", 133 | "notifications": [ 134 | 135 | ], 136 | "id": 1501, 137 | "condition": "capture(value['load1']) > 10 or capture(value['load5']) > 8 or capture(value['load15']) > 6", 138 | "name": "COP: LB - load - GTH", 139 | "parameters": { 140 | 141 | }, 142 | "check_id": 510, 143 | "entities_map": [ 144 | { 145 | "data_center_code": "GTH", 146 | "type": "loadbalancers" 147 | } 148 | ], 149 | "responsible_team": "Platform\/System", 150 | "priority": 1, 151 | "team": "Incident\/COP" 152 | } 153 | ] 154 | ], 155 | "chord": null, 156 | "callbacks": null, 157 | "errbacks": null, 158 | "taskset": null, 159 | "id": "check-510-gth-lv1-zal-de02-1407245139.87", 160 | "retries": 0, 161 | "task": "check_and_notify", 162 | "timelimit": [ 163 | 60, 164 | 30 165 | ], 166 | "eta": null, 167 | "kwargs": { 168 | 169 | } 170 | } 171 | */ 172 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/CeleryWriter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.fasterxml.jackson.databind.node.ObjectNode; 6 | import io.opentracing.Span; 7 | import io.opentracing.Tracer; 8 | import io.opentracing.propagation.Format; 9 | import io.opentracing.propagation.TextMapInjectAdapter; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.xerial.snappy.Snappy; 13 | 14 | import java.io.IOException; 15 | import java.util.HashMap; 16 | import java.util.Map; 17 | 18 | public abstract class CeleryWriter { 19 | private static final String TRACE_OPERATION_NAME = "queue_processing"; 20 | private static final ObjectMapper mapper = new ObjectMapper(); 21 | private static final Logger LOG = LoggerFactory.getLogger(CeleryWriter.class); 22 | 23 | protected final Tracer tracer; 24 | 25 | public static CeleryWriter create(TaskSerializerType t, Tracer tracer) { 26 | switch (t) { 27 | case COMPRESSED: 28 | return new CompressedNestedWriter(tracer); 29 | case PLAIN: 30 | default: 31 | return new PlainWriter(tracer); 32 | } 33 | } 34 | 35 | private CeleryWriter(Tracer tracer) { 36 | this.tracer = tracer; 37 | } 38 | 39 | private static class PlainWriter extends CeleryWriter { 40 | private PlainWriter(final Tracer tracer) { 41 | super(tracer); 42 | } 43 | 44 | @Override 45 | public byte[] asCeleryTask(CeleryBody task) { 46 | final ObjectNode node = mapper.createObjectNode(); 47 | ObjectNode properties = node.putObject("properties"); 48 | properties.put("body_encoding", "nested"); 49 | 50 | try { 51 | Map map = getSpanData(tracer); 52 | properties.set("trace", mapper.valueToTree(map)); 53 | } catch (Throwable e) { 54 | LOG.error("Preparing a trace failed: {}", e.toString()); 55 | } 56 | 57 | try { 58 | node.putPOJO("body", task); 59 | return mapper.writeValueAsString(node).getBytes(); 60 | } catch (JsonProcessingException e) { 61 | LOG.error("Serialize failed for task {}: {}", task, e.getMessage()); 62 | return null; 63 | } 64 | } 65 | 66 | private Map getSpanData(Tracer tracer) throws JsonProcessingException { 67 | Span span = tracer.buildSpan(TRACE_OPERATION_NAME) 68 | .asChildOf(tracer.activeSpan()) 69 | .start(); 70 | 71 | Map map = new HashMap<>(); 72 | TextMapInjectAdapter textMap = new TextMapInjectAdapter(map); 73 | tracer.inject(span.context(), Format.Builtin.TEXT_MAP, textMap); 74 | span.finish(); 75 | 76 | return map; 77 | } 78 | } 79 | 80 | private static class CompressedNestedWriter extends PlainWriter { 81 | private CompressedNestedWriter(final Tracer tracer) { 82 | super(tracer); 83 | } 84 | 85 | @Override 86 | public byte[] asCeleryTask(CeleryBody task) { 87 | try { 88 | byte[] result = super.asCeleryTask(task); 89 | if (result == null) { 90 | return null; 91 | } 92 | 93 | return Snappy.compress(result); 94 | } catch (IOException ex) { 95 | LOG.error("Compression failed", ex); 96 | return null; 97 | } 98 | } 99 | } 100 | 101 | public abstract byte[] asCeleryTask(CeleryBody task); 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/Check.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 4 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 5 | import de.zalando.zmon.scheduler.ng.entities.Entity; 6 | 7 | /** 8 | * Created by jmussler on 30.06.16. 9 | */ 10 | public class Check { 11 | private final int id; 12 | private final CheckRepository checkRepo; 13 | 14 | public Check(int id, CheckRepository checkRepo) { 15 | this.id = id; 16 | this.checkRepo = checkRepo; 17 | } 18 | 19 | public CheckDefinition getCheckDefinition() { 20 | return checkRepo.get(id); 21 | } 22 | 23 | public boolean matchEntity(Entity entity) { 24 | return AlertOverlapGenerator.matchCheckFilter(getCheckDefinition(), entity); 25 | } 26 | 27 | public int getId() { 28 | return id; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/CommandSerializer.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 4 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 5 | import de.zalando.zmon.scheduler.ng.entities.Entity; 6 | import de.zalando.zmon.scheduler.ng.trailruns.TrialRunRequest; 7 | 8 | import io.opentracing.Tracer; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Collection; 12 | import java.util.Date; 13 | import java.util.List; 14 | 15 | /** 16 | * Created by jmussler on 30.06.16. 17 | */ 18 | public class CommandSerializer { 19 | 20 | private final CeleryWriter writer; 21 | 22 | public CommandSerializer(TaskSerializerType type, Tracer tracer) { 23 | this.writer = CeleryWriter.create(type, tracer); 24 | } 25 | 26 | public String expiresTime(long interval) { 27 | Date exp = new Date(System.currentTimeMillis()+(interval * 1000L)); 28 | return LocalDateFormatter.get().format(exp); 29 | } 30 | 31 | public byte[] writeTrialRun(Entity entity, TrialRunRequest request) { 32 | CeleryBody body = new CeleryBody(); 33 | 34 | body.expires = expiresTime(request.interval); // "2015-12-31T00:00:00.000+00:00" 35 | body.id="check-TR:"+request.id+"-"+entity.getId()+"-"+System.currentTimeMillis(); 36 | 37 | body.timelimit.add(request.interval); 38 | body.timelimit.add(request.interval * 2L); 39 | 40 | CeleryBody.TrialRunCeleryCommand command = new CeleryBody.TrialRunCeleryCommand(); 41 | command.check_id = "TR:"+request.id; 42 | command.check_name = request.name; 43 | command.interval = request.interval; 44 | command.command = request.checkCommand; 45 | command.entity = entity.getProperties(); 46 | body.args.add(command); 47 | 48 | List alertList = new java.util.ArrayList<>(); 49 | body.args.add(alertList); 50 | 51 | CeleryBody.TrialRunCeleryAlertArg alertArg = new CeleryBody.TrialRunCeleryAlertArg(); 52 | alertList.add(alertArg); 53 | 54 | alertArg.id = "TR:" + request.id; 55 | alertArg.check_id = "TR:" + request.id; 56 | alertArg.condition = request.alertCondition; 57 | alertArg.name = request.name; 58 | alertArg.period = request.period; 59 | if(alertArg.period == null) { 60 | alertArg.period = ""; 61 | } 62 | alertArg.team = "TRIAL RUN"; 63 | alertArg.responsible_team = "TRIAL RUN"; 64 | alertArg.parameters = request.parameters; 65 | alertArg.entities_map = request.entities; 66 | 67 | body.task = "trial_run"; 68 | 69 | return writer.asCeleryTask(body); 70 | } 71 | 72 | public byte[] write(Entity entity, Check check, Collection alerts, long scheduledTime) { 73 | CeleryBody body = new CeleryBody(); 74 | CheckDefinition checkDef = check.getCheckDefinition(); 75 | 76 | body.expires = expiresTime(checkDef.getInterval()); // "2015-12-31T00:00:00.000+00:00" 77 | body.id = "check-" + check.getId() + "-" + entity.getId() + "-" + System.currentTimeMillis(); 78 | 79 | body.timelimit.add(checkDef.getInterval()); 80 | body.timelimit.add(checkDef.getInterval() * 2); 81 | 82 | CeleryBody.CeleryCommandArg command = new CeleryBody.CeleryCommandArg(); 83 | command.check_id = check.getId(); 84 | command.check_name = checkDef.getName(); 85 | command.interval = checkDef.getInterval(); 86 | command.command = checkDef.getCommand(); 87 | command.entity = entity.getProperties(); 88 | command.schedule_time = ((double)scheduledTime) / 1000.0; 89 | body.args.add(command); 90 | 91 | List alertList = new ArrayList<>(); 92 | body.args.add(alertList); 93 | 94 | for(Alert alert : alerts) { 95 | CeleryBody.CeleryAlertArg alertArg = new CeleryBody.CeleryAlertArg(); 96 | AlertDefinition alertDef = alert.getAlertDefinition(); 97 | 98 | alertArg.id = alert.getId(); 99 | alertArg.check_id = check.getId(); 100 | alertArg.condition = alertDef.getCondition(); 101 | alertArg.name = alertDef.getName(); 102 | alertArg.notifications = alertDef.getNotifications(); 103 | alertArg.period = alertDef.getPeriod(); 104 | alertArg.priority = alertDef.getPriority(); 105 | alertArg.tags = alertDef.getTags(); 106 | 107 | if(alertArg.period == null) { 108 | alertArg.period = ""; 109 | } 110 | 111 | alertArg.team = alertDef.getTeam(); 112 | alertArg.responsible_team = alertDef.getResponsibleTeam(); 113 | alertArg.parameters = alertDef.getParameters(); 114 | alertArg.entities_map = alertDef.getEntities(); 115 | 116 | alertList.add(alertArg); 117 | } 118 | 119 | return writer.asCeleryTask(body); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/DataCenterSubscriber.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import de.zalando.zmon.scheduler.ng.entities.Entity; 4 | import de.zalando.zmon.scheduler.ng.entities.EntityChangeListener; 5 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 6 | 7 | import java.util.*; 8 | 9 | /** 10 | * Created by jmussler on 18.06.16. 11 | */ 12 | public class DataCenterSubscriber implements EntityChangeListener { 13 | 14 | private final Map> pendingTasks = new HashMap<>(); 15 | private final Collection emptyList = new ArrayList<>(0); 16 | private final boolean enabled; 17 | 18 | public DataCenterSubscriber(boolean enabled) { 19 | this.enabled = enabled; 20 | } 21 | 22 | public void forwardRequest(T task) { 23 | if (!enabled) return; 24 | 25 | synchronized (this) { 26 | for (String k : pendingTasks.keySet()) { 27 | pendingTasks.get(k).add(task); 28 | } 29 | } 30 | } 31 | 32 | public Collection getRequests(String dcId) { 33 | synchronized (this) { 34 | if (!pendingTasks.containsKey(dcId)) { 35 | return emptyList; 36 | } 37 | 38 | List tasks = new ArrayList<>(pendingTasks.get(dcId)); 39 | pendingTasks.get(dcId).clear(); 40 | return tasks; 41 | } 42 | } 43 | 44 | public Collection getKnwonDCs() { 45 | synchronized (this) { 46 | return pendingTasks.keySet(); 47 | } 48 | } 49 | 50 | @Override 51 | public void notifyEntityAdd(EntityRepository repo, Entity e) { 52 | if (e.getFilterProperties().get("type").equals("local")) { 53 | // local entities depict remote DCs 54 | if (!pendingTasks.containsKey(e.getId())) { 55 | synchronized (this) { 56 | if (!pendingTasks.containsKey(e.getId())) { 57 | pendingTasks.put(e.getId(), new ArrayList<>()); 58 | } 59 | } 60 | } 61 | } 62 | } 63 | 64 | @Override 65 | public void notifyEntityRemove(EntityRepository repo, Entity e) { 66 | if (e.getFilterProperties().get("type").equals("local")) { 67 | // local entities depict remote DCs 68 | if (pendingTasks.containsKey(e.getId())) { 69 | synchronized (this) { 70 | if (pendingTasks.containsKey(e.getId())) { 71 | pendingTasks.remove(e.getId()); 72 | } 73 | } 74 | } 75 | } 76 | } 77 | 78 | @Override 79 | public void notifyEntityChange(EntityRepository repo, Entity oldEntity, Entity newEntity) { 80 | 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/DefinitionStatus.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | /** 4 | * Created by jmussler on 4/2/15. 5 | */ 6 | public enum DefinitionStatus { 7 | ACTIVE, 8 | INACTIVE, 9 | REJECTED, 10 | DELETED 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/LocalDateFormatter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import java.text.SimpleDateFormat; 4 | 5 | /** 6 | * Created by jmussler on 5/4/15. 7 | */ 8 | public class LocalDateFormatter { 9 | private static final ThreadLocal THREAD_LOCAL_DATEFORMAT = new ThreadLocal() { 10 | protected SimpleDateFormat initialValue() { 11 | return new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); 12 | } 13 | }; 14 | 15 | public static SimpleDateFormat get() { 16 | return THREAD_LOCAL_DATEFORMAT.get(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/RedisResponseHolder.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import redis.clients.jedis.Response; 4 | 5 | /** 6 | * Created by jmussler on 21.06.16. 7 | */ 8 | 9 | public final class RedisResponseHolder { 10 | 11 | private final K key; 12 | private final Response response; 13 | 14 | private RedisResponseHolder(final K key, final Response response) { 15 | this.key = key; 16 | this.response = response; 17 | } 18 | 19 | public K getKey() { 20 | return key; 21 | } 22 | 23 | public Response getResponse() { 24 | return response; 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | final StringBuilder builder = new StringBuilder(); 30 | builder.append("ResponseHolder [key="); 31 | builder.append(key); 32 | builder.append(", response="); 33 | builder.append(response); 34 | builder.append("]"); 35 | return builder.toString(); 36 | } 37 | 38 | public static RedisResponseHolder create(final K key, final Response response) { 39 | return new RedisResponseHolder<>(key, response); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/SchedulePersistType.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | /** 4 | * Created by jmussler on 4/8/15. 5 | */ 6 | public enum SchedulePersistType { 7 | DISABLED, 8 | FILE, 9 | REDIS 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/Source.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import java.util.Collection; 4 | 5 | /** 6 | * Created by jmussler on 4/7/15. 7 | */ 8 | public interface Source { 9 | String getName(); 10 | 11 | Collection getCollection(); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/SourceRegistry.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import java.util.Collection; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | /** 8 | * Created by jmussler on 4/7/15. 9 | */ 10 | public class SourceRegistry> { 11 | 12 | private final Map registry = new HashMap<>(); 13 | 14 | public void register(T source) { 15 | registry.put(source.getName(), source); 16 | } 17 | 18 | public T get(String s) { 19 | return registry.get(s); 20 | } 21 | 22 | public Collection getSourceNames() { 23 | return registry.keySet(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/TaskSerializerType.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | /** 4 | * Created by jmussler on 30.04.15. 5 | */ 6 | public enum TaskSerializerType { 7 | COMPRESSED, 8 | PLAIN 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/TaskWriterType.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | /** 4 | * Created by jmussler on 4/17/15. 5 | */ 6 | public enum TaskWriterType { 7 | REDIS, 8 | LOG, 9 | ARRAY_LIST 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/TokenWrapper.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import org.zalando.stups.tokens.AccessTokens; 4 | 5 | /** 6 | * Created by jmussler on 26.01.16. 7 | */ 8 | public class TokenWrapper { 9 | 10 | private final String token; 11 | private final AccessTokens tokens; 12 | private final String tokenId; 13 | 14 | public TokenWrapper(AccessTokens tokens, String tokenId) { 15 | this.tokenId = tokenId; 16 | this.tokens = tokens; 17 | token = null; 18 | } 19 | 20 | public TokenWrapper(String token) { 21 | this.tokenId = null; 22 | this.tokens = null; 23 | this.token = token; 24 | } 25 | 26 | public String get() { 27 | if (tokens != null) { 28 | return tokens.get(tokenId); 29 | } 30 | return token; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/alerts/AlertChangeListener.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | /** 4 | * Created by jmussler on 4/17/15. 5 | */ 6 | public interface AlertChangeListener { 7 | void notifyAlertNew(AlertDefinition alert); 8 | 9 | void notifyAlertChange(AlertDefinition alert); 10 | 11 | void notifyAlertDelete(AlertDefinition alert); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/alerts/AlertDefinitions.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | import javax.xml.bind.annotation.XmlAccessType; 4 | import javax.xml.bind.annotation.XmlAccessorType; 5 | import javax.xml.bind.annotation.XmlElement; 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | /** 10 | * Created by jmussler on 4/2/15. 11 | */ 12 | @XmlAccessorType(XmlAccessType.NONE) 13 | public class AlertDefinitions { 14 | 15 | @XmlElement 16 | private Long snapshotId; 17 | 18 | @XmlElement 19 | private List alertDefinitions = new ArrayList<>(); 20 | 21 | public Long getSnapshotId() { 22 | return snapshotId; 23 | } 24 | 25 | public void setSnapshotId(final Long snapshotId) { 26 | this.snapshotId = snapshotId; 27 | } 28 | 29 | public List getAlertDefinitions() { 30 | return alertDefinitions; 31 | } 32 | 33 | public void setAlertDefinitions(final List alertDefinitions) { 34 | this.alertDefinitions = alertDefinitions; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | final StringBuilder builder = new StringBuilder(); 40 | builder.append("AlertDefinitions [snapshotId="); 41 | builder.append(snapshotId); 42 | builder.append(", alertDefinitions="); 43 | builder.append(alertDefinitions); 44 | builder.append("]"); 45 | return builder.toString(); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/alerts/AlertRepository.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | import de.zalando.zmon.scheduler.ng.CachedRepository; 4 | 5 | import io.opentracing.Tracer; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Component; 8 | 9 | import java.util.*; 10 | 11 | /** 12 | * Created by jmussler on 4/7/15. 13 | */ 14 | @Component 15 | public class AlertRepository extends CachedRepository { 16 | 17 | private Map> byCheckId; 18 | 19 | private final Set changeListeners = new HashSet<>(); 20 | 21 | public void registerChangeListener(AlertChangeListener listener) { 22 | changeListeners.add(listener); 23 | } 24 | 25 | @Override 26 | public synchronized void fill() { 27 | Map m = new HashMap<>(); 28 | Map> newByCheckId = new HashMap<>(); 29 | 30 | for (String name : registry.getSourceNames()) { 31 | Collection alerts = registry.get(name).getCollection(); 32 | if (null == alerts) { 33 | continue; 34 | } 35 | 36 | for (AlertDefinition ad : alerts) { 37 | m.put(ad.getId(), ad); 38 | if (newByCheckId.containsKey(ad.getCheckDefinitionId())) { 39 | newByCheckId.get(ad.getCheckDefinitionId()).add(ad); 40 | } else { 41 | List ads = new ArrayList<>(1); 42 | ads.add(ad); 43 | newByCheckId.put(ad.getCheckDefinitionId(), ads); 44 | } 45 | } 46 | } 47 | 48 | List changedAlerts = new ArrayList<>(); 49 | List deletedAlerts = new ArrayList<>(); 50 | List addedAlerts = new ArrayList<>(); 51 | 52 | for (Map.Entry e : m.entrySet()) { 53 | if (currentMap.containsKey(e.getKey())) { 54 | if (currentMap.get(e.getKey()).compareForAlertUpdate(e.getValue())) { 55 | changedAlerts.add(e.getValue()); 56 | } 57 | } else { 58 | addedAlerts.add(e.getValue()); 59 | } 60 | } 61 | 62 | for (Map.Entry e : currentMap.entrySet()) { 63 | if (!m.containsKey(e.getKey())) { 64 | deletedAlerts.add(e.getValue()); 65 | } 66 | } 67 | 68 | byCheckId = newByCheckId; 69 | currentMap = m; 70 | 71 | // we notify after update with the new state 72 | // main purpose is now delayed cleanup of alert filter changes 73 | for (AlertChangeListener l : changeListeners) { 74 | for (AlertDefinition ad : changedAlerts) { 75 | l.notifyAlertChange(ad); 76 | } 77 | } 78 | } 79 | 80 | @Override 81 | protected AlertDefinition getNullObject() { 82 | return NULL_OBJ; 83 | } 84 | 85 | private static final AlertDefinition NULL_OBJ; 86 | 87 | static { 88 | NULL_OBJ = new AlertDefinition(); 89 | NULL_OBJ.setCheckDefinitionId(0); 90 | NULL_OBJ.setEntities(new ArrayList<>(0)); 91 | NULL_OBJ.setEntitiesExclude(new ArrayList<>(0)); 92 | } 93 | 94 | @Autowired 95 | public AlertRepository(AlertSourceRegistry registry, Tracer tracer) { 96 | super(registry, tracer); 97 | currentMap = new HashMap<>(); 98 | byCheckId = new HashMap<>(); 99 | fill(); 100 | } 101 | 102 | private final static List EMPTY_LIST = new ArrayList<>(0); 103 | 104 | public Collection getByCheckId(Integer id) { 105 | if (byCheckId.containsKey(id)) { 106 | return byCheckId.get(id); 107 | } else { 108 | return EMPTY_LIST; 109 | } 110 | } 111 | 112 | public Map> getByCheckId() { 113 | return byCheckId; 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/alerts/AlertSource.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | import de.zalando.zmon.scheduler.ng.BaseSource; 4 | 5 | /** 6 | * Created by jmussler on 4/2/15. 7 | */ 8 | public abstract class AlertSource extends BaseSource { 9 | public AlertSource(String name) { 10 | super(name); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/alerts/AlertSourceRegistry.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.stereotype.Component; 5 | import org.springframework.web.client.RestTemplate; 6 | 7 | import com.codahale.metrics.MetricRegistry; 8 | 9 | import de.zalando.zmon.scheduler.ng.SourceRegistry; 10 | import de.zalando.zmon.scheduler.ng.TokenWrapper; 11 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 12 | 13 | /** 14 | * Created by jmussler on 4/7/15. 15 | */ 16 | @Component 17 | public class AlertSourceRegistry extends SourceRegistry { 18 | 19 | @Autowired 20 | public AlertSourceRegistry(final SchedulerConfig config, final MetricRegistry metrics, final TokenWrapper tokens, RestTemplate restTemplate) { 21 | 22 | final String url = config.getControllerUrl() + "/api/v1/checks/all-active-alert-definitions"; 23 | final DefaultAlertSource source = new DefaultAlertSource("alert-source", url, metrics, restTemplate); 24 | 25 | register(source); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/alerts/DefaultAlertSource.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | import com.codahale.metrics.MetricRegistry; 4 | import com.codahale.metrics.Timer; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.http.HttpEntity; 9 | import org.springframework.http.HttpHeaders; 10 | import org.springframework.http.HttpMethod; 11 | import org.springframework.http.ResponseEntity; 12 | import org.springframework.web.client.RestTemplate; 13 | 14 | import java.util.Collection; 15 | 16 | /** 17 | * Created by jmussler on 4/7/15. 18 | */ 19 | public class DefaultAlertSource extends AlertSource { 20 | 21 | private final Timer timer; 22 | 23 | private final String url; 24 | private final RestTemplate restTemplate; 25 | private boolean isFirstLoad = true; 26 | 27 | private Collection lastResults = null; 28 | private long lastResultMaxLastModified = 0; 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(DefaultAlertSource.class); 31 | 32 | @Autowired 33 | public DefaultAlertSource(final String name, final String url, final MetricRegistry metrics, final RestTemplate restTemplate) { 34 | super(name); 35 | this.restTemplate = restTemplate; 36 | LOG.info("configuring alert source url={}", url); 37 | this.url = url; 38 | this.timer = metrics.timer("alert-adapter." + name); 39 | } 40 | 41 | @Override 42 | public Collection getCollection() { 43 | AlertDefinitions defs = new AlertDefinitions(); 44 | try { 45 | LOG.info("Querying alert definitions with token... "); 46 | final HttpEntity request = new HttpEntity<>(new HttpHeaders()); 47 | 48 | HttpHeaders headers = restTemplate.headForHeaders(url); 49 | if (headers.containsKey("Last-Modified")) { 50 | if (!doRefresh(headers.get("Last-Modified").get(0), lastResultMaxLastModified, lastResults)) { 51 | LOG.info("Skipping alert update ... cache={} head={}", lastResultMaxLastModified, headers.get("Last-Modified")); 52 | return lastResults; 53 | } 54 | } 55 | 56 | ResponseEntity response; 57 | Timer.Context ct = timer.time(); 58 | response = restTemplate.exchange(url, HttpMethod.GET, request, AlertDefinitions.class); 59 | ct.stop(); 60 | defs = response.getBody(); 61 | 62 | LOG.info("Got {} alerts from {}", defs.getAlertDefinitions().size(), getName()); 63 | isFirstLoad = false; 64 | } catch (Throwable t) { 65 | LOG.error("Failed to get alert definitions: {}", t.getMessage()); 66 | if (!isFirstLoad) { 67 | // rethrow so that currently alerts are still used not not replaced by empty list 68 | throw t; 69 | } 70 | } 71 | 72 | lastResults = defs.getAlertDefinitions(); 73 | lastResultMaxLastModified = lastResults.stream().map(AlertDefinition::getLastModified).reduce(0L, Math::max); 74 | return defs.getAlertDefinitions(); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/alerts/Parameter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | import javax.validation.constraints.NotNull; 4 | 5 | /** 6 | * Created by jmussler on 4/2/15. 7 | */ 8 | public class Parameter { 9 | 10 | @NotNull(message = "value of the parameter is mandatory") 11 | private Object value; 12 | private String comment; 13 | private String type; 14 | 15 | public Parameter() { 16 | } 17 | 18 | public Parameter(final Object value, final String comment, final String type) { 19 | this.value = value; 20 | this.comment = comment; 21 | this.type = type; 22 | } 23 | 24 | public Object getValue() { 25 | return value; 26 | } 27 | 28 | public void setValue(final Object value) { 29 | this.value = value; 30 | } 31 | 32 | public String getComment() { 33 | return comment; 34 | } 35 | 36 | public void setComment(final String comment) { 37 | this.comment = comment; 38 | } 39 | 40 | public String getType() { 41 | return type; 42 | } 43 | 44 | public void setType(final String type) { 45 | this.type = type; 46 | } 47 | 48 | @Override 49 | public int hashCode() { 50 | final int prime = 31; 51 | int result = 1; 52 | result = prime * result + ((comment == null) ? 0 : comment.hashCode()); 53 | result = prime * result + ((type == null) ? 0 : type.hashCode()); 54 | result = prime * result + ((value == null) ? 0 : value.hashCode()); 55 | return result; 56 | } 57 | 58 | @Override 59 | public boolean equals(final Object obj) { 60 | if (this == obj) { 61 | return true; 62 | } 63 | 64 | if (obj == null) { 65 | return false; 66 | } 67 | 68 | if (getClass() != obj.getClass()) { 69 | return false; 70 | } 71 | 72 | final Parameter other = (Parameter) obj; 73 | if (comment == null) { 74 | if (other.comment != null) { 75 | return false; 76 | } 77 | } else if (!comment.equals(other.comment)) { 78 | return false; 79 | } 80 | 81 | if (type == null) { 82 | if (other.type != null) { 83 | return false; 84 | } 85 | } else if (!type.equals(other.type)) { 86 | return false; 87 | } 88 | 89 | if (value == null) { 90 | if (other.value != null) { 91 | return false; 92 | } 93 | } else if (!value.equals(other.value)) { 94 | return false; 95 | } 96 | 97 | return true; 98 | } 99 | 100 | @Override 101 | public String toString() { 102 | final StringBuilder sb = new StringBuilder("Parameter{"); 103 | sb.append("value=").append(value); 104 | sb.append(", comment='").append(comment).append('\''); 105 | sb.append(", type='").append(type).append('\''); 106 | sb.append('}'); 107 | return sb.toString(); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/alerts/YamlAlertSource.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 6 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.io.File; 11 | import java.util.ArrayList; 12 | import java.util.Collection; 13 | import java.util.List; 14 | 15 | /** 16 | * Created by jmussler on 4/17/15. 17 | */ 18 | public class YamlAlertSource extends AlertSource { 19 | 20 | private static final Logger LOG = LoggerFactory.getLogger(YamlAlertSource.class); 21 | 22 | private String fileName; 23 | 24 | private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); 25 | 26 | static { 27 | mapper.setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); 28 | } 29 | 30 | public YamlAlertSource(String name, String fileName) { 31 | super(name); 32 | this.fileName = fileName; 33 | } 34 | 35 | @Override 36 | public Collection getCollection() { 37 | try { 38 | List list = mapper.readValue(new File(fileName), new TypeReference>() { 39 | }); 40 | return list; 41 | } catch (Exception e) { 42 | LOG.error("", e); 43 | } 44 | return new ArrayList<>(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/CheckChangeListener.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | /** 4 | * Created by jmussler on 4/17/15. 5 | */ 6 | public interface CheckChangeListener { 7 | void notifyNewCheck(CheckRepository repo, int checkId); 8 | 9 | void notifyCheckIntervalChange(CheckRepository repo, int checkId); 10 | 11 | void notifyDeleteCheck(CheckRepository repo, int checkId); 12 | 13 | void notifyFilterChange(int checkId); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/CheckChangedListener.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | import de.zalando.zmon.scheduler.ng.scheduler.Scheduler; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | /** 8 | * Created by jmussler on 18.06.16. 9 | */ 10 | public class CheckChangedListener implements CheckChangeListener { 11 | 12 | private final Scheduler scheduler; 13 | private final Logger log = LoggerFactory.getLogger(CheckChangedListener.class); 14 | 15 | public CheckChangedListener(Scheduler scheduler) { 16 | this.scheduler = scheduler; 17 | } 18 | 19 | @Override 20 | public void notifyNewCheck(CheckRepository repo, int checkId) { 21 | log.info("New check discovered: checkId={}", checkId); 22 | scheduler.schedule(checkId, 0); 23 | } 24 | 25 | @Override 26 | public void notifyCheckIntervalChange(CheckRepository repo, int checkId) { 27 | log.info("Check interval changed: checkId={}", checkId); 28 | scheduler.executeImmediate(checkId); 29 | } 30 | 31 | @Override 32 | public void notifyDeleteCheck(CheckRepository repo, int checkId) { 33 | log.info("Check removed or inactive: checkId={}", checkId); 34 | scheduler.unschedule(checkId); 35 | } 36 | 37 | @Override 38 | public void notifyFilterChange(int checkId) { 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/CheckDefinitions.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | /** 4 | * Created by jmussler on 4/2/15. 5 | */ 6 | 7 | import javax.xml.bind.annotation.XmlAccessType; 8 | import javax.xml.bind.annotation.XmlAccessorType; 9 | import javax.xml.bind.annotation.XmlElement; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | @XmlAccessorType(XmlAccessType.NONE) 14 | public class CheckDefinitions { 15 | 16 | @XmlElement 17 | private Long snapshotId; 18 | 19 | @XmlElement 20 | private List checkDefinitions = new ArrayList<>(); 21 | 22 | public Long getSnapshotId() { 23 | return snapshotId; 24 | } 25 | 26 | public void setSnapshotId(final Long snapshotId) { 27 | this.snapshotId = snapshotId; 28 | } 29 | 30 | public List getCheckDefinitions() { 31 | return checkDefinitions; 32 | } 33 | 34 | public void setCheckDefinitions(final List checkDefinitions) { 35 | this.checkDefinitions = checkDefinitions; 36 | } 37 | 38 | @Override 39 | public String toString() { 40 | final StringBuilder builder = new StringBuilder(); 41 | builder.append("CheckDefinitions [snapshotId="); 42 | builder.append(snapshotId); 43 | builder.append(", checkDefinitions="); 44 | builder.append(checkDefinitions); 45 | builder.append("]"); 46 | return builder.toString(); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/CheckRepository.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | import de.zalando.zmon.scheduler.ng.CachedRepository; 4 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 5 | 6 | import io.opentracing.Tracer; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.*; 11 | 12 | /** 13 | * Created by jmussler on 4/7/15. 14 | */ 15 | @Component 16 | public class CheckRepository extends CachedRepository { 17 | 18 | private final Set listeners = new HashSet<>(); 19 | private final MinIntervalEntityFetcher minIntervalEntityFetcher; 20 | 21 | public synchronized void registerListener(CheckChangeListener l) { 22 | listeners.add(l); 23 | } 24 | 25 | private void update(Map oldMap, Map newMap) { 26 | List intervalChanged = new ArrayList<>(); 27 | List newChecks = new ArrayList<>(); 28 | List removedChecks = new ArrayList<>(); 29 | 30 | for (Integer id : oldMap.keySet()) { 31 | if (newMap.containsKey(id)) { 32 | if (!oldMap.get(id).getInterval().equals(newMap.get(id).getInterval())) { 33 | intervalChanged.add(id); 34 | } 35 | } else { 36 | removedChecks.add(id); 37 | } 38 | } 39 | 40 | for (Integer id : newMap.keySet()) { 41 | if (!oldMap.containsKey(id)) { 42 | newChecks.add(id); 43 | } 44 | } 45 | 46 | for (CheckChangeListener l : listeners) { 47 | for (Integer id : newChecks) { 48 | l.notifyNewCheck(this, id); 49 | } 50 | 51 | for (Integer id : removedChecks) { 52 | l.notifyDeleteCheck(this, id); 53 | } 54 | 55 | for (Integer id : intervalChanged) { 56 | l.notifyCheckIntervalChange(this, id); 57 | } 58 | } 59 | } 60 | 61 | @Override 62 | public synchronized void fill() { 63 | Map m = new HashMap<>(); 64 | 65 | minIntervalEntityFetcher.fetch(); 66 | MinIntervalEntityFetcher.MinCheckIntervalData minIntervalData = minIntervalEntityFetcher.getCheckInterval(); 67 | 68 | for (String name : registry.getSourceNames()) { 69 | for (CheckDefinition cd : registry.get(name).getCollection()) { 70 | Integer id = cd.getId(); 71 | Long interval = cd.getInterval(); 72 | if (interval < minIntervalData.getMinCheckInterval()) { 73 | if (minIntervalData.getWhitelistedChecks().contains(id)) { 74 | if (interval < minIntervalData.getMinWhitelistedCheckInterval()) { 75 | cd.setInterval(minIntervalData.getMinWhitelistedCheckInterval()); 76 | } 77 | } else { 78 | cd.setInterval(minIntervalData.getMinCheckInterval()); 79 | } 80 | } 81 | m.put(id, cd); 82 | } 83 | } 84 | Map oldMap = currentMap; 85 | currentMap = m; // switching here to new map instance before notifications 86 | update(oldMap, currentMap); 87 | } 88 | 89 | @Override 90 | protected CheckDefinition getNullObject() { 91 | return null; 92 | } 93 | 94 | @Autowired 95 | public CheckRepository(CheckSourceRegistry registry, Tracer tracer, MinIntervalEntityFetcher minIntervalEntityFetcher) { 96 | super(registry, tracer); 97 | this.currentMap = new HashMap<>(); 98 | this.minIntervalEntityFetcher = minIntervalEntityFetcher; 99 | fill(); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/CheckSource.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | import de.zalando.zmon.scheduler.ng.BaseSource; 4 | 5 | /** 6 | * Created by jmussler on 3/31/15. 7 | */ 8 | public abstract class CheckSource extends BaseSource { 9 | public CheckSource(String name) { 10 | super(name); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/CheckSourceRegistry.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | import com.codahale.metrics.MetricRegistry; 4 | import de.zalando.zmon.scheduler.ng.*; 5 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Component; 8 | import org.springframework.web.client.RestTemplate; 9 | 10 | /** 11 | * Created by jmussler on 4/2/15. 12 | */ 13 | @Component 14 | public class CheckSourceRegistry extends SourceRegistry { 15 | 16 | public CheckSourceRegistry(MetricRegistry metrics) { 17 | 18 | } 19 | 20 | @Autowired 21 | public CheckSourceRegistry(SchedulerConfig config, final TokenWrapper tokens, RestTemplate restTemplate) { 22 | final String url = config.getControllerUrl() + "/api/v1/checks/all-active-check-definitions"; 23 | final DefaultCheckSource source = new DefaultCheckSource("check-source", url, restTemplate); 24 | register(source); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/DefaultCheckSource.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.http.*; 7 | import org.springframework.web.client.RestTemplate; 8 | 9 | import java.util.Collection; 10 | 11 | /** 12 | * Created by jmussler on 4/2/15. 13 | */ 14 | public class DefaultCheckSource extends CheckSource { 15 | 16 | private final static Logger LOG = LoggerFactory.getLogger(DefaultCheckSource.class); 17 | 18 | private String url; 19 | private boolean isFirstLoad = true; 20 | private final RestTemplate restTemplate; 21 | private Collection lastResults = null; 22 | private long lastResultMaxLastModified = 0; 23 | 24 | @Autowired 25 | public DefaultCheckSource(String name, String url, final RestTemplate restTemplate) { 26 | super(name); 27 | this.restTemplate = restTemplate; 28 | LOG.info("configuring check source url={}", url); 29 | 30 | this.url = url; 31 | } 32 | 33 | @Override 34 | public Collection getCollection() { 35 | CheckDefinitions defs = new CheckDefinitions(); 36 | try { 37 | LOG.info("Querying check definitions with token..."); 38 | HttpEntity request = new HttpEntity<>(new HttpHeaders()); 39 | 40 | HttpHeaders headers = restTemplate.headForHeaders(url); 41 | if (headers.containsKey("Last-Modified")) { 42 | if (!doRefresh(headers.get("Last-Modified").get(0), lastResultMaxLastModified, lastResults)) { 43 | LOG.info("Skipping check update ...{}", headers.get("Last-Modified")); 44 | return lastResults; 45 | } 46 | } 47 | 48 | ResponseEntity response; 49 | response = restTemplate.exchange(url, HttpMethod.GET, request, CheckDefinitions.class); 50 | defs = response.getBody(); 51 | LOG.info("Got {} checks from {}", defs.getCheckDefinitions().size(), getName()); 52 | isFirstLoad = false; 53 | } catch (Throwable t) { 54 | LOG.error("Failed to get check definitions: {}", t.getMessage()); 55 | if (!isFirstLoad) { 56 | // rethrow so that currently used checks are still used and not replaced by empty list 57 | throw t; 58 | } 59 | } 60 | 61 | lastResults = defs.getCheckDefinitions(); 62 | lastResultMaxLastModified = lastResults.stream().map(CheckDefinition::getLastModified).reduce(0L, Math::max); 63 | return defs.getCheckDefinitions(); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/MinIntervalEntityFetcher.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | import com.codahale.metrics.Meter; 4 | import com.codahale.metrics.MetricRegistry; 5 | import com.codahale.metrics.Timer; 6 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 7 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.core.ParameterizedTypeReference; 12 | import org.springframework.http.HttpEntity; 13 | import org.springframework.http.HttpHeaders; 14 | import org.springframework.http.HttpMethod; 15 | import org.springframework.http.ResponseEntity; 16 | import org.springframework.stereotype.Component; 17 | import org.springframework.web.client.RestTemplate; 18 | 19 | import java.io.UnsupportedEncodingException; 20 | import java.net.URI; 21 | import java.net.URLEncoder; 22 | import java.util.Collections; 23 | import java.util.List; 24 | 25 | @Component 26 | public class MinIntervalEntityFetcher { 27 | 28 | private final static Logger LOG = LoggerFactory.getLogger(MinIntervalEntityFetcher.class); 29 | 30 | private final URI entityServiceUrl; 31 | private final RestTemplate restTemplate; 32 | private final Timer timer; 33 | private final Meter totalFetches; 34 | private final Meter totalErrors; 35 | private MinCheckIntervalData checkInterval; 36 | 37 | private final String path = "/api/v1/entities?query="; 38 | private final String query = "{\"id\":\"zmon-min-check-interval\",\"type\":\"zmon_config\"}"; 39 | 40 | @Autowired 41 | public MinIntervalEntityFetcher(SchedulerConfig config, RestTemplate restTemplate, MetricRegistry metrics) throws UnsupportedEncodingException { 42 | this.restTemplate = restTemplate; 43 | this.timer = metrics.timer("interval-fetcher.timer"); 44 | this.totalFetches = metrics.meter("interval-fetcher.total-fetches"); 45 | this.totalErrors = metrics.meter("interval-fetcher.total-errors"); 46 | 47 | this.entityServiceUrl = URI.create(config.getEntityServiceUrl() + path + URLEncoder.encode(query, "UTF-8")); 48 | 49 | this.checkInterval = new MinCheckIntervalData(); 50 | } 51 | 52 | @JsonIgnoreProperties(ignoreUnknown = true) 53 | public static class MinCheckIntervalData { 54 | private List whitelistedChecks = Collections.emptyList(); 55 | private Long minCheckInterval = 15L; 56 | private Long minWhitelistedCheckInterval = 15L; 57 | 58 | public List getWhitelistedChecks() { 59 | return whitelistedChecks; 60 | } 61 | 62 | public void setWhitelistedChecks(List whitelistedChecks) { 63 | this.whitelistedChecks = whitelistedChecks; 64 | } 65 | 66 | public Long getMinCheckInterval() { 67 | return minCheckInterval; 68 | } 69 | 70 | public void setMinCheckInterval(Long minCheckInterval) { 71 | this.minCheckInterval = minCheckInterval; 72 | } 73 | 74 | public Long getMinWhitelistedCheckInterval() { 75 | return minWhitelistedCheckInterval; 76 | } 77 | 78 | public void setMinWhitelistedCheckInterval(Long minWhitelistedCheckInterval) { 79 | this.minWhitelistedCheckInterval = minWhitelistedCheckInterval; 80 | } 81 | } 82 | 83 | @JsonIgnoreProperties(ignoreUnknown = true) 84 | public static class MinCheckInterval { 85 | private MinCheckIntervalData data; 86 | 87 | public MinCheckIntervalData getData() { 88 | return data; 89 | } 90 | 91 | public void setData(MinCheckIntervalData data) { 92 | this.data = data; 93 | } 94 | } 95 | 96 | public MinCheckIntervalData getCheckInterval() { 97 | return checkInterval; 98 | } 99 | 100 | public void fetch() { 101 | LOG.info("Fetching 'zmon-min-check-interval' entity"); 102 | HttpEntity request = new HttpEntity<>(new HttpHeaders()); 103 | 104 | try { 105 | Timer.Context t = timer.time(); 106 | ResponseEntity> response = restTemplate.exchange(entityServiceUrl, HttpMethod.GET, request, new ParameterizedTypeReference>() { 107 | }); 108 | LOG.info("MinInterval Entity Fetcher used: {}ms", t.stop() / 1_000_000); 109 | 110 | List results = response.getBody(); 111 | if (results.size() != 1) { 112 | throw new IndexOutOfBoundsException("Wrong number of entities returned - " + results.size()); 113 | } 114 | 115 | this.checkInterval = response.getBody().get(0).getData(); 116 | 117 | totalFetches.mark(); 118 | } catch (Throwable e) { 119 | LOG.error("MinInterval Entity Fetcher could not fetch, falling back to default/old", e); 120 | totalErrors.mark(); 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/checks/YamlCheckSource.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 6 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.io.File; 11 | import java.util.ArrayList; 12 | import java.util.Collection; 13 | import java.util.List; 14 | 15 | /** 16 | * Created by jmussler on 4/17/15. 17 | */ 18 | public class YamlCheckSource extends CheckSource { 19 | 20 | private static final Logger LOG = LoggerFactory.getLogger(YamlCheckSource.class); 21 | 22 | private String fileName; 23 | 24 | public YamlCheckSource(String name, String fileName) { 25 | super(name); 26 | this.fileName = fileName; 27 | } 28 | 29 | private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); 30 | 31 | static { 32 | mapper.setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); 33 | } 34 | 35 | @Override 36 | public Collection getCollection() { 37 | try { 38 | List list = mapper.readValue(new File(fileName), new TypeReference>() { 39 | }); 40 | return list; 41 | } catch (Exception e) { 42 | LOG.error("", e); 43 | } 44 | return new ArrayList<>(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/cleanup/AlertChangedCleaner.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import de.zalando.zmon.scheduler.ng.AlertOverlapGenerator; 4 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 5 | import de.zalando.zmon.scheduler.ng.alerts.AlertChangeListener; 6 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 7 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 8 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 9 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 10 | import de.zalando.zmon.scheduler.ng.entities.Entity; 11 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 12 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | import redis.clients.jedis.Jedis; 16 | import redis.clients.jedis.JedisPool; 17 | import redis.clients.jedis.Pipeline; 18 | 19 | import java.util.Set; 20 | import java.util.concurrent.ScheduledThreadPoolExecutor; 21 | import java.util.concurrent.TimeUnit; 22 | import java.util.stream.Collectors; 23 | 24 | /** 25 | * Created by jmussler on 02.06.16. 26 | */ 27 | public class AlertChangedCleaner implements AlertChangeListener { 28 | 29 | private final static Logger LOG = LoggerFactory.getLogger(AlertChangeListener.class); 30 | 31 | private final AlertRepository alertRepository; 32 | private final CheckRepository checkRepository; 33 | private final EntityRepository entityRepository; 34 | 35 | private final JedisPool redisPool; 36 | 37 | private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); 38 | 39 | public AlertChangedCleaner(AlertRepository alertRepo, CheckRepository checkRepository, EntityRepository entityRepo, SchedulerConfig config) { 40 | this.alertRepository = alertRepo; 41 | this.checkRepository = checkRepository; 42 | this.entityRepository = entityRepo; 43 | 44 | GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig(); 45 | poolConfig.setTestOnBorrow(true); 46 | 47 | redisPool = new JedisPool(poolConfig, config.getRedisHost(), config.getRedisPort()); 48 | } 49 | 50 | @Override 51 | public void notifyAlertNew(AlertDefinition alert) { 52 | 53 | } 54 | 55 | @Override 56 | public void notifyAlertChange(AlertDefinition alert) { 57 | final AlertChangedCleaner c = this; 58 | // schedule twice, 1 fast for UI/UX the long run is only for multi scheduler setup where results may come in too late. 59 | executor.schedule(() -> c.doCleanup(alert.getId(), alert.getCheckDefinitionId()), 10, TimeUnit.SECONDS); 60 | executor.schedule(() -> c.doCleanup(alert.getId(), alert.getCheckDefinitionId()), 90, TimeUnit.SECONDS); 61 | } 62 | 63 | @Override 64 | public void notifyAlertDelete(AlertDefinition alert) { 65 | 66 | } 67 | 68 | private Set getMatchingEntities(int alertId, int checkId) { 69 | CheckDefinition cd = checkRepository.get(checkId); 70 | AlertDefinition ad = alertRepository.get(alertId); 71 | 72 | return entityRepository.getUnfiltered().stream() 73 | .filter(e -> AlertOverlapGenerator.matchCheckFilter(cd, e)) 74 | .filter(e -> AlertOverlapGenerator.matchAlertFilter(ad, e)) 75 | .map(Entity::getId) 76 | .collect(Collectors.toSet()); 77 | } 78 | 79 | public void doCleanup(int alertId, int checkId) { 80 | try { 81 | try (Jedis j = redisPool.getResource()) { 82 | 83 | final String alertKey = "zmon:alerts:" + alertId; 84 | final String alertMapKey = "zmon:alerts:" + alertId + ":entities"; 85 | 86 | Set entityIdsInAlert = j.smembers(alertKey); 87 | Set entityIdsTotal = j.hkeys(alertMapKey); 88 | 89 | Set matchedEntityIds = getMatchingEntities(alertId, checkId); 90 | 91 | entityIdsInAlert.removeAll(matchedEntityIds); 92 | entityIdsTotal.removeAll(matchedEntityIds); 93 | 94 | LOG.info("Cleaning alertId={} checkId={} srem={} hdel={}", alertId, checkId, entityIdsInAlert.size(), entityIdsTotal.size()); 95 | 96 | Pipeline p = j.pipelined(); 97 | for (String e : entityIdsInAlert) { 98 | p.srem(alertKey, e); 99 | } 100 | 101 | for (String e : entityIdsTotal) { 102 | p.hdel(alertMapKey, e); 103 | } 104 | p.sync(); 105 | } 106 | } catch (Throwable t) { 107 | LOG.error("Uncaught/Unexpected exception: msg={} check_id={} alert_id={}", t, checkId, alertId); 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/cleanup/AllTrialRunCleanupTask.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import redis.clients.jedis.Jedis; 7 | 8 | import java.util.Set; 9 | 10 | /** 11 | * Created by jmussler on 05.06.16. 12 | */ 13 | public class AllTrialRunCleanupTask implements Runnable { 14 | 15 | public static final Logger LOG = LoggerFactory.getLogger(AllTrialRunCleanupTask.class); 16 | 17 | private final SchedulerConfig config; 18 | 19 | public AllTrialRunCleanupTask(SchedulerConfig config) { 20 | this.config = config; 21 | } 22 | 23 | @Override 24 | public void run() { 25 | try { 26 | LOG.info("Starting cleanup of old trial run results"); 27 | try (Jedis jedis = new Jedis(config.getRedisHost(), config.getRedisPort())) { 28 | Set keys = jedis.keys("zmon:trial_run:*"); 29 | // No pipeline here, considering this startup task to be fast enough 30 | // Dont want this to fail on not cleaned appliance deployments 31 | for (String k : keys) { 32 | jedis.del(k); 33 | } 34 | 35 | LOG.info("Finished trial run cleanup: count={}", keys.size()); 36 | } 37 | } catch (Throwable t) { 38 | LOG.error("Failed to cleanup old trial runs: msg={}", t.getMessage()); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/cleanup/CheckChangedCleaner.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 4 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 5 | import de.zalando.zmon.scheduler.ng.checks.CheckChangeListener; 6 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 7 | 8 | import java.util.Collection; 9 | 10 | /** 11 | * Created by jmussler on 02.06.16. 12 | */ 13 | public class CheckChangedCleaner implements CheckChangeListener { 14 | 15 | private final AlertChangedCleaner alertCleaner; 16 | 17 | private final AlertRepository alertRepository; 18 | 19 | public CheckChangedCleaner(AlertRepository alertRepo, AlertChangedCleaner alertCleaner) { 20 | this.alertRepository = alertRepo; 21 | this.alertCleaner = alertCleaner; 22 | } 23 | 24 | @Override 25 | public void notifyNewCheck(CheckRepository repo, int checkId) { 26 | 27 | } 28 | 29 | @Override 30 | public void notifyCheckIntervalChange(CheckRepository repo, int checkId) { 31 | 32 | } 33 | 34 | @Override 35 | public void notifyDeleteCheck(CheckRepository repo, int checkId) { 36 | 37 | } 38 | 39 | @Override 40 | public void notifyFilterChange(int checkId) { 41 | /* Just redirect to clean up all child alerts */ 42 | Collection alertDefs = alertRepository.getByCheckId(checkId); 43 | for (AlertDefinition ad : alertDefs) { 44 | alertCleaner.notifyAlertChange(ad); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/cleanup/DowntimeCleanup.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 6 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import redis.clients.jedis.Jedis; 10 | 11 | import java.io.IOException; 12 | import java.util.Map; 13 | import java.util.Set; 14 | import java.util.concurrent.ScheduledThreadPoolExecutor; 15 | import java.util.concurrent.TimeUnit; 16 | 17 | /** 18 | * Created by jmussler on 13.12.16. 19 | */ 20 | public class DowntimeCleanup implements Runnable { 21 | 22 | private final Logger log = LoggerFactory.getLogger(DowntimeCleanup.class); 23 | private final SchedulerConfig config; 24 | private final ObjectMapper mapper = (new ObjectMapper()).setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); 25 | private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); 26 | 27 | public DowntimeCleanup(SchedulerConfig config) { 28 | this.config = config; 29 | executor.scheduleAtFixedRate(this, 5, 15, TimeUnit.MINUTES); 30 | } 31 | 32 | // not pipelining on purpose, this is just background task, dont want to block 33 | // just deleting entries not valid any more. Cleaning up upper level entries is done on next iteration 34 | private void doCleanup() { 35 | long now = System.currentTimeMillis() / 1000L; 36 | 37 | try(Jedis jedis = new Jedis(config.getRedisHost(), config.getRedisPort())) { 38 | Set alertIdsInDowntime = jedis.smembers("zmon:downtimes"); 39 | log.info("Alerts in downtime: count={}", alertIdsInDowntime.size()); 40 | int cleanupCounter = 0; 41 | for(String alertId : alertIdsInDowntime) { 42 | Set entityIds = jedis.smembers("zmon:downtimes:" + alertId); 43 | if (null == entityIds || entityIds.size() == 0) { 44 | cleanupCounter++; 45 | jedis.srem("zmon:downtimes", alertId); 46 | } 47 | 48 | for(String entityId : entityIds) { 49 | Map downtimes = jedis.hgetAll("zmon:downtimes:" + alertId + ":" + entityId); 50 | if(null == downtimes || downtimes.size() == 0) { 51 | cleanupCounter++; 52 | jedis.srem("zmon:downtimes:" + alertId, entityId); 53 | } 54 | 55 | for(Map.Entry dt : downtimes.entrySet()) { 56 | try { 57 | JsonNode d = mapper.readTree(dt.getValue()); 58 | if(d.has("end_time") && Long.compare(now, d.get("end_time").asLong()) >= 0) { 59 | jedis.hdel("zmon:downtimes:" + alertId + ":" + entityId, dt.getKey()); 60 | cleanupCounter++; 61 | } 62 | } 63 | catch(IOException ex) { 64 | // malformed json redis, delete downtime entry 65 | log.info("Failed to read downtime: entity={} id={}", entityId, dt.getKey()); 66 | jedis.hdel("zmon:downtimes:" + alertId + ":" + entityId, dt.getKey()); 67 | } 68 | } 69 | } 70 | } 71 | log.info("Cleanup downtimes ended: count={}", cleanupCounter); 72 | } 73 | } 74 | 75 | @Override 76 | public void run() { 77 | try { 78 | doCleanup(); 79 | } 80 | catch(Throwable t) { 81 | log.error("Cleanup downtimes failed", t); 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/cleanup/EntityChangedCleaner.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import de.zalando.zmon.scheduler.ng.AlertOverlapGenerator; 4 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 5 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 6 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 7 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 8 | import de.zalando.zmon.scheduler.ng.entities.Entity; 9 | import de.zalando.zmon.scheduler.ng.entities.EntityChangeListener; 10 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.concurrent.ScheduledThreadPoolExecutor; 15 | import java.util.concurrent.TimeUnit; 16 | 17 | /** 18 | * Created by jmussler on 06.06.16. 19 | */ 20 | public class EntityChangedCleaner implements EntityChangeListener { 21 | 22 | private final static Logger LOG = LoggerFactory.getLogger(EntityChangedCleaner.class); 23 | 24 | private final AlertRepository alertRepo; 25 | private final CheckRepository checkRepo; 26 | private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); 27 | private final AlertChangedCleaner alertCleaner; 28 | 29 | public EntityChangedCleaner(AlertRepository alertRepo, CheckRepository checkRepo, AlertChangedCleaner alertCleaner) { 30 | this.alertRepo = alertRepo; 31 | this.checkRepo = checkRepo; 32 | this.alertCleaner = alertCleaner; 33 | } 34 | 35 | private class EntityChangeCleanupTask implements Runnable { 36 | private final Entity oldEntity; 37 | private final Entity newEntity; 38 | 39 | public EntityChangeCleanupTask(Entity oldEntity, Entity newEntity) { 40 | this.oldEntity = oldEntity; 41 | this.newEntity = newEntity; 42 | } 43 | 44 | @Override 45 | public void run() { 46 | try { 47 | for (CheckDefinition cd : checkRepo.get()) { 48 | if (AlertOverlapGenerator.matchCheckFilter(cd, oldEntity)) { 49 | for (AlertDefinition ad : alertRepo.getByCheckId(cd.getId())) { 50 | if (AlertOverlapGenerator.matchAlertFilter(ad, oldEntity)) { 51 | if (!AlertOverlapGenerator.matchCheckFilter(cd, newEntity) 52 | || !AlertOverlapGenerator.matchAlertFilter(ad, newEntity)) { 53 | // reuse code, this cleans up all non matching entities 54 | alertCleaner.notifyAlertChange(ad); 55 | } 56 | } 57 | } 58 | } 59 | } 60 | } catch (Throwable t) { 61 | LOG.error("Failed to cleanup changed entity: id={}", newEntity.getId()); 62 | } 63 | } 64 | } 65 | 66 | @Override 67 | public void notifyEntityAdd(EntityRepository repo, Entity e) { 68 | 69 | } 70 | 71 | @Override 72 | public void notifyEntityRemove(EntityRepository repo, Entity e) { 73 | 74 | } 75 | 76 | protected void notifyEntityChangeNoWait(EntityRepository repo, Entity entityOld, Entity entityNew) { 77 | executor.schedule(new EntityChangeCleanupTask(entityOld, entityNew), 0, TimeUnit.SECONDS); 78 | } 79 | 80 | @Override 81 | public void notifyEntityChange(EntityRepository repo, Entity entityOld, Entity entityNew) { 82 | executor.schedule(new EntityChangeCleanupTask(entityOld, entityNew), 5, TimeUnit.SECONDS); 83 | executor.schedule(new EntityChangeCleanupTask(entityOld, entityNew), 60, TimeUnit.SECONDS); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/cleanup/MetricsCleanup.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import redis.clients.jedis.Jedis; 7 | 8 | import java.util.Set; 9 | import java.util.concurrent.ScheduledThreadPoolExecutor; 10 | import java.util.concurrent.TimeUnit; 11 | 12 | /** 13 | * Created by jmussler on 11.07.16. 14 | */ 15 | public class MetricsCleanup { 16 | private final SchedulerConfig config; 17 | private final Logger log = LoggerFactory.getLogger(MetricsCleanup.class); 18 | private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); 19 | 20 | public MetricsCleanup(SchedulerConfig config) { 21 | this.config = config; 22 | executor.scheduleAtFixedRate(new CleanupTask(), 0, 5, TimeUnit.MINUTES); 23 | } 24 | 25 | private class CleanupTask implements Runnable { 26 | 27 | @Override 28 | public void run() { 29 | try (Jedis jedis = new Jedis(config.getRedisHost(), config.getRedisPort())) { 30 | float currentTimeSeconds = System.currentTimeMillis() / 1000; 31 | int count = 0; 32 | Set members = jedis.smembers("zmon:metrics"); 33 | for (String m : members) { 34 | // NOTE: the timestamp is in seconds since epoch (Python worker writes a float value) 35 | float lastUpdateSeconds = Float.parseFloat(jedis.get("zmon:metrics:" + m + ":ts")); 36 | if (lastUpdateSeconds < (currentTimeSeconds - 60 * 60)) { 37 | // cleanup entries that have no activity for 60 min for now 38 | jedis.del("zmon:metrics:" + m + ":ts"); 39 | jedis.del("zmon:metrics:" + m + ":check.count"); 40 | jedis.srem("zmon:metrics", m); 41 | count++; 42 | } 43 | } 44 | log.info("removed {} entities", count); 45 | } catch (Throwable t) { 46 | log.error("Metrics cleanup failed: {}", t.getMessage()); 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/cleanup/SingleEntityCleanup.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import java.util.Collection; 4 | import java.util.List; 5 | import java.util.concurrent.ScheduledThreadPoolExecutor; 6 | import java.util.concurrent.TimeUnit; 7 | import java.util.stream.Collectors; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import de.zalando.zmon.scheduler.ng.AlertOverlapGenerator; 13 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 14 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 15 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 16 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 17 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 18 | import de.zalando.zmon.scheduler.ng.entities.Entity; 19 | import de.zalando.zmon.scheduler.ng.entities.EntityChangeListener; 20 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 21 | import redis.clients.jedis.Jedis; 22 | import redis.clients.jedis.JedisPool; 23 | import redis.clients.jedis.JedisPoolConfig; 24 | 25 | /** 26 | * Created by jmussler on 30.06.16. 27 | */ 28 | public class SingleEntityCleanup implements EntityChangeListener{ 29 | 30 | private final static Logger LOG = LoggerFactory.getLogger(SingleEntityCleanup.class); 31 | 32 | private final AlertRepository alertRepo; 33 | private final CheckRepository checkRepo; 34 | 35 | private final JedisPool jedisPool; 36 | private final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); 37 | 38 | public SingleEntityCleanup(SchedulerConfig config, AlertRepository alertRepo, CheckRepository checkRepo, EntityRepository entityRepo) { 39 | this.alertRepo = alertRepo; 40 | this.checkRepo = checkRepo; 41 | 42 | JedisPoolConfig poolConfig = new JedisPoolConfig(); 43 | poolConfig.setTestOnBorrow(true); 44 | 45 | jedisPool = new JedisPool(poolConfig, config.getRedisHost(), config.getRedisPort()); 46 | } 47 | 48 | @Override 49 | public void notifyEntityChange(EntityRepository repo, Entity entityOld, Entity entityNew) { 50 | 51 | } 52 | 53 | @Override 54 | public void notifyEntityRemove(EntityRepository repo, Entity e) { 55 | executor.schedule(new EntityCleanupTask(e), 90, TimeUnit.SECONDS); 56 | executor.schedule(new EntityCleanupTask(e), 300, TimeUnit.SECONDS); 57 | } 58 | 59 | @Override 60 | public void notifyEntityAdd(EntityRepository repo, Entity e) { 61 | 62 | } 63 | 64 | private class EntityCleanupTask implements Runnable { 65 | private final Entity entity; 66 | 67 | public EntityCleanupTask(Entity entity) { 68 | this.entity = entity; 69 | } 70 | 71 | private void doCleanup(int checkId, Collection alertIds, String entityId) { 72 | try(Jedis jedis = jedisPool.getResource()) { 73 | jedis.del("zmon:checks:" + checkId + ":" + entityId); 74 | jedis.srem("zmon:checks:" + checkId, entityId); 75 | 76 | for(Integer alertId : alertIds) { 77 | jedis.srem("zmon:alerts:" + alertId, entityId); 78 | jedis.del("zmon:alerts:" + alertId + ":" + entityId); 79 | jedis.hdel("zmon:alerts:" + alertId + ":entities", entityId); 80 | } 81 | } 82 | catch(Throwable t) { 83 | LOG.error("Error during cleanup: entity={} checkId={}", entityId, checkId); 84 | } 85 | } 86 | 87 | @Override 88 | public void run() { 89 | int checksCleaned = 0; 90 | int alertsCleaned = 0; 91 | 92 | for (CheckDefinition checkDef : checkRepo.get()) { 93 | if (AlertOverlapGenerator.matchCheckFilter(checkDef, entity)) { 94 | List alerts = alertRepo.getByCheckId(checkDef.getId()).stream() 95 | .filter(x->AlertOverlapGenerator.matchAlertFilter(x, entity)) 96 | .map(AlertDefinition::getId) 97 | .collect(Collectors.toList()); 98 | 99 | if (!alerts.isEmpty()) { 100 | doCleanup(checkDef.getId(), alerts, entity.getId()); 101 | checksCleaned++; 102 | alertsCleaned += alerts.size(); 103 | } 104 | } 105 | } 106 | 107 | LOG.info("Cleanup entity={} checks={} alerts={}", entity.getId(), checksCleaned, alertsCleaned); 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/cleanup/TrialRunCleanupTask.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import redis.clients.jedis.Jedis; 7 | 8 | /** 9 | * Created by jmussler on 05.06.16. 10 | */ 11 | public class TrialRunCleanupTask implements Runnable { 12 | 13 | public static final Logger LOG = LoggerFactory.getLogger(TrialRunCleanupTask.class); 14 | 15 | private final SchedulerConfig config; 16 | private String trialRunId; 17 | 18 | public TrialRunCleanupTask(String trialRunId, SchedulerConfig config) { 19 | this.config = config; 20 | this.trialRunId = trialRunId; 21 | } 22 | 23 | @Override 24 | public void run() { 25 | try { 26 | LOG.info("Trial run cleanup: id={}", this.trialRunId); 27 | try (Jedis jedis = new Jedis(config.getRedisHost(), config.getRedisPort())) { 28 | jedis.del("zmon:trial_run:" + trialRunId); 29 | jedis.del("zmon:trial_run:" + trialRunId + ":results"); 30 | } 31 | } catch (Throwable t) { 32 | LOG.error("Failed to cleanup trial run data: id={} msg={}", this.trialRunId, t.getMessage()); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/config/CleanupConfiguration.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.config; 2 | 3 | import de.zalando.zmon.scheduler.ng.cleanup.*; 4 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 5 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 6 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | 12 | /** 13 | * Created by jmussler on 03.06.16. 14 | */ 15 | @Configuration 16 | public class CleanupConfiguration { 17 | 18 | private final static Logger LOG = LoggerFactory.getLogger(CleanupConfiguration.class); 19 | 20 | @Bean 21 | public CheckChangedCleaner checkChangedCleaner(AlertRepository alertRepo, CheckRepository checkRepo, AlertChangedCleaner alertCleaner) { 22 | LOG.info("Registering checkChangedCleaner..."); 23 | CheckChangedCleaner l = new CheckChangedCleaner(alertRepo, alertCleaner); 24 | checkRepo.registerListener(l); 25 | return l; 26 | } 27 | 28 | @Bean 29 | public AlertChangedCleaner alertChangedCleaner(AlertRepository alertRepo, CheckRepository checkRepo, EntityRepository entityRepo, SchedulerConfig config) { 30 | LOG.info("Registering alertChangedCleaner..."); 31 | AlertChangedCleaner l = new AlertChangedCleaner(alertRepo, checkRepo, entityRepo, config); 32 | alertRepo.registerChangeListener(l); 33 | return l; 34 | } 35 | 36 | @Bean 37 | public EntityChangedCleaner entityChangedCleaner(EntityRepository entityRepo, AlertRepository alertRepo, CheckRepository checkRepo, AlertChangedCleaner alertCleaner) { 38 | LOG.info("Registering entityChangedCleaner..."); 39 | EntityChangedCleaner l = new EntityChangedCleaner(alertRepo, checkRepo, alertCleaner); 40 | entityRepo.registerListener(l); 41 | return l; 42 | } 43 | 44 | @Bean 45 | public MetricsCleanup metricsCleanup(SchedulerConfig config) { 46 | LOG.info("Registering metricsCleaner..."); 47 | MetricsCleanup cleaner = new MetricsCleanup(config); 48 | return cleaner; 49 | } 50 | 51 | @Bean 52 | public DowntimeCleanup downtimeCleanup(SchedulerConfig config){ 53 | LOG.info("Registering downtimeCleanup..."); 54 | DowntimeCleanup cleaner = new DowntimeCleanup(config); 55 | return cleaner; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/config/Oauth2Config.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.config; 2 | 3 | import de.zalando.zmon.scheduler.ng.TokenWrapper; 4 | import org.springframework.context.annotation.Bean; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.zalando.stups.tokens.AccessTokenConfiguration; 7 | import org.zalando.stups.tokens.AccessTokens; 8 | import org.zalando.stups.tokens.Tokens; 9 | 10 | import java.net.URI; 11 | 12 | /** 13 | * Created by jmussler on 26.01.16. 14 | */ 15 | @Configuration 16 | public class Oauth2Config { 17 | 18 | /* 19 | * In case where no oauth2 infrastructure is used, we always use/send the provided static token 20 | * */ 21 | @Bean 22 | public TokenWrapper accessTokens(SchedulerConfig config) { 23 | if (config.getOauth2AccessTokenUrl() == null) { 24 | return new TokenWrapper(config.getOauth2StaticToken()); 25 | } else { 26 | AccessTokenConfiguration tokenConfig = Tokens.createAccessTokensWithUri(URI.create(config.getOauth2AccessTokenUrl())) 27 | .manageToken("zmon-read"); 28 | 29 | for (String scope : config.getOauth2Scopes()) { 30 | tokenConfig.addScope(scope); 31 | } 32 | 33 | AccessTokens tokens = tokenConfig.done().start(); 34 | 35 | return new TokenWrapper(tokens, "zmon-read"); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/config/QueueSelectorConfiguration.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.config; 2 | 3 | import de.zalando.zmon.scheduler.ng.TaskWriterType; 4 | import de.zalando.zmon.scheduler.ng.queue.ArrayQueueWriter; 5 | import de.zalando.zmon.scheduler.ng.queue.JedisQueueWriter; 6 | import de.zalando.zmon.scheduler.ng.queue.LogQueueWriter; 7 | import de.zalando.zmon.scheduler.ng.queue.QueueSelector; 8 | import de.zalando.zmon.scheduler.ng.queue.QueueWriter; 9 | 10 | import com.codahale.metrics.MetricRegistry; 11 | import io.opentracing.Tracer; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | import org.springframework.beans.factory.annotation.Autowired; 15 | import org.springframework.context.annotation.Bean; 16 | import org.springframework.context.annotation.Configuration; 17 | 18 | /** 19 | * Created by jmussler on 30.06.16. 20 | */ 21 | @Configuration 22 | public class QueueSelectorConfiguration { 23 | 24 | @Autowired 25 | private Tracer tracer; 26 | 27 | private static final Logger LOG = LoggerFactory.getLogger(QueueSelectorConfiguration.class); 28 | 29 | public static QueueWriter createWriter(SchedulerConfig schedulerConfig, MetricRegistry metrics) { 30 | if(schedulerConfig.getTaskWriterType() == TaskWriterType.REDIS) { 31 | LOG.info("Creating queue writer: Redis host={} port={}", schedulerConfig.getRedisHost(), schedulerConfig.getRedisPort()); 32 | return new JedisQueueWriter(schedulerConfig.getRedisHost(), schedulerConfig.getRedisPort(), metrics); 33 | } 34 | else if (schedulerConfig.getTaskWriterType() == TaskWriterType.ARRAY_LIST) { 35 | LOG.info("creating queue writer: ArrayQueueWriter"); 36 | return new ArrayQueueWriter(metrics); 37 | } 38 | else { 39 | LOG.info("Creating queue writer: LOG writer"); 40 | return new LogQueueWriter(metrics); 41 | } 42 | } 43 | 44 | @Bean 45 | public QueueWriter getWriter(SchedulerConfig config, MetricRegistry metrics) { 46 | return createWriter(config, metrics); 47 | } 48 | 49 | @Bean 50 | public QueueSelector getSelector(QueueWriter writer, SchedulerConfig config) { 51 | return new QueueSelector(writer, config, tracer); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/config/RestTemplateConfiguration.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.config; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 5 | import de.zalando.zmon.scheduler.ng.TokenWrapper; 6 | import org.apache.http.impl.client.HttpClientBuilder; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.http.HttpHeaders; 10 | import org.springframework.http.HttpRequest; 11 | import org.springframework.http.MediaType; 12 | import org.springframework.http.client.*; 13 | import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; 14 | import org.springframework.web.client.RestTemplate; 15 | 16 | import java.io.IOException; 17 | import java.util.Collections; 18 | 19 | /** 20 | * Created by jmussler on 11.02.16. 21 | */ 22 | @Configuration 23 | public class RestTemplateConfiguration { 24 | 25 | private final static ObjectMapper mapper = (new ObjectMapper()).setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); 26 | 27 | public static class TokenInterceptor implements ClientHttpRequestInterceptor { 28 | 29 | private TokenWrapper tokens; 30 | 31 | public TokenInterceptor(TokenWrapper tokens) { 32 | this.tokens = tokens; 33 | } 34 | 35 | @Override 36 | public ClientHttpResponse intercept( 37 | HttpRequest request, byte[] body, ClientHttpRequestExecution execution) 38 | throws IOException { 39 | 40 | HttpHeaders headers = request.getHeaders(); 41 | headers.add("Authorization", "Bearer " + tokens.get()); 42 | return execution.execute(request, body); 43 | } 44 | } 45 | 46 | @Bean 47 | public ClientHttpRequestFactory getFactory() { 48 | HttpComponentsClientHttpRequestFactory factory = new HttpComponentsClientHttpRequestFactory(HttpClientBuilder.create().build()); 49 | factory.setReadTimeout(60000); 50 | factory.setConnectTimeout(3000); 51 | return factory; 52 | } 53 | 54 | @Bean 55 | public RestTemplate restTemplate(ClientHttpRequestFactory clientFactory, TokenWrapper tokens) { 56 | RestTemplate rt = new RestTemplate(clientFactory); 57 | 58 | TokenInterceptor ti = new TokenInterceptor(tokens); 59 | 60 | MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); 61 | converter.setObjectMapper(mapper); 62 | converter.setSupportedMediaTypes(Collections.singletonList(MediaType.ALL)); 63 | 64 | rt.getMessageConverters().clear(); 65 | rt.getMessageConverters().add(converter); 66 | 67 | rt.getInterceptors().add(ti); 68 | 69 | return rt; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/config/SchedulerFactory.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.config; 2 | 3 | import de.zalando.zmon.scheduler.ng.TokenWrapper; 4 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 5 | import de.zalando.zmon.scheduler.ng.checks.CheckChangedListener; 6 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 7 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 8 | import de.zalando.zmon.scheduler.ng.downtimes.DowntimeForwarder; 9 | import de.zalando.zmon.scheduler.ng.downtimes.DowntimeService; 10 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 11 | import de.zalando.zmon.scheduler.ng.instantevaluations.InstantEvalForwarder; 12 | import de.zalando.zmon.scheduler.ng.instantevaluations.InstantEvalHttpSubscriber; 13 | import de.zalando.zmon.scheduler.ng.queue.QueueSelector; 14 | import de.zalando.zmon.scheduler.ng.scheduler.Scheduler; 15 | import de.zalando.zmon.scheduler.ng.trailruns.TrialRunForwarder; 16 | import de.zalando.zmon.scheduler.ng.trailruns.TrialRunHttpSubscriber; 17 | 18 | import com.codahale.metrics.MetricRegistry; 19 | import io.opentracing.Tracer; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | import org.springframework.beans.factory.annotation.Autowired; 23 | import org.springframework.context.annotation.Bean; 24 | import org.springframework.context.annotation.Configuration; 25 | import org.springframework.web.client.RestTemplate; 26 | 27 | /** 28 | * Created by jmussler on 30.06.16. 29 | */ 30 | 31 | @Configuration 32 | public class SchedulerFactory { 33 | 34 | @Autowired 35 | private Tracer tracer; 36 | 37 | private final static Logger LOG = LoggerFactory.getLogger(SchedulerFactory.class); 38 | 39 | @Bean 40 | public CheckChangedListener getCheckChangeListener(CheckRepository checkRepo, Scheduler scheduler){ 41 | CheckChangedListener listener = new CheckChangedListener(scheduler); 42 | checkRepo.registerListener(listener); 43 | return listener; 44 | } 45 | 46 | @Bean 47 | public Scheduler getScheduler(AlertRepository alertRepo, 48 | CheckRepository checkRepo, 49 | EntityRepository entityRepo, 50 | QueueSelector queueSelector, 51 | InstantEvalForwarder instantForwarder, 52 | TrialRunForwarder trialRunForwarder, 53 | DowntimeForwarder downtimeForwarder, 54 | DowntimeService downtimeService, 55 | TokenWrapper tokenWrapper, 56 | RestTemplate restTemplate, 57 | SchedulerConfig config, 58 | MetricRegistry metrics) { 59 | 60 | LOG.info("Creating scheduler instance"); 61 | Scheduler newScheduler = new Scheduler(alertRepo, 62 | checkRepo, 63 | entityRepo, 64 | queueSelector, 65 | config, 66 | metrics, 67 | tracer); 68 | 69 | LOG.info("Check ID filter: {}", config.getCheckFilter()); 70 | 71 | LOG.info("Initial scheduling of all checks"); 72 | for (CheckDefinition cd : checkRepo.get()) { 73 | newScheduler.scheduleCheck(cd.getId()); 74 | } 75 | SchedulerFactory.LOG.info("Initial scheduling of all checks done"); 76 | 77 | if (config.isInstantEvalForward()) { 78 | entityRepo.registerListener(instantForwarder); 79 | } 80 | 81 | if (config.isTrialRunForward()) { 82 | entityRepo.registerListener(trialRunForwarder); 83 | } 84 | 85 | if (config.isDowntimeForward()) { 86 | entityRepo.registerListener(downtimeForwarder); 87 | } 88 | 89 | if (config.getTrialRunHttpUrl() != null) { 90 | new TrialRunHttpSubscriber(newScheduler, config, restTemplate); 91 | } 92 | 93 | if (config.getInstantEvalHttpUrl() != null) { 94 | new InstantEvalHttpSubscriber(newScheduler, config, restTemplate); 95 | } 96 | 97 | return newScheduler; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/config/SingleEntityCleanupConfiguration.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.config; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 4 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 5 | import de.zalando.zmon.scheduler.ng.cleanup.SingleEntityCleanup; 6 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.context.annotation.Bean; 10 | import org.springframework.context.annotation.Configuration; 11 | 12 | /** 13 | * Created by jmussler on 30.06.16. 14 | */ 15 | @Configuration 16 | public class SingleEntityCleanupConfiguration { 17 | private final static Logger LOG = LoggerFactory.getLogger(SingleEntityCleanupConfiguration.class); 18 | 19 | @Bean 20 | SingleEntityCleanup getSingleEntityCleanup(SchedulerConfig config, AlertRepository alertRepo, CheckRepository checkRepo, EntityRepository entityRepository) { 21 | SingleEntityCleanup cleanup = new SingleEntityCleanup(config, alertRepo, checkRepo, entityRepository); 22 | LOG.info("Registering SingleEntityCleanUp job"); 23 | entityRepository.registerListener(cleanup); 24 | return cleanup; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/config/StupsOAuthConfig.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.config; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | import org.springframework.stereotype.Component; 5 | 6 | import java.util.Map; 7 | import java.util.Set; 8 | 9 | /** 10 | * Created by jmussler on 5/11/15. 11 | */ 12 | @Component 13 | @ConfigurationProperties(prefix = "oauth2") 14 | public class StupsOAuthConfig { 15 | private String authURI; 16 | private String tokenURI; 17 | private String metaFolder; 18 | private Long lifeTime = 60 * 60 * 1000L; 19 | 20 | public Long getLifeTime() { 21 | return lifeTime; 22 | } 23 | 24 | public String getMetaFolder() { 25 | return metaFolder; 26 | } 27 | 28 | public void setMetaFolder(String metaFolder) { 29 | this.metaFolder = metaFolder; 30 | } 31 | 32 | public String getAuthURI() { 33 | return authURI; 34 | } 35 | 36 | public void setAuthURI(String authURI) { 37 | this.authURI = authURI; 38 | } 39 | 40 | public String getTokenURI() { 41 | return tokenURI; 42 | } 43 | 44 | public void setTokenURI(String tokenURI) { 45 | this.tokenURI = tokenURI; 46 | } 47 | 48 | public Map> getScopes() { 49 | return scopes; 50 | } 51 | 52 | public void setScopes(Map> scopes) { 53 | this.scopes = scopes; 54 | } 55 | 56 | public Map> scopes; 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeAlertRequest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * Created by jmussler on 18.06.16. 7 | */ 8 | public class DowntimeAlertRequest { 9 | private int alertId; 10 | 11 | // Stores entityId -> UUID map, UUID from controller to delete individual downtimes 12 | private Map entityIds; 13 | 14 | public DowntimeAlertRequest() { 15 | } 16 | 17 | public int getAlertId() { 18 | return alertId; 19 | } 20 | 21 | public void setAlertId(int alertId) { 22 | this.alertId = alertId; 23 | } 24 | 25 | public Map getEntityIds() { 26 | return entityIds; 27 | } 28 | 29 | public void setEntityIds(Map entityIds) { 30 | this.entityIds = entityIds; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeConfiguration.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.web.client.RestTemplate; 9 | 10 | /** 11 | * Created by jmussler on 18.06.16. 12 | */ 13 | @Configuration 14 | public class DowntimeConfiguration { 15 | 16 | public static Logger LOG = LoggerFactory.getLogger(DowntimeConfiguration.class); 17 | 18 | @Bean 19 | DowntimeHttpSubscriber downtimeHttpSubscriber(DowntimeService downtimeService, SchedulerConfig config, RestTemplate restTemplate) { 20 | if (config.getDowntimeHttpUrl() != null) { 21 | LOG.info("Registering DowntimeHttpSubscriber"); 22 | return new DowntimeHttpSubscriber(downtimeService, config, restTemplate); 23 | } 24 | return null; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeData.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | 4 | /** 5 | * Created by jmussler on 18.06.16. 6 | */ 7 | public class DowntimeData { 8 | 9 | private String comment; 10 | private Long startTime; 11 | private Long endTime; 12 | private String id; 13 | private Integer alertId; 14 | private String entity; 15 | // uuid identifying groups of downtimes 16 | private String groupId; 17 | private String createdBy; 18 | 19 | public DowntimeData() { 20 | } 21 | 22 | public String getComment() { 23 | return comment; 24 | } 25 | 26 | public void setComment(String comment) { 27 | this.comment = comment; 28 | } 29 | 30 | public Long getStartTime() { 31 | return startTime; 32 | } 33 | 34 | public void setStartTime(Long startTime) { 35 | this.startTime = startTime; 36 | } 37 | 38 | public Long getEndTime() { 39 | return endTime; 40 | } 41 | 42 | public void setEndTime(Long endTime) { 43 | this.endTime = endTime; 44 | } 45 | 46 | public String getId() { 47 | return id; 48 | } 49 | 50 | public void setId(String id) { 51 | this.id = id; 52 | } 53 | 54 | public Integer getAlertId() { 55 | return alertId; 56 | } 57 | 58 | public void setAlertId(Integer alertId) { 59 | this.alertId = alertId; 60 | } 61 | 62 | public String getEntity() { 63 | return entity; 64 | } 65 | 66 | public void setEntity(String entity) { 67 | this.entity = entity; 68 | } 69 | 70 | public String getGroupId() { 71 | return groupId; 72 | } 73 | 74 | public void setGroupId(String groupId) { 75 | this.groupId = groupId; 76 | } 77 | 78 | public String getCreatedBy() { 79 | return createdBy; 80 | } 81 | 82 | public void setCreatedBy(String createdBy) { 83 | this.createdBy = createdBy; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeForwardTask.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | import java.util.Collection; 4 | 5 | /** 6 | * Created by jmussler on 18.06.16. 7 | */ 8 | public class DowntimeForwardTask { 9 | 10 | private DowntimeRequest request; 11 | private DowntimeTaskType type; 12 | private String groupId; 13 | private Collection ids; 14 | 15 | public DowntimeForwardTask() { 16 | } 17 | 18 | public static DowntimeForwardTask DeleteGroupTask(String groupId) { 19 | DowntimeForwardTask t = new DowntimeForwardTask(); 20 | t.setType(DowntimeTaskType.DELETE_GROUP); 21 | t.setGroupId(groupId); 22 | return t; 23 | } 24 | 25 | public static DowntimeForwardTask NewDowntimeTask(DowntimeRequest request) { 26 | DowntimeForwardTask t = new DowntimeForwardTask(); 27 | t.setType(DowntimeTaskType.NEW); 28 | t.setRequest(request); 29 | return t; 30 | } 31 | 32 | public static DowntimeForwardTask DeleteDowntimeTask(Collection ids) { 33 | DowntimeForwardTask t = new DowntimeForwardTask(); 34 | t.setType(DowntimeTaskType.DELETE); 35 | t.setIds(ids); 36 | return t; 37 | } 38 | 39 | public Collection getIds() { 40 | return ids; 41 | } 42 | 43 | public void setIds(Collection ids) { 44 | this.ids = ids; 45 | } 46 | 47 | public DowntimeRequest getRequest() { 48 | return request; 49 | } 50 | 51 | public void setRequest(DowntimeRequest request) { 52 | this.request = request; 53 | } 54 | 55 | public DowntimeTaskType getType() { 56 | return type; 57 | } 58 | 59 | public void setType(DowntimeTaskType type) { 60 | this.type = type; 61 | } 62 | 63 | public String getGroupId() { 64 | return groupId; 65 | } 66 | 67 | public void setGroupId(String groupId) { 68 | this.groupId = groupId; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeForwarder.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | import de.zalando.zmon.scheduler.ng.DataCenterSubscriber; 4 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * Created by jmussler on 18.06.16. 10 | */ 11 | @Component 12 | public class DowntimeForwarder extends DataCenterSubscriber { 13 | 14 | @Autowired 15 | public DowntimeForwarder(SchedulerConfig config) { 16 | super(config.isDowntimeForward()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeHttpSubscriber.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.core.ParameterizedTypeReference; 7 | import org.springframework.http.HttpEntity; 8 | import org.springframework.http.HttpHeaders; 9 | import org.springframework.http.HttpMethod; 10 | import org.springframework.http.ResponseEntity; 11 | import org.springframework.web.client.RestTemplate; 12 | 13 | import java.util.List; 14 | import java.util.concurrent.ScheduledExecutorService; 15 | import java.util.concurrent.ScheduledThreadPoolExecutor; 16 | import java.util.concurrent.TimeUnit; 17 | 18 | /** 19 | * Created by jmussler on 5/22/15. 20 | */ 21 | public class DowntimeHttpSubscriber implements Runnable { 22 | 23 | private final static Logger LOG = LoggerFactory.getLogger(DowntimeHttpSubscriber.class); 24 | 25 | private final DowntimeService service; 26 | private final String url; 27 | private final ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1); 28 | private final RestTemplate restTemplate; 29 | 30 | public DowntimeHttpSubscriber(DowntimeService service, SchedulerConfig config, RestTemplate restTemplate) { 31 | url = config.getDowntimeHttpUrl(); 32 | this.service = service; 33 | this.restTemplate = restTemplate; 34 | 35 | 36 | LOG.info("Subscribing for downtimes: {}", url); 37 | if (url != null && !url.equals("")) { 38 | executor.scheduleAtFixedRate(this, 60, 5, TimeUnit.SECONDS); 39 | } 40 | } 41 | 42 | @Override 43 | public void run() { 44 | try { 45 | HttpHeaders headers = new HttpHeaders(); 46 | HttpEntity request = new HttpEntity<>(headers); 47 | 48 | ResponseEntity> response = restTemplate.exchange(url, HttpMethod.GET, request, new ParameterizedTypeReference>() { 49 | }); 50 | 51 | for (DowntimeForwardTask task : response.getBody()) { 52 | 53 | switch (task.getType()) { 54 | case NEW: 55 | LOG.info("Received downtime request: type={} groupdId={}", task.getType(), task.getRequest().getGroupId()); 56 | service.storeDowntime(task.getRequest()); 57 | break; 58 | case DELETE: 59 | LOG.info("Received downtime delete request: type={} ids={}", task.getType(), task.getIds()); 60 | service.deleteDowntimes(task.getIds()); 61 | break; 62 | case DELETE_GROUP: 63 | LOG.info("Received downtime group delete request: type={} (unexpected)", task.getType()); 64 | service.deleteDowntimeGroup(task.getGroupId()); 65 | break; 66 | } 67 | } 68 | 69 | } catch (Throwable ex) { 70 | LOG.error("msg={}", ex.getMessage()); 71 | } 72 | } 73 | } -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeRequest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by jmussler on 18.06.16. 7 | */ 8 | public class DowntimeRequest { 9 | private String comment; 10 | private Long startTime; 11 | private Long endTime; 12 | private String createdBy; 13 | private String groupId; 14 | 15 | private List downtimeEntities; 16 | 17 | public DowntimeRequest() { 18 | } 19 | 20 | public String getComment() { 21 | return comment; 22 | } 23 | 24 | public void setComment(String comment) { 25 | this.comment = comment; 26 | } 27 | 28 | public Long getStartTime() { 29 | return startTime; 30 | } 31 | 32 | public void setStartTime(Long startTime) { 33 | this.startTime = startTime; 34 | } 35 | 36 | public Long getEndTime() { 37 | return endTime; 38 | } 39 | 40 | public void setEndTime(Long endTime) { 41 | this.endTime = endTime; 42 | } 43 | 44 | public String getCreatedBy() { 45 | return createdBy; 46 | } 47 | 48 | public void setCreatedBy(String createdBy) { 49 | this.createdBy = createdBy; 50 | } 51 | 52 | public List getDowntimeEntities() { 53 | return downtimeEntities; 54 | } 55 | 56 | public void setDowntimeEntities(List downtimeEntities) { 57 | this.downtimeEntities = downtimeEntities; 58 | } 59 | 60 | public String getGroupId() { 61 | return groupId; 62 | } 63 | 64 | public void setGroupId(String groupId) { 65 | this.groupId = groupId; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeRequestResult.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | /** 7 | * Created by jmussler on 27.06.16. 8 | */ 9 | public class DowntimeRequestResult { 10 | public String groupId; 11 | public Map ids = new HashMap<>(); 12 | 13 | public DowntimeRequestResult() { 14 | } 15 | 16 | public DowntimeRequestResult(String groupId) { 17 | this.groupId = groupId; 18 | } 19 | 20 | public String getGroupId() { 21 | return groupId; 22 | } 23 | 24 | public void setGroupId(String groupId) { 25 | this.groupId = groupId; 26 | } 27 | 28 | public Map getIds() { 29 | return ids; 30 | } 31 | 32 | public void setIds(Map ids) { 33 | this.ids = ids; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimeTaskType.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | /** 4 | * Created by jmussler on 18.06.16. 5 | */ 6 | public enum DowntimeTaskType { 7 | NEW, 8 | DELETE, 9 | DELETE_GROUP 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/downtimes/DowntimesAPI.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.downtimes; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 4 | import de.zalando.zmon.scheduler.ng.scheduler.Scheduler; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.web.bind.annotation.*; 9 | 10 | import java.util.*; 11 | 12 | /** 13 | * Created by jmussler on 18.06.16. 14 | */ 15 | @RestController 16 | public class DowntimesAPI { 17 | 18 | private final Logger log = LoggerFactory.getLogger(DowntimesAPI.class); 19 | 20 | @Autowired 21 | Scheduler scheduler; 22 | 23 | @Autowired 24 | AlertRepository alertRepo; 25 | 26 | @Autowired 27 | DowntimeService downtimeService; 28 | 29 | @Autowired 30 | DowntimeForwarder downtimeForwarder; 31 | 32 | @RequestMapping(value = "/api/v1/downtimes/{dc}/") 33 | Collection getPendingDowntimes(@PathVariable(value = "dc") String dcId) { 34 | return downtimeForwarder.getRequests(dcId); 35 | } 36 | 37 | @RequestMapping(value = "/api/v1/downtimes") 38 | Collection getKnownDowntimeForwardDCs() { 39 | return downtimeForwarder.getKnwonDCs(); 40 | } 41 | 42 | @RequestMapping(value = "/api/v1/downtimes", method = RequestMethod.POST) 43 | DowntimeRequestResult postDowntime(@RequestBody DowntimeRequest request) { 44 | DowntimeRequestResult result = downtimeService.storeDowntime(request); 45 | downtimeForwarder.forwardRequest(DowntimeForwardTask.NewDowntimeTask(request)); 46 | 47 | // trigger evaluation locally 48 | for (DowntimeAlertRequest r : request.getDowntimeEntities()) { 49 | scheduler.executeImmediate(alertRepo.get(r.getAlertId()).getCheckDefinitionId()); 50 | } 51 | 52 | return result; 53 | } 54 | 55 | @RequestMapping(value = "/api/v1/downtimes/{id}", method = RequestMethod.DELETE) 56 | void deleteDowntime(@PathVariable(value = "id") String id) { 57 | log.info("Deleting downtime: id={}", id); 58 | Set ids = new TreeSet<>(); 59 | ids.add(id); 60 | downtimeForwarder.forwardRequest(DowntimeForwardTask.DeleteDowntimeTask(ids)); 61 | downtimeService.deleteDowntimes(ids); 62 | } 63 | 64 | @RequestMapping(value = "/api/v1/downtime-groups/{groupId}", method = RequestMethod.DELETE) 65 | void deleteDowntimeGroup(@PathVariable(value = "groupId") String groupId) { 66 | log.info("Deleting downtime-group: groupId={}", groupId); 67 | downtimeForwarder.forwardRequest(DowntimeForwardTask.DeleteGroupTask(groupId)); 68 | downtimeService.deleteDowntimeGroup(groupId); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/Entity.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by jmussler on 3/31/15. 10 | */ 11 | public class Entity { 12 | private final Map filterProperties = new HashMap<>(); 13 | private final Map properties = new HashMap<>(); 14 | private final String id; 15 | 16 | public Entity(String id) { 17 | this.id = id; 18 | 19 | filterProperties.put("id", id); 20 | properties.put("id", id); 21 | } 22 | 23 | public Entity(String id, String adapterName) { 24 | this(id); 25 | } 26 | 27 | public String getId() { 28 | return id; 29 | } 30 | 31 | public Map getFilterProperties() { 32 | return filterProperties; 33 | } 34 | 35 | public Map getProperties() { 36 | return properties; 37 | } 38 | 39 | private void addFilterProperties(Map valueMap) { 40 | for (Map.Entry e : valueMap.entrySet()) { 41 | if (e.getValue() instanceof String) { 42 | filterProperties.put(e.getKey(), e.getValue()); 43 | } else if (e.getValue() instanceof Integer) { 44 | filterProperties.put(e.getKey(), e.getValue() + ""); 45 | } else if (e.getValue() instanceof java.util.Collection) { 46 | List list = new ArrayList<>(); 47 | for (Object o : (java.util.List) e.getValue()) { 48 | if (o instanceof String || o instanceof Integer) 49 | list.add(o); 50 | } 51 | filterProperties.put(e.getKey(), list); 52 | } 53 | } 54 | } 55 | 56 | public void addProperty(String key, Object value) { 57 | properties.put(key, value); 58 | if (value instanceof String) { 59 | filterProperties.put(key, value); 60 | } else if (value instanceof Integer) { 61 | filterProperties.put(key, value); 62 | } 63 | } 64 | 65 | public void addProperties(Map valueMap) { 66 | properties.putAll(valueMap); 67 | addFilterProperties(valueMap); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/EntityAdapter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities; 2 | 3 | import de.zalando.zmon.scheduler.ng.BaseSource; 4 | 5 | /** 6 | * Created by jmussler on 3/27/15. 7 | */ 8 | public abstract class EntityAdapter extends BaseSource { 9 | public EntityAdapter(String name) { 10 | super(name); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/EntityAdapterRegistry.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities; 2 | 3 | import de.zalando.zmon.scheduler.ng.entities.adapters.*; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Component; 8 | 9 | import com.codahale.metrics.MetricRegistry; 10 | import com.fasterxml.jackson.core.JsonProcessingException; 11 | import com.fasterxml.jackson.databind.ObjectMapper; 12 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 13 | import de.zalando.zmon.scheduler.ng.SourceRegistry; 14 | import org.springframework.web.client.RestTemplate; 15 | 16 | import java.io.UnsupportedEncodingException; 17 | import java.net.URI; 18 | import java.net.URLEncoder; 19 | 20 | /** 21 | * Created by jmussler on 4/1/15. 22 | */ 23 | @Component 24 | public class EntityAdapterRegistry extends SourceRegistry { 25 | 26 | private final static Logger LOG = LoggerFactory.getLogger(EntityAdapterRegistry.class); 27 | 28 | private final static EntityAdapter EMPTY_ADAPTER = new EmptyAdapter(); 29 | 30 | public EntityAdapterRegistry(MetricRegistry metrics) { 31 | } 32 | 33 | @Autowired(required = false) 34 | public EntityAdapterRegistry(SchedulerConfig config, MetricRegistry metrics, RestTemplate restTemplate) { 35 | register(EMPTY_ADAPTER); 36 | 37 | if (config.isEnableGlobalEntity()) { 38 | register(new GlobalAdapter()); 39 | } 40 | 41 | if (config.getEntityServiceUrl() != null && !config.getEntityServiceUrl().equals("")) { 42 | String entityServiceUrl = config.getEntityServiceUrl() + "/api/v1/entities"; 43 | try { 44 | if (config.getEntityBaseFilterStr() != null && !"".equals(config.getEntityBaseFilterStr()) && config.isBaseFilterForward()) { 45 | entityServiceUrl = entityServiceUrl + "?query=" + URLEncoder.encode(config.getEntityBaseFilterStr(), "UTF-8"); 46 | } else if (config.getEntitySkipOnField() != null && !"".equals(config.getEntitySkipOnField())) { 47 | try { 48 | entityServiceUrl = entityServiceUrl + "?exclude=" + URLEncoder.encode(config.getEntitySkipOnField(), "UTF-8"); 49 | } catch(UnsupportedEncodingException ex) { 50 | LOG.error("Encoding of skip filter query param failed"); 51 | } 52 | } else if (config.getEntityBaseFilter() != null && config.isBaseFilterForward()) { 53 | ObjectMapper mapper = new ObjectMapper(); 54 | try { 55 | entityServiceUrl = entityServiceUrl + "?query=" + URLEncoder.encode(mapper.writeValueAsString(config.getEntityBaseFilter()), "UTF-8"); 56 | } catch (JsonProcessingException ex) { 57 | LOG.error("Could not serialize base filter: {}", ex.getMessage()); 58 | } 59 | } 60 | } 61 | catch(UnsupportedEncodingException ex) { 62 | LOG.error("Encoding of base filter query param failed"); 63 | } 64 | 65 | EntityServiceAdapter e = new EntityServiceAdapter(URI.create(entityServiceUrl), metrics, restTemplate); 66 | register(e); 67 | } 68 | 69 | if (config.getDummyCities() != null && !config.getDummyCities().equals("")) { 70 | register(new YamlEntityAdapter("dummy-cities", config.getDummyCities(), "city")); 71 | } 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/EntityChangeListener.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities; 2 | 3 | /** 4 | * Created by jmussler on 4/17/15. 5 | */ 6 | public interface EntityChangeListener { 7 | void notifyEntityChange(EntityRepository repo, Entity entityOld, Entity entityNew); 8 | 9 | void notifyEntityRemove(EntityRepository repo, Entity e); 10 | 11 | void notifyEntityAdd(EntityRepository repo, Entity e); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/Environments.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | /** 7 | * Created by jmussler on 4/16/15. 8 | */ 9 | public class Environments { 10 | private final static Map environmentMap = new HashMap<>(); 11 | 12 | static { 13 | environmentMap.put("release", "release-staging"); 14 | environmentMap.put("release-staging", "release-staging"); 15 | environmentMap.put("be-staging", "release-staging"); 16 | environmentMap.put("release.staging", "release-staging"); 17 | environmentMap.put("patch", "patch-staging"); 18 | environmentMap.put("fe-staging", "patch-staging"); 19 | environmentMap.put("patch.staging", "patch-staging"); 20 | environmentMap.put("patch-staging", "patch-staging"); 21 | environmentMap.put("perf", "performance-staging"); 22 | environmentMap.put("perf-staging", "performance-staging"); 23 | environmentMap.put("perf.staging", "performance-staging"); 24 | environmentMap.put("performance-staging", "performance-staging"); 25 | environmentMap.put("live", "live"); 26 | environmentMap.put("integration", "integration"); 27 | } 28 | 29 | public static String getNormalized(String name) { 30 | String v = environmentMap.get(name); 31 | if (null == v) return name; 32 | return v; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/Teams.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities; 2 | 3 | /** 4 | * Created by jmussler on 4/16/15. 5 | */ 6 | public class Teams { 7 | public static String getNormalizedTeam(String team) { 8 | if (null == team) { 9 | return ""; 10 | } 11 | 12 | if (team.startsWith("Zalando/Technology/")) { 13 | return team.substring(19); 14 | } 15 | 16 | return team; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/adapters/EmptyAdapter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities.adapters; 2 | 3 | import de.zalando.zmon.scheduler.ng.entities.Entity; 4 | import de.zalando.zmon.scheduler.ng.entities.EntityAdapter; 5 | 6 | import java.util.ArrayList; 7 | import java.util.Collection; 8 | 9 | /** 10 | * Created by jmussler on 4/2/15. 11 | */ 12 | public class EmptyAdapter extends EntityAdapter { 13 | public EmptyAdapter() { 14 | super("empty-entity-adapter"); 15 | } 16 | 17 | @Override 18 | public Collection getCollection() { 19 | return new ArrayList<>(0); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/adapters/EntityServiceAdapter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities.adapters; 2 | 3 | import com.codahale.metrics.MetricRegistry; 4 | import com.codahale.metrics.Timer; 5 | import de.zalando.zmon.scheduler.ng.entities.Entity; 6 | import de.zalando.zmon.scheduler.ng.entities.EntityAdapter; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.http.HttpEntity; 11 | import org.springframework.http.HttpHeaders; 12 | import org.springframework.http.HttpMethod; 13 | import org.springframework.http.ResponseEntity; 14 | import org.springframework.web.client.RestTemplate; 15 | 16 | import java.net.URI; 17 | import java.util.ArrayList; 18 | import java.util.Collection; 19 | import java.util.HashMap; 20 | import java.util.List; 21 | 22 | /** 23 | * Created by jmussler on 4/2/15. 24 | */ 25 | public class EntityServiceAdapter extends EntityAdapter { 26 | 27 | private URI url; 28 | 29 | private final Timer timer; 30 | private boolean isFirstLoad = true; 31 | 32 | private RestTemplate restTemplate; 33 | 34 | private static final Logger LOG = LoggerFactory.getLogger(EntityServiceAdapter.class); 35 | 36 | @Autowired 37 | public EntityServiceAdapter(URI url, MetricRegistry metrics, RestTemplate restTemplate) { 38 | super("EntityServiceAdapter"); 39 | this.restTemplate = restTemplate; 40 | LOG.info("configuring entity service url={}", url); 41 | this.url = url; 42 | this.timer = metrics.timer("entity-adapter.entity-service"); 43 | } 44 | 45 | private static class BaseEntity extends HashMap { 46 | } 47 | 48 | private static class BaseEntityList extends ArrayList { 49 | } 50 | 51 | @Override 52 | public Collection getCollection() { 53 | HttpEntity request; 54 | 55 | LOG.info("Querying entities with token..."); 56 | request = new HttpEntity<>(new HttpHeaders()); 57 | 58 | try { 59 | Timer.Context tC = timer.time(); 60 | ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, request, BaseEntityList.class); 61 | LOG.info("Entity Service Adapter used: {}ms", tC.stop() / 1000000); 62 | 63 | BaseEntityList list = response.getBody(); 64 | List entityList = new ArrayList<>(list.size()); 65 | 66 | for (BaseEntity base : list) { 67 | Entity entity = new Entity((String) base.get("id"), this.getName()); 68 | entity.addProperties(base); 69 | entityList.add(entity); 70 | } 71 | 72 | isFirstLoad = false; 73 | return entityList; 74 | } catch (Throwable t) { 75 | LOG.error("Failed to get entities: {}", t.getMessage()); 76 | if (!isFirstLoad) { 77 | // rethrow, continue to used already loaded entities 78 | throw t; 79 | } 80 | } 81 | return new ArrayList<>(0); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/adapters/GlobalAdapter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities.adapters; 2 | 3 | import de.zalando.zmon.scheduler.ng.entities.Entity; 4 | import de.zalando.zmon.scheduler.ng.entities.EntityAdapter; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.*; 9 | 10 | /** 11 | * Created by jmussler on 4/2/15. 12 | */ 13 | public class GlobalAdapter extends EntityAdapter { 14 | public GlobalAdapter() { 15 | super("global-adapter"); 16 | } 17 | 18 | private final Logger LOG = LoggerFactory.getLogger(GlobalAdapter.class); 19 | 20 | private final static Entity GLOBAL_ENTITY = new Entity("GLOBAL", "global-adapter"); 21 | private final static List GLOBAL_COLLECTION = new ArrayList<>(1); 22 | 23 | static { 24 | Map p = new HashMap<>(); 25 | p.put("type", "GLOBAL"); 26 | GLOBAL_ENTITY.addProperties(p); 27 | GLOBAL_COLLECTION.add(GLOBAL_ENTITY); 28 | } 29 | 30 | @Override 31 | public Collection getCollection() { 32 | LOG.info("Returning 1 global entity"); 33 | return GLOBAL_COLLECTION; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/entities/adapters/YamlEntityAdapter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities.adapters; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 6 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; 7 | import de.zalando.zmon.scheduler.ng.entities.Entity; 8 | import de.zalando.zmon.scheduler.ng.entities.EntityAdapter; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.File; 13 | import java.util.ArrayList; 14 | import java.util.Collection; 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | /** 19 | * Created by jmussler on 4/17/15. 20 | */ 21 | public class YamlEntityAdapter extends EntityAdapter { 22 | 23 | public interface IdGenerator { 24 | String f(Map e); 25 | } 26 | 27 | private static final Logger LOG = LoggerFactory.getLogger(YamlEntityAdapter.class); 28 | 29 | private String fileName; 30 | private String type = null; 31 | private IdGenerator idGenerator = m -> (String) m.get("id"); 32 | 33 | private static final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); 34 | 35 | static { 36 | mapper.setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); 37 | } 38 | 39 | public YamlEntityAdapter(String name, String fileName, String type, IdGenerator g) { 40 | super(name); 41 | this.fileName = fileName; 42 | this.type = type; 43 | this.idGenerator = g; 44 | } 45 | 46 | public YamlEntityAdapter(String name, String fileName, String type) { 47 | super(name); 48 | this.fileName = fileName; 49 | this.type = type; 50 | } 51 | 52 | public YamlEntityAdapter(String name, String fileName) { 53 | super(name); 54 | this.fileName = fileName; 55 | } 56 | 57 | @Override 58 | public Collection getCollection() { 59 | try { 60 | List> list = mapper.readValue(new File(fileName), new TypeReference>>() { 61 | }); 62 | List entityList = new ArrayList<>(); 63 | 64 | for (Map m : list) { 65 | String id = idGenerator.f(m); 66 | if (null == id || id.equals("")) continue; 67 | Entity e = new Entity(id, getName()); 68 | if (type != null) { 69 | m.put("type", type); 70 | } 71 | e.addProperties(m); 72 | entityList.add(e); 73 | } 74 | 75 | return entityList; 76 | } catch (Exception e) { 77 | LOG.error("", e); 78 | } 79 | return new ArrayList<>(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/instantevaluations/InstantEvalForwarder.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.instantevaluations; 2 | 3 | import de.zalando.zmon.scheduler.ng.DataCenterSubscriber; 4 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * Created by jmussler on 5/22/15. 10 | */ 11 | @Component 12 | public class InstantEvalForwarder extends DataCenterSubscriber { 13 | 14 | @Autowired 15 | public InstantEvalForwarder(SchedulerConfig config) { 16 | super(config.isInstantEvalForward()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/instantevaluations/InstantEvalHttpSubscriber.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.instantevaluations; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | import de.zalando.zmon.scheduler.ng.scheduler.Scheduler; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.core.ParameterizedTypeReference; 8 | import org.springframework.http.HttpEntity; 9 | import org.springframework.http.HttpHeaders; 10 | import org.springframework.http.HttpMethod; 11 | import org.springframework.http.ResponseEntity; 12 | import org.springframework.web.client.RestTemplate; 13 | 14 | import java.util.List; 15 | import java.util.concurrent.ScheduledExecutorService; 16 | import java.util.concurrent.ScheduledThreadPoolExecutor; 17 | import java.util.concurrent.TimeUnit; 18 | 19 | /** 20 | * Created by jmussler on 5/22/15. 21 | */ 22 | public class InstantEvalHttpSubscriber implements Runnable { 23 | 24 | private final static Logger LOG = LoggerFactory.getLogger(InstantEvalHttpSubscriber.class); 25 | 26 | private final String url; 27 | private final Scheduler scheduler; 28 | private final ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1); 29 | private final RestTemplate restTemplate; 30 | 31 | public InstantEvalHttpSubscriber(Scheduler scheduler, SchedulerConfig config, RestTemplate restTemplate) { 32 | url = config.getInstantEvalHttpUrl(); 33 | this.restTemplate = restTemplate; 34 | 35 | LOG.info("Subscribing for instant evaluations: {}", url); 36 | this.scheduler = scheduler; 37 | if (url != null && !url.equals("")) { 38 | executor.scheduleAtFixedRate(this, 60, 5, TimeUnit.SECONDS); 39 | } 40 | } 41 | 42 | @Override 43 | public void run() { 44 | try { 45 | 46 | HttpEntity request; 47 | 48 | HttpHeaders headers = new HttpHeaders(); 49 | request = new HttpEntity<>(headers); 50 | 51 | ResponseEntity> response = restTemplate.exchange(url, HttpMethod.GET, request, new ParameterizedTypeReference>() { 52 | }); 53 | 54 | for (Integer checkId : response.getBody()) { 55 | LOG.info("Received instant evaluation request: {}", checkId); 56 | scheduler.executeImmediate(checkId); 57 | } 58 | } catch (Throwable ex) { 59 | LOG.error("msg={}", ex.getMessage()); 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/instantevaluations/InstantEvaluationAPI.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.instantevaluations; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 4 | import de.zalando.zmon.scheduler.ng.scheduler.Scheduler; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.web.bind.annotation.PathVariable; 7 | import org.springframework.web.bind.annotation.RequestMapping; 8 | import org.springframework.web.bind.annotation.RequestMethod; 9 | import org.springframework.web.bind.annotation.RestController; 10 | 11 | import java.util.Collection; 12 | 13 | /** 14 | * Created by jmussler on 18.06.16. 15 | */ 16 | @RestController 17 | public class InstantEvaluationAPI { 18 | 19 | @Autowired 20 | AlertRepository alertRepo; 21 | 22 | @Autowired 23 | Scheduler scheduler; 24 | 25 | @Autowired 26 | private InstantEvalForwarder instantEvalForwarder; 27 | 28 | @RequestMapping(value = "/api/v1/instant-evaluations/{dc}/") 29 | Collection getPendingInstantEvaluations(@PathVariable(value = "dc") String dcId) { 30 | return instantEvalForwarder.getRequests(dcId); 31 | } 32 | 33 | @RequestMapping(value = "/api/v1/instant-evaluations/") 34 | Collection getKnownInstantEvalForwardDCs() { 35 | return instantEvalForwarder.getKnwonDCs(); 36 | } 37 | 38 | @RequestMapping(value = "/api/v1/checks/{id}/instant-eval", method = RequestMethod.POST) 39 | public void triggerInstantEvaluationByCheck(@PathVariable(value = "id") int checkId) { 40 | scheduler.executeImmediate(checkId); 41 | instantEvalForwarder.forwardRequest(checkId); 42 | } 43 | 44 | @RequestMapping(value = "/api/v1/alerts/{id}/instant-eval", method = RequestMethod.POST) 45 | public void triggerInstantEvaluation(@PathVariable(value = "id") int id) { 46 | int checkId = alertRepo.get(id).getCheckDefinitionId(); 47 | scheduler.executeImmediate(checkId); 48 | instantEvalForwarder.forwardRequest(checkId); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/ArrayQueueWriter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import com.codahale.metrics.MetricRegistry; 4 | 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | 8 | /** 9 | * Created by jmussler on 30.06.16. 10 | */ 11 | public class ArrayQueueWriter extends QueueWriter { 12 | private final HashMap> tasks = new HashMap<>(); 13 | 14 | public ArrayQueueWriter(MetricRegistry metrics) { 15 | super(metrics); 16 | } 17 | 18 | @Override 19 | protected void write(String queue, byte[] command) { 20 | synchronized (this) { 21 | ArrayList l = tasks.getOrDefault(queue, null); 22 | if (null == l) { 23 | l = new ArrayList<>(); 24 | tasks.put(queue, l); 25 | } 26 | l.add(command); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/HardCodedSelector.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import de.zalando.zmon.scheduler.ng.Alert; 4 | import de.zalando.zmon.scheduler.ng.Check; 5 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 6 | import de.zalando.zmon.scheduler.ng.entities.Entity; 7 | 8 | 9 | import java.util.Collection; 10 | import java.util.HashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | /** 15 | * Created by jmussler on 30.06.16. 16 | */ 17 | public class HardCodedSelector implements Selector { 18 | 19 | private final Map queueMapping = new HashMap<>(); 20 | 21 | public HardCodedSelector(SchedulerConfig config) { 22 | for(Map.Entry> entry : config.getQueueMapping().entrySet()) { 23 | for(Integer id : entry.getValue()) { 24 | queueMapping.put(id, entry.getKey()); 25 | } 26 | } 27 | } 28 | 29 | @Override 30 | public String getQueue(Entity entity, Check check, Collection alerts) { 31 | if (null == check) { 32 | return null; 33 | } 34 | 35 | return queueMapping.getOrDefault(check.getId(), null); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/JedisQueueWriter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import com.codahale.metrics.MetricRegistry; 4 | import redis.clients.jedis.Jedis; 5 | import redis.clients.jedis.JedisPool; 6 | import redis.clients.jedis.JedisPoolConfig; 7 | 8 | /** 9 | * Created by jmussler on 30.06.16. 10 | */ 11 | public class JedisQueueWriter extends QueueWriter { 12 | 13 | private final JedisPool pool; 14 | 15 | public JedisQueueWriter(String host, int port, MetricRegistry metrics) { 16 | super(metrics); 17 | JedisPoolConfig config = new JedisPoolConfig(); 18 | config.setMinIdle(8); 19 | this.pool = new JedisPool(config, host, port); 20 | } 21 | 22 | @Override 23 | protected void write(String queue, byte[] command) { 24 | try(Jedis jedis = pool.getResource()) { 25 | jedis.rpush(queue.getBytes(), command); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/LogQueueWriter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import com.codahale.metrics.MetricRegistry; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | /** 8 | * Created by jmussler on 30.06.16. 9 | */ 10 | public class LogQueueWriter extends QueueWriter { 11 | 12 | private final Logger logger; 13 | 14 | public LogQueueWriter(MetricRegistry metrics) { 15 | super(metrics); 16 | logger = LoggerFactory.getLogger(LogQueueWriter.class); 17 | } 18 | 19 | @Override 20 | public void write(String queue, byte[] command) { 21 | logger.info("q={} command={}", queue, command); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/PropertyQueueSelector.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import de.zalando.zmon.scheduler.ng.Alert; 4 | import de.zalando.zmon.scheduler.ng.AlertOverlapGenerator; 5 | import de.zalando.zmon.scheduler.ng.Check; 6 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 7 | import de.zalando.zmon.scheduler.ng.entities.Entity; 8 | 9 | import java.util.Collection; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | /** 14 | * Created by jmussler on 30.06.16. 15 | */ 16 | public class PropertyQueueSelector implements Selector { 17 | 18 | private final SchedulerConfig config; 19 | 20 | public PropertyQueueSelector(SchedulerConfig config) { 21 | this.config = config; 22 | } 23 | 24 | @Override 25 | public String getQueue(Entity entity, Check check, Collection alerts) { 26 | if (null == check) { 27 | return null; 28 | } 29 | 30 | for(Map.Entry>> entry : config.getQueuePropertyMapping().entrySet()) { 31 | for(Map ps : entry.getValue()) { 32 | if(AlertOverlapGenerator.filter(ps, entity.getFilterProperties())) { 33 | return entry.getKey(); 34 | } 35 | } 36 | } 37 | 38 | return null; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/QueueMetrics.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import com.codahale.metrics.Meter; 4 | import com.codahale.metrics.MetricRegistry; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | /** 10 | * Created by jmussler on 30.06.16. 11 | */ 12 | public class QueueMetrics { 13 | private final MetricRegistry metrics; 14 | 15 | private final Map meters = new HashMap<>(); 16 | private final Meter throughPutMeter; 17 | 18 | public QueueMetrics(MetricRegistry metrics) { 19 | this.metrics = metrics; 20 | this.throughPutMeter = metrics.meter("scheduler.byte-throughput"); 21 | } 22 | 23 | public void incThroughput(int l) { 24 | throughPutMeter.mark(l); 25 | } 26 | 27 | public void mark(String q) { 28 | if(meters.containsKey(q)) { 29 | meters.get(q).mark(); 30 | } 31 | else { 32 | synchronized (this) { 33 | if(meters.containsKey(q)) { 34 | meters.get(q).mark(); 35 | return; 36 | } 37 | 38 | Meter m = metrics.meter("scheduler.queue." + q); 39 | meters.put(q, m); 40 | m.mark(); 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/QueueSelector.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import de.zalando.zmon.scheduler.ng.Alert; 4 | import de.zalando.zmon.scheduler.ng.Check; 5 | import de.zalando.zmon.scheduler.ng.CommandSerializer; 6 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 7 | import de.zalando.zmon.scheduler.ng.entities.Entity; 8 | 9 | import io.opentracing.Tracer; 10 | 11 | import java.util.ArrayList; 12 | import java.util.Collection; 13 | import java.util.List; 14 | 15 | /** 16 | * Created by jmussler on 30.06.16. 17 | */ 18 | public class QueueSelector { 19 | 20 | private final QueueWriter writer; 21 | private final SchedulerConfig config; 22 | private final List selectors = new ArrayList<>(); 23 | private final CommandSerializer serializer; 24 | private final PropertyQueueSelector propertySelector; 25 | 26 | public QueueSelector(QueueWriter writer, SchedulerConfig config, Tracer tracer) { 27 | this.writer = writer; 28 | this.config = config; 29 | this.propertySelector = new PropertyQueueSelector(config); 30 | 31 | selectors.add(new RepoSelector(config)); 32 | selectors.add(new HardCodedSelector(config)); 33 | selectors.add(propertySelector); 34 | 35 | serializer = new CommandSerializer(config.getTaskSerializer(), tracer); 36 | } 37 | 38 | public void execute(Entity entity, byte[] command) { 39 | execute(entity, command, null); 40 | } 41 | 42 | public void execute(Entity entity, byte[] command, String targetQueue) { 43 | String queue = targetQueue; 44 | if (null == queue) { 45 | queue = propertySelector.getQueue(entity, null, null); 46 | } 47 | 48 | if (null == queue) { 49 | queue = config.getDefaultQueue(); 50 | } 51 | 52 | writer.exec(queue, command); 53 | } 54 | 55 | public void execute(Entity entity, Check check, Collection alerts, long scheduledTime) { 56 | byte[] command = serializer.write(entity, check, alerts, scheduledTime); 57 | 58 | String queue = null; 59 | 60 | for(Selector s : selectors) { 61 | if (null == queue) { 62 | queue = s.getQueue(entity, check, alerts); 63 | } 64 | } 65 | 66 | if (null == queue) { 67 | queue = config.getDefaultQueue(); 68 | } 69 | 70 | writer.exec(queue, command); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/QueueWriter.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import com.codahale.metrics.MetricRegistry; 4 | 5 | /** 6 | * Created by jmussler on 30.06.16. 7 | */ 8 | public abstract class QueueWriter { 9 | private final QueueMetrics metrics; 10 | 11 | public QueueWriter(MetricRegistry metrics) { 12 | this.metrics = new QueueMetrics(metrics); 13 | } 14 | 15 | public void exec(String queue, byte[] command) { 16 | write(queue, command); 17 | metrics.mark(queue); 18 | metrics.incThroughput(command.length); 19 | 20 | } 21 | 22 | protected abstract void write(String queue, byte[] command); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/RepoSelector.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import de.zalando.zmon.scheduler.ng.Alert; 4 | import de.zalando.zmon.scheduler.ng.Check; 5 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 6 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 7 | import de.zalando.zmon.scheduler.ng.entities.Entity; 8 | 9 | import java.util.Collection; 10 | import java.util.Map; 11 | 12 | /** 13 | * Created by jmussler on 30.06.16. 14 | */ 15 | public class RepoSelector implements Selector { 16 | 17 | private final SchedulerConfig config; 18 | 19 | public RepoSelector(SchedulerConfig config) { 20 | this.config = config; 21 | } 22 | 23 | @Override 24 | public String getQueue(Entity entity, Check check, Collection alerts) { 25 | if (null == check) { 26 | return null; 27 | } 28 | 29 | for(Map.Entry entry : config.getQueueMappingByUrl().entrySet()) { 30 | CheckDefinition checkDefinition = check.getCheckDefinition(); 31 | if (null != checkDefinition) { 32 | if (null != checkDefinition.getSourceUrl() && checkDefinition.getSourceUrl().startsWith(entry.getKey())) { 33 | return entry.getValue(); 34 | } 35 | } 36 | } 37 | 38 | return null; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/queue/Selector.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.queue; 2 | 3 | import de.zalando.zmon.scheduler.ng.Alert; 4 | import de.zalando.zmon.scheduler.ng.Check; 5 | import de.zalando.zmon.scheduler.ng.entities.Entity; 6 | 7 | import java.util.Collection; 8 | 9 | /** 10 | * Created by jmussler on 30.06.16. 11 | */ 12 | public interface Selector { 13 | String getQueue(Entity entity, Check check, Collection alerts); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/scheduler/RedisMetricsUpdater.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.scheduler; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import redis.clients.jedis.Jedis; 7 | import redis.clients.jedis.Pipeline; 8 | 9 | import java.net.InetAddress; 10 | import java.net.UnknownHostException; 11 | 12 | /** 13 | * Created by jmussler on 30.06.16. 14 | */ 15 | public class RedisMetricsUpdater implements Runnable { 16 | 17 | private final String name; 18 | private final SchedulerConfig config; 19 | private final SchedulerMetrics metrics; 20 | private final Logger logger = LoggerFactory.getLogger(RedisMetricsUpdater.class); 21 | 22 | public RedisMetricsUpdater(SchedulerConfig config, SchedulerMetrics metrics) { 23 | String n; 24 | try { 25 | n = "s-p" + config.getServerPort() + "." + InetAddress.getLocalHost().getHostName(); 26 | } 27 | catch (UnknownHostException ex) { 28 | n = "s-p" + config.getServerPort() + ".unknown"; 29 | } 30 | name = n; 31 | this.config = config; 32 | this.metrics = metrics; 33 | } 34 | 35 | @Override 36 | public void run() { 37 | try(Jedis jedis = new Jedis(config.getRedisHost(), config.getRedisPort())) { 38 | Pipeline p = jedis.pipelined(); 39 | p.sadd("zmon:metrics", name); 40 | p.set("zmon:metrics:" + name + ":check.count", metrics.getTotalChecks() + ""); 41 | p.set("zmon:metrics:" + name + ":ts", System.currentTimeMillis() / 1000 + ""); 42 | p.sync(); 43 | } 44 | catch (Throwable t) { 45 | logger.error("Metrics update failed: {} host={} port={}", t.getMessage(), config.getRedisHost(), config.getRedisPort()); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/scheduler/SchedulePersister.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.scheduler; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | /** 7 | * Created by jmussler on 30.06.16. 8 | */ 9 | public class SchedulePersister implements Runnable { 10 | 11 | public static Map loadSchedule() { 12 | return new HashMap<>(); 13 | } 14 | 15 | public static void writeSchedule(Map schedule) { 16 | } 17 | 18 | @Override 19 | public void run() { 20 | 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/scheduler/ScheduledCheck.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.scheduler; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collection; 5 | import java.util.List; 6 | import java.util.concurrent.ScheduledExecutorService; 7 | import java.util.concurrent.ScheduledFuture; 8 | import java.util.concurrent.TimeUnit; 9 | import java.util.stream.Collectors; 10 | 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import com.codahale.metrics.Meter; 15 | 16 | import de.zalando.zmon.scheduler.ng.Alert; 17 | import de.zalando.zmon.scheduler.ng.AlertOverlapGenerator; 18 | import de.zalando.zmon.scheduler.ng.Check; 19 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 20 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 21 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 22 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 23 | import de.zalando.zmon.scheduler.ng.entities.Entity; 24 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 25 | import de.zalando.zmon.scheduler.ng.queue.QueueSelector; 26 | 27 | /** 28 | * Created by jmussler on 30.06.16. 29 | */ 30 | public class ScheduledCheck implements Runnable { 31 | private static final Logger LOG = LoggerFactory.getLogger(ScheduledCheck.class); 32 | 33 | private long lastRun = 0; 34 | private final int id; 35 | private final Check check; 36 | private final Meter meter; 37 | private final AlertRepository alertRepo; 38 | private final EntityRepository entityRepo; 39 | private final QueueSelector selector; 40 | private final SchedulerMetrics metrics; 41 | 42 | private ScheduledFuture taskFuture = null; 43 | private volatile boolean cancel = false; 44 | 45 | private final List lastRunEntities = new ArrayList<>(0); 46 | 47 | public ScheduledCheck(int id, QueueSelector selector, CheckRepository checkRepo, AlertRepository alertRepo, 48 | EntityRepository entityRepo, SchedulerConfig config, SchedulerMetrics metrics) { 49 | 50 | this.id = id; 51 | this.alertRepo = alertRepo; 52 | this.entityRepo = entityRepo; 53 | this.selector = selector; 54 | this.metrics = metrics; 55 | 56 | this.check = new Check(id, checkRepo); 57 | if(config.isCheckDetailMetrics()) { 58 | this.meter = metrics.getMetrics().meter("scheduler.check." + id); 59 | } 60 | else { 61 | this.meter = null; 62 | } 63 | } 64 | 65 | public synchronized void schedule(ScheduledExecutorService service, long delay) { 66 | if (null == taskFuture && delay > 0) { 67 | lastRun = System.currentTimeMillis() - (check.getCheckDefinition().getInterval() * 1000L - delay * 1000L); 68 | } 69 | 70 | if (null != taskFuture) { 71 | taskFuture.cancel(false); 72 | } 73 | 74 | taskFuture = service.scheduleAtFixedRate(this, delay, check.getCheckDefinition().getInterval(), TimeUnit.SECONDS); 75 | } 76 | 77 | public void execute(Entity entity, Collection alerts) { 78 | if (cancel) { 79 | taskFuture.cancel(false); 80 | LOG.info("Canceling future executions of: id={}", check.getId()); 81 | return; 82 | } 83 | 84 | selector.execute(entity, check, alerts, lastRun); 85 | 86 | 87 | if (null != meter) { 88 | meter.mark(); 89 | } 90 | metrics.incTotal(); 91 | } 92 | 93 | public Collection getAlerts() { 94 | return alertRepo.getByCheckId(check.getId()).stream().map(x -> new Alert(x.getId(), alertRepo)).collect(Collectors.toList()); 95 | } 96 | 97 | public List runCheck() { 98 | return runCheck(false); 99 | } 100 | 101 | public List runCheck(boolean dryRun) { 102 | lastRunEntities.clear(); 103 | 104 | boolean setLastRun = false; 105 | CheckDefinition checkDef = check.getCheckDefinition(); 106 | if (null == checkDef) { 107 | LOG.debug("Probably inactive/deleted check still scheduled: " + check.getId()); 108 | return lastRunEntities; 109 | } 110 | 111 | if (checkDef.getInterval() <= 15 && (System.currentTimeMillis() - lastRun < (checkDef.getInterval() * 750L))) { 112 | // skip high frequency checks too close to last execution 113 | return lastRunEntities; 114 | } 115 | 116 | for(Entity entity : entityRepo.get()) { 117 | if (AlertOverlapGenerator.matchCheckFilter(checkDef, entity)) { 118 | List viableAlerts = getAlerts().stream().filter(x->x.matchEntity(entity)).collect(Collectors.toList()); 119 | 120 | if(!viableAlerts.isEmpty()) { 121 | if(!dryRun) { 122 | if(!setLastRun) { 123 | lastRun = System.currentTimeMillis(); 124 | setLastRun = true; 125 | } 126 | execute(entity, viableAlerts); 127 | } 128 | lastRunEntities.add(entity); 129 | } 130 | } 131 | } 132 | 133 | return lastRunEntities; 134 | } 135 | 136 | @Override 137 | public void run() { 138 | try { 139 | runCheck(); 140 | } 141 | catch(Throwable t) { 142 | metrics.incError(); 143 | LOG.error("Error in check execution: checkId={} msg={}", id, t.getMessage()); 144 | } 145 | } 146 | 147 | public void cancelExecution() { 148 | cancel = true; 149 | } 150 | 151 | public long getLastRun() { 152 | return lastRun; 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/scheduler/SchedulerMetrics.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.scheduler; 2 | 3 | import com.codahale.metrics.Meter; 4 | import com.codahale.metrics.MetricRegistry; 5 | 6 | /** 7 | * Created by jmussler on 30.06.16. 8 | */ 9 | public class SchedulerMetrics { 10 | private final Meter totalChecks; 11 | private final Meter errorCount; 12 | private final MetricRegistry metrics; 13 | 14 | public SchedulerMetrics(MetricRegistry metrics) { 15 | totalChecks = metrics.meter("scheduler.total-checks"); 16 | errorCount = metrics.meter("scheduler.total-errors"); 17 | this.metrics = metrics; 18 | } 19 | 20 | public void incTotal() { 21 | totalChecks.mark(); 22 | } 23 | 24 | public void incError() { 25 | errorCount.mark(); 26 | } 27 | 28 | public long getTotalChecks() { 29 | return this.totalChecks.getCount(); 30 | } 31 | 32 | public long getErrorCount() { 33 | return errorCount.getCount(); 34 | } 35 | 36 | public MetricRegistry getMetrics() { 37 | return metrics; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/trailruns/TrialRunAPI.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.trailruns; 2 | 3 | import de.zalando.zmon.scheduler.ng.scheduler.Scheduler; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.web.bind.annotation.*; 6 | 7 | import java.util.Collection; 8 | 9 | /** 10 | * Created by jmussler on 18.06.16. 11 | */ 12 | @RestController 13 | public class TrialRunAPI { 14 | 15 | @Autowired 16 | Scheduler scheduler; 17 | 18 | @Autowired 19 | private TrialRunForwarder trialRunForwarder; 20 | 21 | @RequestMapping(value = "/api/v1/trial-runs/") 22 | Collection getKnownTrialRunDCs() { 23 | return trialRunForwarder.getKnwonDCs(); 24 | } 25 | 26 | @RequestMapping(value = "/api/v1/trial-runs/{dc}/") 27 | Collection getPendingTrialRuns(@PathVariable(value = "dc") String dcId) { 28 | return trialRunForwarder.getRequests(dcId); 29 | } 30 | 31 | @RequestMapping(value = "/api/v1/trial-runs", method = RequestMethod.POST) 32 | public void postTrialRun(@RequestBody TrialRunRequest trialRun) { 33 | scheduler.scheduleTrialRun(trialRun); 34 | trialRunForwarder.forwardRequest(trialRun); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/trailruns/TrialRunForwarder.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.trailruns; 2 | 3 | import de.zalando.zmon.scheduler.ng.DataCenterSubscriber; 4 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * Created by jmussler on 5/22/15. 10 | */ 11 | @Component 12 | public class TrialRunForwarder extends DataCenterSubscriber { 13 | 14 | @Autowired 15 | public TrialRunForwarder(SchedulerConfig config) { 16 | super(config.isTrialRunForward()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/trailruns/TrialRunHttpSubscriber.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.trailruns; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | import de.zalando.zmon.scheduler.ng.TokenWrapper; 5 | import de.zalando.zmon.scheduler.ng.scheduler.Scheduler; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.core.ParameterizedTypeReference; 9 | import org.springframework.http.HttpEntity; 10 | import org.springframework.http.HttpHeaders; 11 | import org.springframework.http.HttpMethod; 12 | import org.springframework.http.ResponseEntity; 13 | import org.springframework.web.client.RestTemplate; 14 | 15 | import java.util.List; 16 | import java.util.concurrent.ScheduledExecutorService; 17 | import java.util.concurrent.ScheduledThreadPoolExecutor; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | /** 21 | * Created by jmussler on 5/22/15. 22 | */ 23 | public class TrialRunHttpSubscriber implements Runnable { 24 | 25 | private final static Logger LOG = LoggerFactory.getLogger(TrialRunHttpSubscriber.class); 26 | 27 | private final String url; 28 | private final Scheduler scheduler; 29 | private final ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1); 30 | private final RestTemplate restTemplate; 31 | 32 | public TrialRunHttpSubscriber(Scheduler scheduler, SchedulerConfig config, RestTemplate restTemplate) { 33 | url = config.getTrialRunHttpUrl(); 34 | this.restTemplate = restTemplate; 35 | 36 | LOG.info("Subscribing for trial runs: {}", url); 37 | this.scheduler = scheduler; 38 | if (url != null && !url.equals("")) { 39 | executor.scheduleAtFixedRate(this, 60, 5, TimeUnit.SECONDS); 40 | } 41 | } 42 | 43 | @Override 44 | public void run() { 45 | try { 46 | HttpEntity request; 47 | 48 | HttpHeaders headers = new HttpHeaders(); 49 | request = new HttpEntity<>(headers); 50 | 51 | ResponseEntity> response = restTemplate.exchange(url, HttpMethod.GET, request, new ParameterizedTypeReference>() { 52 | }); 53 | 54 | for (TrialRunRequest trialRunRequest : response.getBody()) { 55 | LOG.info("Received trial run request: {}", trialRunRequest); 56 | scheduler.scheduleTrialRun(trialRunRequest); 57 | } 58 | } catch (Throwable ex) { 59 | LOG.error("msg={}", ex.getMessage()); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/de/zalando/zmon/scheduler/ng/trailruns/TrialRunRequest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.trailruns; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.Parameter; 4 | 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | /** 9 | * Created by jmussler on 4/27/15. 10 | */ 11 | public class TrialRunRequest { 12 | public Long interval; 13 | public String createdBy; 14 | public String name; 15 | public String id; 16 | public String checkCommand; 17 | public String alertCondition; 18 | public List> entities; 19 | public List> entitiesExclude; 20 | public Map parameters; 21 | public String period; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/resources/config/application.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8085 3 | 4 | endpoints: 5 | enabled: false 6 | metrics: 7 | enabled: true 8 | health: 9 | enabled: true 10 | sensitive: true 11 | 12 | spring: 13 | jackson: 14 | property-naming-strategy: CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES 15 | serialization: 16 | indent_output: true 17 | 18 | scheduler: 19 | redis_host: "localhost" 20 | redis_port: 6379 21 | instant_eval_forward: false 22 | trial_run_forward: false 23 | default_queue: "zmon:queue:default" 24 | last_run_persist: DISABLED 25 | check_detail_metrics: true 26 | oauth2_scopes: ["uid"] 27 | thread_count: 8 28 | check_filter: [] 29 | disable_persisted_schedule: true 30 | enable_global_entity: true 31 | task_writer_type: REDIS 32 | entity_service_url: http://localhost:8080 33 | controller_url: http://localhost:8080 34 | entity_properties_key: "zmon:entity:properties" 35 | 36 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/CeleryWriterTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.fasterxml.jackson.databind.node.ObjectNode; 5 | import io.opentracing.noop.NoopTracerFactory; 6 | import org.junit.Test; 7 | import org.xerial.snappy.Snappy; 8 | 9 | import java.io.IOException; 10 | 11 | import static org.junit.Assert.assertEquals; 12 | 13 | public class CeleryWriterTest { 14 | private final ObjectMapper mapper = new ObjectMapper(); 15 | 16 | @Test 17 | public void plainWriterProducesJSONWithBodyAndTrace() throws IOException { 18 | CeleryWriter writer = CeleryWriter.create(TaskSerializerType.PLAIN, NoopTracerFactory.create()); 19 | byte[] message = writer.asCeleryTask(new CeleryBody()); 20 | ObjectNode node = mapper.readValue(message, ObjectNode.class); 21 | 22 | assertEquals(true, node.has("body")); 23 | assertEquals(true, node.has("properties")); 24 | assertEquals(true, node.get("properties").has("trace")); 25 | } 26 | 27 | 28 | @Test 29 | public void plainWriterProducesJSONWithoutTraceIfNoTracerProvided() throws IOException { 30 | CeleryWriter writer = CeleryWriter.create(TaskSerializerType.PLAIN, null); 31 | byte[] message = writer.asCeleryTask(new CeleryBody()); 32 | ObjectNode node = mapper.readValue(message, ObjectNode.class); 33 | 34 | assertEquals(true, node.has("body")); 35 | assertEquals(true, node.has("properties")); 36 | assertEquals(false, node.get("properties").has("trace")); 37 | } 38 | 39 | @Test 40 | public void compressedWriterProducesJSONWithBodyAndTrace() throws IOException { 41 | CeleryWriter writer = CeleryWriter.create(TaskSerializerType.COMPRESSED, NoopTracerFactory.create()); 42 | byte[] encoded = writer.asCeleryTask(new CeleryBody()); 43 | byte[] message = Snappy.uncompress(encoded); 44 | 45 | ObjectNode node = mapper.readValue(message, ObjectNode.class); 46 | 47 | assertEquals(true, node.has("body")); 48 | assertEquals(true, node.has("properties")); 49 | assertEquals(true, node.get("properties").has("trace")); 50 | } 51 | 52 | @Test 53 | public void compressedWriterProducesJSONWithoutTraceIfNoTracerProvided() throws IOException { 54 | CeleryWriter writer = CeleryWriter.create(TaskSerializerType.COMPRESSED, null); 55 | byte[] encoded = writer.asCeleryTask(new CeleryBody()); 56 | byte[] message = Snappy.uncompress(encoded); 57 | 58 | ObjectNode node = mapper.readValue(message, ObjectNode.class); 59 | 60 | assertEquals(true, node.has("body")); 61 | assertEquals(true, node.has("properties")); 62 | assertEquals(false, node.get("properties").has("trace")); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/FilterTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 4 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 5 | import de.zalando.zmon.scheduler.ng.entities.Entity; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.mockito.runners.MockitoJUnitRunner; 11 | 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | import static java.util.Arrays.asList; 17 | import static org.mockito.Mockito.*; 18 | 19 | /** 20 | * Created by jmussler on 08.06.16. 21 | */ 22 | @RunWith(MockitoJUnitRunner.class) 23 | public class FilterTest { 24 | 25 | AlertDefinition emptyAlerty = when(mock(AlertDefinition.class).getId()).thenReturn(1).getMock(); 26 | 27 | CheckDefinition emptyCheck = when(mock(CheckDefinition.class).getId()).thenReturn(1).getMock(); 28 | 29 | private final static Map simpleEntityProperties = new HashMap() {{ 30 | put("id", "entity-1"); 31 | put("type", "host"); 32 | put("application", "zmon-scheduler"); 33 | }}; 34 | 35 | private final static Map typeHostFilter = new HashMap() {{ 36 | put("type", "host"); 37 | }}; 38 | 39 | private final static Map typeApplicationFilter = new HashMap() {{ 40 | put("type", "application"); 41 | }}; 42 | 43 | Entity simpleEntity = when(mock(Entity.class).getFilterProperties()).thenReturn(simpleEntityProperties).getMock(); 44 | 45 | CheckDefinition hostCheck = when(mock(CheckDefinition.class).getEntities()).thenReturn(asList(typeHostFilter)).getMock(); 46 | 47 | CheckDefinition applicationCheck = when(mock(CheckDefinition.class).getEntities()).thenReturn(asList(typeApplicationFilter)).getMock(); 48 | 49 | CheckDefinition bothCheck = when(mock(CheckDefinition.class).getEntities()).thenReturn(asList(typeApplicationFilter, typeHostFilter)).getMock(); 50 | 51 | AlertDefinition hostAlert = when(mock(AlertDefinition.class).getEntities()).thenReturn(asList(typeHostFilter)).getMock(); 52 | 53 | AlertDefinition applicationAlert = when(mock(AlertDefinition.class).getEntities()).thenReturn(asList(typeApplicationFilter)).getMock(); 54 | 55 | AlertDefinition bothAlert = when(mock(AlertDefinition.class).getEntities()).thenReturn(asList(typeApplicationFilter, typeHostFilter)).getMock(); 56 | 57 | @Test 58 | public void testEmptyAlertFilter() { 59 | boolean matchesEmptyAlert = AlertOverlapGenerator.matchAlertFilter(emptyAlerty, simpleEntity); 60 | assertEquals(true, matchesEmptyAlert); 61 | } 62 | 63 | @Test 64 | public void testEmptyCheck() { 65 | boolean matchesEmptyCheck = AlertOverlapGenerator.matchCheckFilter(emptyCheck, simpleEntity); 66 | assertEquals(false, matchesEmptyCheck); 67 | } 68 | 69 | @Test 70 | public void testCheckFilterMatch() { 71 | boolean matchHostCheck = AlertOverlapGenerator.matchCheckFilter(hostCheck, simpleEntity); 72 | assertEquals(true, matchHostCheck); 73 | 74 | boolean matchApplicationCheck = AlertOverlapGenerator.matchCheckFilter(applicationCheck, simpleEntity); 75 | assertEquals(false, matchApplicationCheck); 76 | 77 | boolean matchBothCheck = AlertOverlapGenerator.matchCheckFilter(bothCheck, simpleEntity); 78 | assertEquals(true, matchBothCheck); 79 | } 80 | 81 | @Test 82 | public void testAlertFilterMatch() { 83 | boolean matchHostAlert = AlertOverlapGenerator.matchAlertFilter(hostAlert, simpleEntity); 84 | assertEquals(true, matchHostAlert); 85 | 86 | boolean matchApplicationAlert = AlertOverlapGenerator.matchAlertFilter(applicationAlert, simpleEntity); 87 | assertEquals(false, matchApplicationAlert); 88 | 89 | boolean matchBothAlert = AlertOverlapGenerator.matchAlertFilter(bothAlert, simpleEntity); 90 | assertEquals(true, matchBothAlert); 91 | } 92 | 93 | @Test 94 | public void testAddProperties() { 95 | Map newProperties = new HashMap<>(); 96 | newProperties.put("type", "host"); 97 | newProperties.put("vendor", "zalando"); 98 | newProperties.put("generation", 1); 99 | 100 | 101 | Entity entity = new Entity("host-1"); 102 | entity.addProperties(newProperties); 103 | 104 | Map filter = new HashMap<>(); 105 | filter.put("type", "host"); 106 | List> filters = asList(filter); 107 | 108 | Map filter2 = new HashMap<>(); 109 | filter.put("generation", "1"); 110 | List> filters2 = asList(filter2); 111 | 112 | assertEquals(true, AlertOverlapGenerator.matchAnyFilter(filters, entity)); 113 | assertEquals(true, AlertOverlapGenerator.matchAnyFilter(filters2, entity)); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/SchedulerTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 4 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 5 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 6 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 7 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 8 | import de.zalando.zmon.scheduler.ng.entities.Entity; 9 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 10 | import de.zalando.zmon.scheduler.ng.queue.QueueSelector; 11 | import de.zalando.zmon.scheduler.ng.scheduler.Scheduler; 12 | import de.zalando.zmon.scheduler.ng.trailruns.TrialRunRequest; 13 | 14 | import com.codahale.metrics.MetricRegistry; 15 | import io.opentracing.noop.NoopTracerFactory; 16 | import org.junit.Test; 17 | import org.mockito.Mockito; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | import static java.util.Arrays.asList; 23 | import static org.mockito.AdditionalMatchers.gt; 24 | import static org.mockito.Matchers.any; 25 | import static org.mockito.Matchers.eq; 26 | import static org.mockito.Mockito.mock; 27 | import static org.mockito.Mockito.never; 28 | import static org.mockito.Mockito.timeout; 29 | import static org.mockito.Mockito.verify; 30 | import static org.mockito.Mockito.when; 31 | 32 | public class SchedulerTest { 33 | 34 | @Test 35 | public void scheduleOne() throws InterruptedException { 36 | final CheckRepository checkRepo = mock(CheckRepository.class); 37 | CheckDefinition check = new CheckDefinition(); 38 | check.setId(1); 39 | check.setInterval(1L); 40 | Map includeFilter = new HashMap<>(); 41 | includeFilter.put("type", "host"); 42 | check.setEntities(asList(includeFilter)); 43 | when(checkRepo.get(1)).thenReturn(check); 44 | 45 | Map excludeFilter = new HashMap<>(); 46 | excludeFilter.put("id", "entity-3"); 47 | 48 | final EntityRepository entityRepo = mock(EntityRepository.class); 49 | Entity entity1= new Entity("entity-1"); 50 | entity1.addProperty("type", "host"); 51 | 52 | Entity entity2 = new Entity("entity-2"); 53 | entity2.addProperty("type", "instance"); 54 | 55 | Entity entity3 = new Entity("entity-3"); 56 | entity3.addProperty("type", "host"); 57 | 58 | when(entityRepo.get()).thenReturn(asList(entity1, entity2, entity3)); 59 | 60 | final AlertRepository alertRepo = mock(AlertRepository.class); 61 | AlertDefinition alert = new AlertDefinition(); 62 | alert.setId(2); 63 | alert.setEntities(asList()); 64 | alert.setEntitiesExclude(asList(excludeFilter)); 65 | 66 | when(alertRepo.getByCheckId(check.getId())).thenReturn(asList(alert)); 67 | when(alertRepo.get(alert.getId())).thenReturn(alert); 68 | 69 | QueueSelector queueSelector = mock(QueueSelector.class); 70 | SchedulerConfig config = new SchedulerConfig(); 71 | MetricRegistry metricRegistry = new MetricRegistry(); 72 | Scheduler scheduler = new Scheduler(alertRepo, checkRepo, entityRepo, queueSelector, config, metricRegistry, NoopTracerFactory.create()); 73 | 74 | // now schedule our check 75 | long beforeSchedule = System.currentTimeMillis(); 76 | scheduler.scheduleCheck(check.getId()); 77 | 78 | // verify that our entity was writen to the "queue" 79 | verify(queueSelector, timeout(2000)).execute(eq(entity1), any(), any(), gt(beforeSchedule)); 80 | 81 | // non matching entity not executed 82 | verify(queueSelector, Mockito.after(2000).never()).execute(eq(entity2), any(), any(), gt(beforeSchedule)); 83 | 84 | // excluded entity not executed 85 | verify(queueSelector, Mockito.after(2000).never()).execute(eq(entity3), any(), any(), gt(beforeSchedule)); 86 | } 87 | 88 | @Test 89 | public void scheduleTrialRun() { 90 | final EntityRepository entityRepo = mock(EntityRepository.class); 91 | Entity entity1 = new Entity("included-entity"); 92 | entity1.addProperty("type", "host"); 93 | 94 | Entity entity2 = new Entity("included-entity"); 95 | entity2.addProperty("type", "instance"); 96 | 97 | Entity entityExcluded = new Entity("excluded-entity"); 98 | entityExcluded.addProperty("type", "host"); 99 | when(entityRepo.get()).thenReturn(asList(entity1, entity2, entityExcluded)); 100 | 101 | QueueSelector queueSelector = mock(QueueSelector.class); 102 | SchedulerConfig config = new SchedulerConfig(); 103 | MetricRegistry metricRegistry = new MetricRegistry(); 104 | Scheduler scheduler = new Scheduler(null, null, entityRepo, queueSelector, config, metricRegistry, NoopTracerFactory.create()); 105 | 106 | TrialRunRequest request = new TrialRunRequest(); 107 | request.id = "test"; 108 | request.interval = 60L; 109 | Map includeFilter = new HashMap<>(); 110 | includeFilter.put("type", "host"); 111 | request.entities = asList(includeFilter); 112 | 113 | Map excludeFilter = new HashMap<>(); 114 | excludeFilter.put("id", "excluded-entity"); 115 | request.entitiesExclude = asList(excludeFilter); 116 | 117 | scheduler.scheduleTrialRun(request); 118 | 119 | verify(queueSelector).execute(eq(entity1), any(), eq("zmon:queue:default")); 120 | verify(queueSelector, never()).execute(eq(entity2), any(), eq("zmon:queue:default")); 121 | verify(queueSelector, never()).execute(eq(entityExcluded), any(), eq("zmon:queue:default")); 122 | } 123 | 124 | } 125 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/alerts/AlertPropertiesChangedTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | import org.junit.BeforeClass; 4 | import org.junit.Test; 5 | 6 | import java.util.Arrays; 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | 12 | 13 | /** 14 | * Created by jmussler on 08.06.16. 15 | */ 16 | public class AlertPropertiesChangedTest { 17 | 18 | private final static Map hostFilterProperties = new HashMap() {{ 19 | put("id", "entity-1"); 20 | put("type", "host"); 21 | put("application", "zmon-scheduler"); 22 | }}; 23 | 24 | private final static Map applicationFilterProperties = new HashMap() {{ 25 | put("id", "entity-1"); 26 | put("type", "application"); 27 | put("application", "zmon-scheduler"); 28 | }}; 29 | 30 | private final static Map typeHostFilter = new HashMap() {{ 31 | put("type", "host"); 32 | }}; 33 | 34 | private final static Map typeHostFilterTwo = new HashMap() {{ 35 | put("type", "host"); 36 | }}; 37 | 38 | private static AlertDefinition emptyAlertFilter = new AlertDefinition(); 39 | private static AlertDefinition emptyAlertFilter2 = new AlertDefinition(); 40 | private static AlertDefinition oneFilter = new AlertDefinition(); 41 | private static AlertDefinition twoFilter = new AlertDefinition(); 42 | private static AlertDefinition twoFilterOther = new AlertDefinition(); 43 | private static AlertDefinition nullFilter = new AlertDefinition(); 44 | private static AlertDefinition hostFilter = new AlertDefinition(); 45 | private static AlertDefinition applicationFilter = new AlertDefinition(); 46 | 47 | private static AlertDefinition excludeFilter = new AlertDefinition(); 48 | private static AlertDefinition excludeFilterChanged = new AlertDefinition(); 49 | private static AlertDefinition nullExcludeFilter = new AlertDefinition(); 50 | 51 | 52 | @BeforeClass 53 | public static void setupMocks() { 54 | oneFilter.setEntities(Arrays.asList(typeHostFilter)); 55 | twoFilter.setEntities(Arrays.asList(typeHostFilter, typeHostFilter)); 56 | twoFilterOther.setEntities(Arrays.asList(typeHostFilterTwo, typeHostFilterTwo)); 57 | nullFilter.setEntities(null); 58 | 59 | hostFilter.setEntities(Arrays.asList(hostFilterProperties)); 60 | applicationFilter.setEntities(Arrays.asList(applicationFilterProperties)); 61 | 62 | excludeFilter.setEntitiesExclude(Arrays.asList(hostFilterProperties)); 63 | excludeFilterChanged.setEntitiesExclude(Arrays.asList(applicationFilterProperties)); 64 | nullExcludeFilter.setEntitiesExclude(null); 65 | } 66 | 67 | @Test 68 | public void testEntitiesFilterEmpty() { 69 | assertEquals(false, emptyAlertFilter.compareForAlertUpdate(emptyAlertFilter2)); 70 | } 71 | 72 | @Test 73 | public void testTwoNonEmpty() { 74 | assertEquals(true, oneFilter.compareForAlertUpdate(twoFilter)); 75 | assertEquals(true, twoFilter.compareForAlertUpdate(oneFilter)); 76 | assertEquals(false, twoFilter.compareForAlertUpdate(twoFilter)); 77 | } 78 | 79 | @Test 80 | public void testNullTrue() { 81 | assertEquals(true, nullFilter.compareForAlertUpdate(twoFilter)); 82 | assertEquals(true, twoFilter.compareForAlertUpdate(nullFilter)); 83 | } 84 | 85 | @Test 86 | public void testFilterPropertiesChange() { 87 | assertEquals(true, hostFilter.compareForAlertUpdate(applicationFilter)); 88 | } 89 | 90 | @Test 91 | public void testFilterRemoveChange() { 92 | assertEquals(true, twoFilter.compareForAlertUpdate(oneFilter)); 93 | } 94 | 95 | @Test 96 | public void testNoChange() { 97 | assertEquals(false, twoFilterOther.compareForAlertUpdate(twoFilter)); 98 | } 99 | 100 | @Test 101 | public void testExcludeFilters() { 102 | assertEquals(false, excludeFilter.compareForAlertUpdate(excludeFilter)); 103 | assertEquals(true, excludeFilter.compareForAlertUpdate(excludeFilterChanged)); 104 | assertEquals(true, excludeFilter.compareForAlertUpdate(nullExcludeFilter)); 105 | assertEquals(true, nullExcludeFilter.compareForAlertUpdate(excludeFilter)); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/alerts/AlertRepositoryTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.alerts; 2 | 3 | /** 4 | * Created by jmussler on 02.07.16. 5 | */ 6 | public class AlertRepositoryTest { 7 | } 8 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/checks/CheckDefinitionTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | import com.fasterxml.jackson.databind.DeserializationFeature; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; 6 | import org.junit.Test; 7 | 8 | import java.io.IOException; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | import static org.junit.Assert.assertTrue; 12 | 13 | public class CheckDefinitionTest { 14 | private static final ObjectMapper mapper = new ObjectMapper(); 15 | 16 | @Test 17 | public void testJsonParsing() throws IOException { 18 | String json = "{\n" + 19 | " \"id\": 123,\n" + 20 | " \"name\": \"SomeName\",\n" + 21 | " \"description\": \"Track errors in logs\",\n" + 22 | " \"technical_details\": null,\n" + 23 | " \"potential_analysis\": null,\n" + 24 | " \"potential_impact\": null,\n" + 25 | " \"potential_solution\": null,\n" + 26 | " \"owning_team\": \"someteam\",\n" + 27 | " \"entities\": [\n" + 28 | " {\n" + 29 | " \"cluster_id\": \"aws:260111690941:eu-central-1:kube-1\",\n" + 30 | " \"lifecycle_status\": \"ready\",\n" + 31 | " \"type\": \"kubernetes_cluster\"\n" + 32 | " }\n" + 33 | " ],\n" + 34 | " \"interval\": 120,\n" + 35 | " \"command\": \"check command\",\n" + 36 | " \"status\": \"ACTIVE\",\n" + 37 | " \"source_url\": null,\n" + 38 | " \"last_modified_by\": \"someone\",\n" + 39 | " \"last_modified\": 1556539968226,\n" + 40 | " \"criticality\": \"important\",\n" + 41 | " \"unknown_field\": \"some_value\",\n" + 42 | " \"deleted\": false\n" + 43 | "}"; 44 | 45 | CheckDefinition check = mapper.readValue(json, CheckDefinition.class); 46 | assertEquals("SomeName", check.getName()); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/checks/CheckRepositoryTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.checks; 2 | 3 | /** 4 | * Created by jmussler on 02.07.16. 5 | */ 6 | public class CheckRepositoryTest { 7 | } 8 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/cleanup/EntityPropertyChangeTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.cleanup; 2 | 3 | import de.zalando.zmon.scheduler.ng.alerts.AlertDefinition; 4 | import de.zalando.zmon.scheduler.ng.alerts.AlertRepository; 5 | import de.zalando.zmon.scheduler.ng.checks.CheckDefinition; 6 | import de.zalando.zmon.scheduler.ng.checks.CheckRepository; 7 | import de.zalando.zmon.scheduler.ng.entities.Entity; 8 | import de.zalando.zmon.scheduler.ng.entities.EntityRepository; 9 | import org.junit.BeforeClass; 10 | import org.junit.Test; 11 | import org.mockito.Mockito; 12 | 13 | import java.util.HashMap; 14 | import java.util.Map; 15 | 16 | import static java.util.Arrays.asList; 17 | import static org.mockito.Matchers.anyInt; 18 | import static org.mockito.Mockito.*; 19 | 20 | /** 21 | * Created by jmussler on 08.06.16. 22 | */ 23 | public class EntityPropertyChangeTest { 24 | 25 | private final static Map simpleEntityProperties = new HashMap() {{ 26 | put("id", "entity-1"); 27 | put("type", "host"); 28 | put("application", "zmon-scheduler"); 29 | }}; 30 | 31 | private final static Map simpleEntityPropertiesChanged = new HashMap() {{ 32 | put("id", "entity-1"); 33 | put("type", "application"); 34 | put("application", "zmon-scheduler"); 35 | }}; 36 | 37 | private final static Map typeHostFilter = new HashMap() {{ 38 | put("type", "host"); 39 | }}; 40 | 41 | static CheckDefinition hostCheck = when(mock(CheckDefinition.class).getEntities()).thenReturn(asList(typeHostFilter)).getMock(); 42 | static AlertDefinition hostAlert = when(mock(AlertDefinition.class).getEntities()).thenReturn(asList(typeHostFilter)).getMock(); 43 | 44 | static AlertRepository alertRepo = mock(AlertRepository.class); 45 | static CheckRepository checkRepo = mock(CheckRepository.class); 46 | static EntityRepository entityRepo = mock(EntityRepository.class); 47 | 48 | static AlertChangedCleaner alertCleaner = mock(AlertChangedCleaner.class); 49 | static EntityChangedCleaner cleaner = new EntityChangedCleaner(alertRepo, checkRepo, alertCleaner); 50 | 51 | static Entity simpleEntity = when(mock(Entity.class).getFilterProperties()).thenReturn(simpleEntityProperties).getMock(); 52 | static Entity simpleEntityChanged = when(mock(Entity.class).getFilterProperties()).thenReturn(simpleEntityPropertiesChanged).getMock(); 53 | 54 | @BeforeClass 55 | public static void setupMocks() { 56 | when(hostCheck.getId()).thenReturn(1); 57 | when(hostAlert.getId()).thenReturn(1); 58 | when(hostAlert.getCheckDefinitionId()).thenReturn(1); 59 | 60 | when(checkRepo.get()).thenReturn(asList(hostCheck)); 61 | when(alertRepo.getByCheckId(anyInt())).thenReturn(asList(hostAlert)); 62 | } 63 | 64 | @Test 65 | public void testTrigger() { 66 | reset(alertCleaner); 67 | cleaner.notifyEntityChangeNoWait(entityRepo, simpleEntity, simpleEntityChanged); 68 | verify(alertCleaner, timeout(1000).times(1)).notifyAlertChange(hostAlert); 69 | } 70 | 71 | @Test 72 | public void testNoTrigger() { 73 | reset(alertCleaner); 74 | cleaner.notifyEntityChangeNoWait(entityRepo, simpleEntity, simpleEntity); 75 | verify(alertCleaner, Mockito.after(1000).atMost(0)).notifyAlertChange(hostAlert); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/test/java/de/zalando/zmon/scheduler/ng/entities/EntityRepositoryTest.java: -------------------------------------------------------------------------------- 1 | package de.zalando.zmon.scheduler.ng.entities; 2 | 3 | import de.zalando.zmon.scheduler.ng.config.SchedulerConfig; 4 | 5 | import io.opentracing.noop.NoopTracerFactory; 6 | import org.junit.Test; 7 | import org.mockito.Mockito; 8 | 9 | import java.util.List; 10 | 11 | import static java.util.Arrays.asList; 12 | import static org.junit.Assert.*; 13 | import static org.mockito.Mockito.*; 14 | 15 | /** 16 | * Created by jmussler on 02.07.16. 17 | */ 18 | public class EntityRepositoryTest { 19 | 20 | @Test 21 | public void BaseFilterTest() { 22 | SchedulerConfig config = new SchedulerConfig(); 23 | config.setEntityBaseFilterStr("[{\"type\":\"host\"}]"); 24 | 25 | EntityAdapterRegistry registry = Mockito.mock(EntityAdapterRegistry.class); 26 | 27 | Entity instance = new Entity("instance-1"); 28 | instance.addProperty("type", "instance"); 29 | 30 | Entity host = new Entity("host-1"); 31 | host.addProperty("type", "host"); 32 | 33 | List entities = asList(instance, host); 34 | 35 | EntityAdapter adapter = Mockito.mock(EntityAdapter.class); 36 | when(adapter.getCollection()).thenReturn(entities); 37 | 38 | when(registry.getSourceNames()).thenReturn(asList("entities")); 39 | when(registry.get("entities")).thenReturn(adapter); 40 | 41 | EntityRepository repository = new EntityRepository(registry, config, NoopTracerFactory.create()); 42 | 43 | assertEquals(1, repository.getCurrentMap().size()); 44 | } 45 | 46 | @Test 47 | public void TestNoChangeOnException() { 48 | SchedulerConfig config = new SchedulerConfig(); 49 | config.setEntityBaseFilterStr("[{\"type\":\"host\"}]"); 50 | 51 | EntityAdapterRegistry registry = Mockito.mock(EntityAdapterRegistry.class); 52 | 53 | Entity instance = new Entity("instance-1"); 54 | instance.addProperty("type", "instance"); 55 | 56 | Entity host = new Entity("host-1"); 57 | host.addProperty("type", "host"); 58 | 59 | List entities = asList(instance, host); 60 | 61 | EntityAdapter adapter = Mockito.mock(EntityAdapter.class); 62 | when(adapter.getCollection()).thenReturn(entities).thenThrow(new RuntimeException("Loading of entities failed")); 63 | 64 | when(registry.getSourceNames()).thenReturn(asList("entities")); 65 | when(registry.get("entities")).thenReturn(adapter); 66 | 67 | EntityRepository repository = new EntityRepository(registry, config, NoopTracerFactory.create()); 68 | assertEquals(1, repository.getCurrentMap().size()); 69 | 70 | try { 71 | repository.fill(); 72 | fail("Exception not thrown"); 73 | } catch (Exception t) { 74 | assertTrue("Exception of unexpected type", t instanceof RuntimeException); 75 | } 76 | 77 | assertEquals(1, repository.getCurrentMap().size()); 78 | } 79 | 80 | @Test 81 | public void TestNotify() { 82 | SchedulerConfig config = new SchedulerConfig(); 83 | config.setEntityBaseFilterStr("[{\"type\":\"host\"}]"); 84 | 85 | EntityAdapterRegistry registry = Mockito.mock(EntityAdapterRegistry.class); 86 | 87 | Entity instance = new Entity("instance-1"); 88 | instance.addProperty("type", "instance"); 89 | 90 | Entity host1 = new Entity("host-1"); 91 | host1.addProperty("type", "host"); 92 | 93 | Entity host1_changed = new Entity("host-1"); 94 | host1_changed.addProperty("type", "host"); 95 | host1_changed.addProperty("traffic", "true"); 96 | 97 | Entity host2 = new Entity("host-2"); 98 | host2.addProperty("type", "host"); 99 | 100 | Entity host3 = new Entity("host-3"); 101 | host3.addProperty("type", "host"); 102 | 103 | List entities = asList(instance, host1, host3); 104 | List entities2 = asList(instance, host1_changed, host2); 105 | 106 | EntityAdapter adapter = Mockito.mock(EntityAdapter.class); 107 | when(adapter.getCollection()).thenReturn(entities).thenReturn(entities2); 108 | 109 | when(registry.getSourceNames()).thenReturn(asList("entities")); 110 | when(registry.get("entities")).thenReturn(adapter); 111 | 112 | EntityChangeListener listener = Mockito.mock(EntityChangeListener.class); 113 | 114 | EntityRepository repository = new EntityRepository(registry, config, NoopTracerFactory.create()); 115 | repository.registerListener(listener); 116 | assertEquals(2, repository.getCurrentMap().size()); 117 | 118 | repository.fill(); 119 | assertEquals(2, repository.getCurrentMap().size()); 120 | 121 | verify(listener).notifyEntityAdd(eq(repository), eq(host2)); 122 | verify(listener).notifyEntityRemove(eq(repository), eq(host3)); 123 | verify(listener).notifyEntityChange(eq(repository), eq(host1), eq(host1_changed)); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | exec java $JAVA_OPTS $(java-dynamic-memory-opts) -jar zmon-scheduler.jar 4 | --------------------------------------------------------------------------------