getMetricSnapshot(Group metricGroup) {
111 |
112 | MetricManager mm = registryRepository.getRegistry(metricGroup);
113 | if (mm != null) {
114 | return Optional.of(mm.getPrometheusRegistry().scrape());
115 | }
116 | return Optional.empty();
117 | }
118 |
119 | public boolean resetMetricGroup(Group group) {
120 | try {
121 | MetricManager metricManager = this.registryRepository.getRegistry(group);
122 | metricManager.resetRegistries();
123 | } catch (Exception e) {
124 | return false;
125 | }
126 | return true;
127 | }
128 |
129 | public boolean resetConfigs() throws InterruptedException {
130 | ReentrantReadWriteLock.WriteLock writeLock = reentrantReadWriteLock.writeLock();
131 | if (writeLock.tryLock(20, TimeUnit.SECONDS)) {
132 | try {
133 | metricsAccumulatorConfiguration.resetConfiguration();
134 | registryRepository.reset();
135 | scheduledTasks.reset();
136 | return true;
137 | } catch (Throwable e) {
138 | log.error("Failed to reset. Please reset configurations");
139 | return false;
140 | } finally {
141 | writeLock.unlock();
142 | }
143 | } else {
144 | return false;
145 | }
146 | }
147 | }
148 |
--------------------------------------------------------------------------------
/src/main/java/io/bpoole6/accumulator/service/MetricValue.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator.service;
2 |
3 | import lombok.Data;
4 |
5 | @Data
6 | public class MetricValue {
7 | private Double value;
8 | private long timestamp;
9 | public MetricValue(Double value, long timestamp) {
10 | this.value = value;
11 | this.timestamp = timestamp;
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/src/main/java/io/bpoole6/accumulator/service/MetricsAccumulatorConfiguration.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator.service;
2 |
3 | import io.bpoole6.accumulator.service.metricgroup.Global;
4 | import io.bpoole6.accumulator.service.metricgroup.Group;
5 | import io.bpoole6.accumulator.service.metricgroup.Root;
6 | import java.util.function.Function;
7 | import lombok.Data;
8 | import lombok.extern.slf4j.Slf4j;
9 | import org.springframework.boot.ApplicationArguments;
10 | import org.springframework.stereotype.Component;
11 | import org.springframework.util.ResourceUtils;
12 | import org.yaml.snakeyaml.Yaml;
13 |
14 | import java.io.File;
15 | import java.io.IOException;
16 | import java.nio.file.Files;
17 | import java.nio.file.Paths;
18 | import java.util.List;
19 | import java.util.Map;
20 | import java.util.Objects;
21 | import java.util.StringJoiner;
22 | import java.util.stream.Collectors;
23 |
24 | /**
25 | * Reason for using a configuration file over an Enum.
26 | *
27 | * 1.) Because we are using an apikey to control access to updating the metrics we need a seperation
28 | * between dev/prod. An enum class wouldn't be able to easily differentiate between the two profiles
29 | * like you can do with configurations files
30 | */
31 | @Component
32 | @Data
33 | @Slf4j
34 | public class MetricsAccumulatorConfiguration {
35 | public static final String CONFIGURATION_OPTS = "config-file";
36 | private String hostAddress;
37 | private Global global;
38 | private Map metricGroups;
39 | private Map metricGroupByApiKey;
40 | private String configurationFile;
41 | private String fileContent;
42 | public MetricsAccumulatorConfiguration(ApplicationArguments arguments)
43 | throws IOException {
44 | if(!arguments.containsOption(CONFIGURATION_OPTS)){
45 | log.error("--%s option is not set".formatted(CONFIGURATION_OPTS));
46 | throw new RuntimeException();
47 | }
48 | configurationFile = Paths.get(arguments.getOptionValues(CONFIGURATION_OPTS).get(0)).toAbsolutePath().toString();
49 | resetConfiguration();
50 | }
51 |
52 | public void resetConfiguration() throws IOException {
53 | File file = ResourceUtils.getFile(configurationFile);
54 | Yaml yaml = new Yaml();
55 | String content = Files.readString(file.toPath());
56 | Root root = yaml.loadAs(content, Root.class);
57 | validateConfigs(root);
58 | this.metricGroups = root.getMetricGroups().values().stream().collect(Collectors.toMap(i->i.getName(),
59 | Function.identity()));
60 | this.hostAddress = root.getGlobal().getHostAddress();
61 | this.metricGroupByApiKey = metricGroups.values().stream().collect(Collectors.toMap(Group::getApiKey, g -> g));
62 | this.global = root.getGlobal();
63 | this.fileContent = content;
64 | }
65 |
66 | public List getNames() {
67 | return metricGroups.values().stream().filter(Group::isDisplayMetrics).map(
68 | Group::getName).toList();
69 | }
70 |
71 | private void validateConfigs(Root root){
72 | StringJoiner sj = new StringJoiner(System.lineSeparator());
73 | if(Objects.isNull(root)){
74 | sj.add("Configuration file parsed but no configs present");
75 | }
76 | if(Objects.isNull(root.getGlobal())){
77 | sj.add("global attribute not set");
78 | }else {
79 | if(Objects.isNull(root.getGlobal().getRestartCronExpression())){
80 | sj.add("global.restartCronExpression not set");
81 | }
82 | if(Objects.isNull(root.getGlobal().getHostAddress())){
83 | sj.add("global.hostAddress not set");
84 | }
85 | }
86 |
87 | if(Objects.isNull(root.getMetricGroups())){
88 | sj.add("metricGroups is not set");
89 | } else if (root.getMetricGroups().isEmpty()){
90 | sj.add("metricGroups is empty");
91 | }else{
92 | root.getMetricGroups().forEach((k,v)->{
93 | if(Objects.isNull(v.getName())){
94 | sj.add("name is unset for metricGroup %s".formatted(k));
95 | }else if (!v.getName().matches("^[-a-zA-Z0-9_]+")){
96 | sj.add("name doesn't match regex ^[-a-zA-Z0-9_]+");
97 | }
98 | if(Objects.isNull(v.getApiKey())){
99 | sj.add("apiKey is unset for metricGroup %s".formatted(k));
100 | }
101 | if(Objects.isNull(v.getMaxTimeSeries())){
102 | sj.add("maxTimeSeries is unset for metricGroup %s".formatted(k));
103 | }
104 | });
105 | }
106 | String errors = sj.toString();
107 | if(!errors.isBlank()){
108 | log.error(errors);
109 | throw new RuntimeException("Failed metric group configurations");
110 | }
111 |
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/src/main/java/io/bpoole6/accumulator/service/RegistryRepository.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator.service;
2 |
3 | import io.bpoole6.accumulator.service.metricgroup.Group;
4 |
5 | import java.util.HashMap;
6 | import java.util.Map;
7 | import lombok.Getter;
8 | import lombok.extern.slf4j.Slf4j;
9 | import org.springframework.stereotype.Component;
10 |
11 | @Component
12 | @Slf4j
13 | public class RegistryRepository {
14 |
15 | public static final String META_DATA_PREFIX = "_metrics_accumulator_";
16 | public static final String META_DATA_LATEST = META_DATA_PREFIX + "latest";
17 |
18 | @Getter
19 | private final Map registryMap = new HashMap<>();
20 | private final MetricsAccumulatorConfiguration metricsAccumulatorConfiguration;
21 |
22 | public RegistryRepository(MetricsAccumulatorConfiguration metricsAccumulatorConfiguration) {
23 | this.metricsAccumulatorConfiguration = metricsAccumulatorConfiguration;
24 | reset();
25 | }
26 |
27 | public void reset(){
28 | registryMap.clear();
29 | for (Group metricGroup : metricsAccumulatorConfiguration.getMetricGroups().values()) {
30 | registryMap.put(metricGroup, new MetricManager(metricGroup.getName()));
31 | }
32 | }
33 | public MetricManager getRegistry(Group metricGroup) {
34 | return this.registryMap.get(metricGroup);
35 | }
36 | public void incrementCounter(MetricKey metricKey, prometheus.types.Counter metric) {
37 | this.registryMap.get(metricKey.getMetricGroup()).incrementCounter(metricKey, metric);
38 | }
39 | public void setGauge(MetricKey metricKey, prometheus.types.Gauge metric) {
40 | this.registryMap.get(metricKey.getMetricGroup()).setGauge(metricKey, metric);
41 | }
42 |
43 |
44 |
45 | public static Map stripMetaData(Map labels) {
46 | Map map = new HashMap<>();
47 | labels.forEach((k, v) -> {
48 | if (!k.startsWith(META_DATA_PREFIX)) {
49 | map.put(k, v);
50 | }
51 | });
52 | return map;
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/main/java/io/bpoole6/accumulator/service/metricgroup/Global.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator.service.metricgroup;
2 |
3 | import lombok.Data;
4 |
5 | @Data
6 | public class Global {
7 | private String restartCronExpression;
8 | private String hostAddress;
9 | }
10 |
--------------------------------------------------------------------------------
/src/main/java/io/bpoole6/accumulator/service/metricgroup/Group.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator.service.metricgroup;
2 |
3 | import lombok.Data;
4 |
5 | import java.util.HashMap;
6 | import java.util.Map;
7 |
8 | @Data
9 | public class Group {
10 | private boolean displayMetrics;
11 | private String name;
12 | private Integer maxTimeSeries;
13 | private String apiKey;
14 | private String restartCronExpression;
15 | private final Map serviceDiscoveryLabels = new HashMap<>();
16 |
17 | public void setServiceDiscoveryLabels(Map serviceDiscoveryLabels){
18 | this.serviceDiscoveryLabels.clear();
19 | this.serviceDiscoveryLabels.putAll(serviceDiscoveryLabels);
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/java/io/bpoole6/accumulator/service/metricgroup/Root.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator.service.metricgroup;
2 |
3 | import java.util.Map;
4 | import lombok.Data;
5 |
6 | @Data
7 | public class Root {
8 | private Global global;
9 | private Map metricGroups;
10 |
11 | }
12 |
--------------------------------------------------------------------------------
/src/main/java/io/bpoole6/accumulator/util/RunnableThrowable.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator.util;
2 |
3 | public interface RunnableThrowable {
4 |
5 | void run() throws Throwable;
6 | }
7 |
--------------------------------------------------------------------------------
/src/main/java/io/bpoole6/accumulator/util/Utils.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator.util;
2 |
3 | import io.bpoole6.accumulator.service.metricgroup.Group;
4 | import io.micrometer.core.instrument.Counter;
5 | import io.micrometer.core.instrument.composite.CompositeMeterRegistry;
6 | import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
7 | import io.micrometer.prometheusmetrics.PrometheusMeterRegistry;
8 | import java.io.ByteArrayInputStream;
9 | import java.io.IOException;
10 | import java.util.ArrayList;
11 | import java.util.List;
12 |
13 | import io.prometheus.metrics.model.registry.PrometheusRegistry;
14 | import prometheus.text.TextPrometheusMetricDataParser;
15 | import prometheus.types.MetricFamily;
16 | import prometheus.types.MetricType;
17 |
18 | public class Utils {
19 | public static List readMetrics(String metrics) throws IOException {
20 |
21 | TextPrometheusMetricDataParser i = new TextPrometheusMetricDataParser(
22 | new ByteArrayInputStream(metrics.getBytes()));
23 | List metricFamilies = new ArrayList<>();
24 | MetricFamily family;
25 | while ((family = i.parse()) != null) {
26 | if((family.getType() == MetricType.COUNTER || family.getType() == MetricType.GAUGE) && !family.getName().endsWith("_info")) {
27 | metricFamilies.add(family);
28 | }
29 | }
30 | return metricFamilies;
31 | }
32 | public static boolean exceedsMaxTimeSeriesLimit(Group group, PrometheusMeterRegistry registry){
33 | synchronized (registry){
34 | return registry.getPrometheusRegistry().scrape().size() >= group.getMaxTimeSeries();
35 | }
36 | }
37 |
38 | public static void main(String[] args) {
39 | PrometheusRegistry r = new PrometheusRegistry();
40 | io.prometheus.metrics.core.metrics.Counter c = io.prometheus.metrics.core.metrics.Counter.builder().name("test").labelNames("tag").register(r);
41 | io.prometheus.metrics.core.metrics.Counter d = io.prometheus.metrics.core.metrics.Counter.builder().name("test").labelNames("b").register(r);
42 | c.labelValues("a").inc();
43 | d.labelValues("b").inc();
44 | System.out.println();
45 | CompositeMeterRegistry s = new CompositeMeterRegistry();
46 | Counter.builder("test")
47 | .tag("tag", "tag").register(s).increment(3);
48 | Counter.builder("test")
49 | .tag("bag", "bag").register(s).increment(34);
50 | System.out.println();
51 |
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/src/main/resources/application.yaml:
--------------------------------------------------------------------------------
1 | spring:
2 | application:
3 | name: metrics-accumulator
4 |
5 | management:
6 | endpoint:
7 | prometheus:
8 | enabled: true
9 | endpoints:
10 | web:
11 | exposure:
12 | include: prometheus
13 | metrics:
14 | tags:
15 | application: ${spring.application.name}
16 | maxBatchedMetrics: 200000
17 | sleepTimeBtwMetrics: 200 # milliseconds
--------------------------------------------------------------------------------
/src/main/resources/banner.txt:
--------------------------------------------------------------------------------
1 |
2 | . . . .
3 | ,8. ,8. 8 8888888888 8888888 8888888888 8 888888888o. 8 8888 ,o888888o. d888888o. .8. ,o888888o. ,o888888o. 8 8888 88 ,8. ,8. 8 8888 88 8 8888 .8. 8888888 8888888888 ,o888888o. 8 888888888o.
4 | ,888. ,888. 8 8888 8 8888 8 8888 `88. 8 8888 8888 `88. .`8888:' `88. .888. 8888 `88. 8888 `88. 8 8888 88 ,888. ,888. 8 8888 88 8 8888 .888. 8 8888 . 8888 `88. 8 8888 `88.
5 | .`8888. .`8888. 8 8888 8 8888 8 8888 `88 8 8888 ,8 8888 `8. 8.`8888. Y8 :88888. ,8 8888 `8. ,8 8888 `8. 8 8888 88 .`8888. .`8888. 8 8888 88 8 8888 :88888. 8 8888 ,8 8888 `8b 8 8888 `88
6 | ,8.`8888. ,8.`8888. 8 8888 8 8888 8 8888 ,88 8 8888 88 8888 `8.`8888. . `88888. 88 8888 88 8888 8 8888 88 ,8.`8888. ,8.`8888. 8 8888 88 8 8888 . `88888. 8 8888 88 8888 `8b 8 8888 ,88
7 | ,8'8.`8888,8^8.`8888. 8 888888888888 8 8888 8 8888. ,88' 8 8888 88 8888 `8.`8888. .8. `88888. 88 8888 88 8888 8 8888 88 ,8'8.`8888,8^8.`8888. 8 8888 88 8 8888 .8. `88888. 8 8888 88 8888 88 8 8888. ,88'
8 | ,8' `8.`8888' `8.`8888. 8 8888 8 8888 8 888888888P' 8 8888 88 8888 `8.`8888. .8`8. `88888. 88 8888 88 8888 8 8888 88 ,8' `8.`8888' `8.`8888. 8 8888 88 8 8888 .8`8. `88888. 8 8888 88 8888 88 8 888888888P'
9 | ,8' `8.`88' `8.`8888. 8 8888 8 8888 8 8888`8b 8 8888 88 8888 `8.`8888. .8' `8. `88888.88 8888 88 8888 8 8888 88 ,8' `8.`88' `8.`8888. 8 8888 88 8 8888 .8' `8. `88888. 8 8888 88 8888 ,8P 8 8888`8b
10 | ,8' `8.`' `8.`8888. 8 8888 8 8888 8 8888 `8b. 8 8888 `8 8888 .8' 8b `8.`8888. .8' `8. `88888`8 8888 .8' `8 8888 .8' ` 8888 ,8P ,8' `8.`' `8.`8888.` 8888 ,8P 8 8888 .8' `8. `88888. 8 8888 `8 8888 ,8P 8 8888 `8b.
11 | ,8' `8 `8.`8888. 8 8888 8 8888 8 8888 `8b. 8 8888 8888 ,88' `8b. ;8.`8888 .888888888. `88888. 8888 ,88' 8888 ,88' 8888 ,d8P ,8' `8 `8.`8888. 8888 ,d8P 8 8888 .888888888. `88888. 8 8888 ` 8888 ,88' 8 8888 `8b.
12 | ,8' ` `8.`8888. 8 888888888888 8 8888 8 8888 `88. 8 8888 `8888888P' `Y8888P ,88P' .8' `8. `88888. `8888888P' `8888888P' `Y88888P' ,8' ` `8.`8888. `Y88888P' 8 888888888888 .8' `8. `88888. 8 8888 `8888888P' 8 8888 `88.
13 |
--------------------------------------------------------------------------------
/src/main/resources/static/blah.txt:
--------------------------------------------------------------------------------
1 | # TYPE mega_total counter
2 | # HELP mega_total
3 | mega_total {span_id="321",trace_id="124",a="df"} 5.0
4 | mega_total 5.0
--------------------------------------------------------------------------------
/src/main/resources/templates/configuration.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Hello
5 |
6 |
15 |
16 |
17 |
18 | Configurations
19 |
20 | Configuration file:
21 |
22 | Configuration loaded
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
--------------------------------------------------------------------------------
/src/test/java/io/bpoole6/accumulator/BasicTest.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator;
2 |
3 | import io.bpoole6.accumulator.service.MetricsAccumulatorConfiguration;
4 | import org.mockito.Mockito;
5 | import org.springframework.boot.ApplicationArguments;
6 | import org.springframework.boot.test.context.TestConfiguration;
7 | import org.springframework.context.annotation.Bean;
8 | import org.springframework.test.context.ContextConfiguration;
9 |
10 | import java.io.IOException;
11 | import java.nio.file.Path;
12 | import java.util.ArrayList;
13 | import java.util.List;
14 |
15 | @ContextConfiguration(classes = BasicTest.TestApplicationArguments.class)
16 | public class BasicTest {
17 |
18 | @TestConfiguration
19 | public static class TestApplicationArguments{
20 | @Bean("springApplicationArguments")
21 | public ApplicationArguments getApplicationArguments() throws IOException {
22 | ApplicationArguments arguments = Mockito.mock(ApplicationArguments.class);
23 | Path path = TestUtils.createMetricsFile();
24 | List configFiles = new ArrayList<>();
25 | configFiles.add(path.toAbsolutePath().toString());
26 | Mockito.when(arguments.containsOption(MetricsAccumulatorConfiguration.CONFIGURATION_OPTS)).thenReturn(true);
27 | Mockito.when(arguments.getOptionValues(MetricsAccumulatorConfiguration.CONFIGURATION_OPTS)).thenReturn(configFiles);
28 |
29 | return arguments;
30 | }
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/src/test/java/io/bpoole6/accumulator/MetricsConsumerApplicationTests.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator;
2 |
3 | import io.bpoole6.accumulator.service.MetricService;
4 | import io.bpoole6.accumulator.service.RegistryRepository;
5 | import io.bpoole6.accumulator.service.metricgroup.Group;
6 | import io.bpoole6.accumulator.service.MetricsAccumulatorConfiguration;
7 | import io.bpoole6.accumulator.util.Utils;
8 | import io.micrometer.prometheusmetrics.PrometheusMeterRegistry;
9 | import java.io.IOException;
10 | import java.nio.charset.Charset;
11 | import java.util.List;
12 | import java.util.Optional;
13 | import org.junit.jupiter.api.Assertions;
14 | import org.junit.jupiter.api.BeforeEach;
15 | import org.junit.jupiter.api.Test;
16 | import org.springframework.beans.factory.annotation.Autowired;
17 | import org.springframework.beans.factory.annotation.Value;
18 | import org.springframework.boot.test.context.SpringBootTest;
19 | import org.springframework.core.io.Resource;
20 | import org.springframework.test.annotation.DirtiesContext;
21 | import prometheus.types.Counter;
22 | import prometheus.types.Gauge;
23 | import prometheus.types.Metric;
24 | import prometheus.types.MetricFamily;
25 | import prometheus.types.MetricType;
26 |
27 | @SpringBootTest
28 | @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) //In the future we may want to just reset the registries and maps between tests. Reloading the context may become expensive
29 | class MetricsConsumerApplicationTests extends BasicTest{
30 |
31 |
32 | @Value("classpath:data/default/metrics")
33 | private Resource metrics1;
34 |
35 | @Value("classpath:data/label_timestamp/metrics-future")
36 | private Resource metricsFuture;
37 |
38 | @Value("classpath:data/label_timestamp/metrics-old")
39 | private Resource metricsOld;
40 |
41 | @Autowired
42 | private RegistryRepository registryRepository;
43 |
44 | @Autowired
45 | private MetricService metricService;
46 |
47 | private Group defaultGroup;
48 | @Autowired
49 | private MetricsAccumulatorConfiguration metricsAccumulatorConfiguration;
50 |
51 | public MetricsConsumerApplicationTests() {
52 |
53 | }
54 |
55 | @BeforeEach
56 | void setup() {
57 | this.defaultGroup = registryRepository.getRegistryMap().keySet().stream().findFirst().get();
58 | }
59 |
60 | @Test
61 | void testCounterAccumulator() throws IOException, InterruptedException {
62 | List originalMetrics = Utils.readMetrics(
63 | new String(metrics1.getContentAsByteArray()));
64 | metricService.modifyMetrics(new String(metrics1.getContentAsByteArray()), defaultGroup);
65 | metricService.modifyMetrics(new String(metrics1.getContentAsByteArray()), defaultGroup);
66 | Thread.sleep(metricService.sleepTimeBtwMetrics+500);
67 | PrometheusMeterRegistry prometheusRegistry =this.registryRepository.getRegistry(defaultGroup).getPrometheusRegistry();
68 | String snapshot = prometheusRegistry.scrape();
69 | List newMetrics =Utils.readMetrics(snapshot);
70 |
71 | evalMetricFamilies(originalMetrics, newMetrics);
72 |
73 | }
74 |
75 | @Test
76 | void testGaugeTimestamp() throws IOException, InterruptedException {
77 | String olderMetrics = metricsOld.getContentAsString(Charset.defaultCharset());
78 | String newerMetrics = metricsFuture.getContentAsString(Charset.defaultCharset());
79 |
80 | this.metricService.modifyMetrics(olderMetrics, defaultGroup);
81 | this.metricService.modifyMetrics(newerMetrics, defaultGroup);
82 | Thread.sleep(metricService.sleepTimeBtwMetrics+500);
83 | Gauge oldGauge = (Gauge) Utils.readMetrics(olderMetrics).get(0).getMetrics().get(0);
84 | Gauge newGauge = (Gauge) Utils.readMetrics(this.registryRepository.getRegistry(defaultGroup).getPrometheusRegistry().scrape()).get(0).getMetrics().get(0);
85 | Assertions.assertNotSame(oldGauge.getValue(), newGauge.getValue());
86 | Assertions.assertEquals(4911.0, newGauge.getValue());
87 | }
88 |
89 | @Test
90 | void testMaxMetrics() throws IOException, InterruptedException {
91 | String formatted = """
92 | # HELP python%s_gc_objects_collected_seconds Objects collected during gc
93 | # TYPE python%s_gc_objects_collected_seconds gauge
94 | python%s_gc_objects_collected_seconds{generation="0",_metrics_accumulator_latest="3992623250"} 4911.0
95 | """;
96 | String strings = "";
97 | for(int i = 0; i < 150; i++) {
98 | strings += formatted.formatted(i,i,i);
99 | }
100 | this.metricService.modifyMetrics(strings, defaultGroup);
101 | Thread.sleep(metricService.sleepTimeBtwMetrics+500);
102 | Assertions.assertEquals(defaultGroup.getMaxTimeSeries(), this.registryRepository.getRegistry(defaultGroup).getPrometheusRegistry().getPrometheusRegistry().scrape().size());
103 | }
104 |
105 |
106 |
107 | public void evalMetricFamilies(List originalMetrics, List newMetrics) {
108 | for (MetricFamily metricFamily : originalMetrics) {
109 | Optional opt = newMetrics.stream()
110 | .filter(i -> i.getName().equals(metricFamily.getName())).findFirst();
111 | if(opt.isEmpty()){
112 | Assertions.fail("Missing MetricFamily "+ metricFamily.getName());
113 | }
114 | MetricFamily foundMetricFamily = opt.get();
115 | Assertions.assertEquals(metricFamily.getType(), foundMetricFamily.getType());
116 | for(Metric metric : metricFamily.getMetrics()){
117 | Metric foundMetric = foundMetricFamily.getMetrics().stream().filter(i-> i.getName().equals(metric.getName()) && i.getLabels().equals(metric.getLabels())).findFirst().get();
118 | Assertions.assertEquals(metric.getLabels().keySet(), foundMetric.getLabels().keySet());
119 | if(metricFamily.getType() == MetricType.COUNTER) {
120 | Counter c = ((prometheus.types.Counter)metric);
121 | Counter foundC = ((prometheus.types.Counter)foundMetric);
122 | Assertions.assertEquals(c.getValue()*2, foundC.getValue() );
123 | }else if(metricFamily.getType() == MetricType.GAUGE){
124 | Gauge g = ((prometheus.types.Gauge)metric);
125 | Gauge foundG = ((prometheus.types.Gauge)foundMetric);
126 | Assertions.assertEquals(g.getValue(), foundG.getValue() );
127 | }
128 |
129 | }
130 | }
131 |
132 | }
133 | }
134 |
--------------------------------------------------------------------------------
/src/test/java/io/bpoole6/accumulator/ResetConfigurationTest.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator;
2 |
3 | import io.bpoole6.accumulator.service.MetricsAccumulatorConfiguration;
4 | import io.bpoole6.accumulator.service.MetricManager;
5 | import io.bpoole6.accumulator.service.MetricService;
6 | import io.bpoole6.accumulator.service.RegistryRepository;
7 | import io.bpoole6.accumulator.service.metricgroup.Group;
8 | import org.junit.jupiter.api.Assertions;
9 | import org.junit.jupiter.api.Test;
10 | import org.springframework.beans.factory.annotation.Autowired;
11 | import org.springframework.boot.test.context.SpringBootTest;
12 | import org.springframework.test.annotation.DirtiesContext;
13 |
14 | import java.io.IOException;
15 | import java.nio.file.Files;
16 | import java.nio.file.Path;
17 | import java.nio.file.Paths;
18 | import java.util.List;
19 | import java.util.Optional;
20 |
21 | @SpringBootTest
22 | @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) //In the future we may want to just reset the registries and maps between tests
23 | public class ResetConfigurationTest extends BasicTest{
24 |
25 | @Autowired
26 | private RegistryRepository registryRepository;
27 |
28 | @Autowired
29 | private MetricService metricService;
30 |
31 | @Autowired
32 | private ScheduledTasks scheduledTasks;
33 |
34 | @Autowired
35 | private MetricsAccumulatorConfiguration metricsAccumulatorConfiguration;
36 |
37 | @Test
38 | void testMetricReset() throws IOException, InterruptedException {
39 |
40 | String oldName = "oldName";
41 | String newName = "newName";
42 | Path path = Paths.get(this.metricsAccumulatorConfiguration.getConfigurationFile());
43 | Files.writeString(path, TestUtils.metricsFile(oldName));
44 | this.metricService.resetConfigs();
45 |
46 | Optional oldGroup = this.registryRepository.getRegistryMap().keySet().stream().findFirst();
47 | Assertions.assertTrue(oldGroup.isPresent(), "Assert that registrymap has the current");
48 | Assertions.assertEquals(oldName, oldGroup.get().getName(), "Assert that Group name is %s".formatted(oldName));
49 |
50 | Files.writeString(path, TestUtils.metricsFile(newName));
51 | this.metricService.resetConfigs();
52 |
53 | Optional newGroup = this.registryRepository.getRegistryMap().keySet().stream().findFirst();
54 | Assertions.assertTrue(newGroup.isPresent(), "Assert that registrymap has the current");
55 | Assertions.assertEquals(newName, newGroup.get().getName(), "Assert that Group name is %s".formatted(newName));
56 | }
57 |
58 |
59 | @Test
60 | public void resetMetricManager() throws InterruptedException, IOException {
61 | String metrics = """
62 | # HELP python_gc_objects_collected_seconds Objects collected during gc
63 | # TYPE python_gc_objects_collected_seconds gauge
64 | python_gc_objects_collected_seconds{generation="0",_metrics_accumulator_latest="3992623250"} 4911.0
65 | """;
66 | Group group = this.registryRepository.getRegistryMap().keySet().stream().findFirst().get();
67 | this.metricService.modifyMetrics(metrics, group);
68 | Thread.sleep(metricService.sleepTimeBtwMetrics+500);
69 | MetricManager oldManager = this.registryRepository.getRegistryMap().values().stream().findFirst().get();
70 | Assertions.assertEquals(1, oldManager.getPrometheusRegistry().getPrometheusRegistry().scrape().size());
71 | oldManager.resetRegistries();
72 | Assertions.assertEquals(0, oldManager.getPrometheusRegistry().getPrometheusRegistry().scrape().size());
73 |
74 | }
75 | @Test
76 | public void testSchedulingReload() throws InterruptedException {
77 | List oldTasks = this.scheduledTasks.getTasks();
78 | this.metricService.resetConfigs();
79 | List newTasks = this.scheduledTasks.getTasks();
80 |
81 | oldTasks.forEach(t->{
82 | Assertions.assertTrue(t.getFuture().isCancelled(), "Task is still running when it should be stopped");
83 | });
84 |
85 | newTasks.forEach(t->{
86 | Assertions.assertFalse(t.getFuture().isCancelled(), "Task is still running when it should be stopped");
87 | });
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/src/test/java/io/bpoole6/accumulator/TestControllers.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator;
2 |
3 | import io.bpoole6.accumulator.controller.MetricsController;
4 | import io.bpoole6.accumulator.controller.response.ServiceDiscovery;
5 | import org.junit.jupiter.api.Assertions;
6 | import org.junit.jupiter.api.Test;
7 | import org.springframework.beans.factory.annotation.Autowired;
8 | import org.springframework.boot.test.context.SpringBootTest;
9 | import org.springframework.http.ResponseEntity;
10 | import org.springframework.test.annotation.DirtiesContext;
11 | import org.springframework.web.client.RestTemplate;
12 |
13 | import java.net.URI;
14 | import java.net.http.HttpRequest;
15 | import java.util.List;
16 |
17 | @SpringBootTest
18 | @DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
19 | public class TestControllers extends BasicTest {
20 |
21 | @Autowired
22 | private MetricsController controller;
23 |
24 | @Test
25 | public void testServiceDiscovery(){
26 | ResponseEntity> entity = controller.serviceDiscovery();
27 | List sds = entity.getBody();
28 | Assertions.assertTrue(sds.size() == 1);
29 | Assertions.assertEquals(3, sds.get(0).getLabels().size());
30 | Assertions.assertEquals("qa", sds.get(0).getLabels().get("env"));
31 | Assertions.assertEquals("v45", sds.get(0).getLabels().get("version"));
32 | }
33 |
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/src/test/java/io/bpoole6/accumulator/TestUtils.java:
--------------------------------------------------------------------------------
1 | package io.bpoole6.accumulator;
2 |
3 | import java.io.IOException;
4 | import java.nio.file.Files;
5 | import java.nio.file.Path;
6 |
7 | public class TestUtils {
8 | public static String DEFAULT_METRIC_NAME = "test-Metrics";
9 | public static Path createMetricsFile() throws IOException {
10 | Path path = Files.createTempFile("config",".yml");
11 | Files.writeString(path,metricsFile(DEFAULT_METRIC_NAME));
12 | return path;
13 | }
14 | public static String metricsFile(String name){
15 | return """
16 | global:
17 | restartCronExpression: "0 0 0 ? * *"
18 | hostAddress: localhost:8080
19 |
20 | metricGroups:
21 | %s:
22 | displayMetrics: true
23 | name: %s
24 | maxTimeSeries: 100
25 | apiKey: 0d98f65f-074b-4d56-b834-576e15a3bfa5
26 | restartCronExpression: "0 0 0 ? * *"
27 | serviceDiscoveryLabels:
28 | env: qa
29 | version: v45
30 | """.formatted(name,name);
31 | }
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/src/test/resources/application.yaml:
--------------------------------------------------------------------------------
1 | spring:
2 | application:
3 | name: MetricsConsumer
4 | server:
5 | port: 8081
6 |
7 | metricGroup:
8 | file: "classpath:metric-groups.yml"
9 | host:
10 | address: "localhost:8081"
11 |
12 | maxBatchedMetrics: 100000
13 | sleepTimeBtwMetrics: 200 # milliseconds
--------------------------------------------------------------------------------
/src/test/resources/data/created_total/metrics:
--------------------------------------------------------------------------------
1 | # HELP proxy_watch_total Total number of connection made
2 | # TYPE proxy_watch_total counter
3 | proxy_watch_total{application="proxy_watch",host="proxy.internal.q6cyber.com",ip="8003"} 1211.0
4 | # HELP proxy_watch_created Total number of connection made
5 | # TYPE proxy_watch_created gauge
6 | #proxy_watch_created{application="proxy_watch",host="proxy.internal.q6cyber.com",ip="8003"} 1.7225315143089755e+09
7 |
8 |
9 |
--------------------------------------------------------------------------------
/src/test/resources/data/default/metrics:
--------------------------------------------------------------------------------
1 | # HELP python_gc_objects_collected_total Objects collected during gc
2 | # TYPE python_gc_objects_collected_total counter
3 | python_gc_objects_collected_total{generation="0"} 4911.0
4 | python_gc_objects_collected_total{generation="1"} 324.0
5 | python_gc_objects_collected_total{generation="2"} 0.0
6 | # HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC
7 | # TYPE python_gc_objects_uncollectable_total counter
8 | python_gc_objects_uncollectable_total{generation="0"} 0.0
9 | python_gc_objects_uncollectable_total{generation="1"} 0.0
10 | python_gc_objects_uncollectable_total{generation="2"} 0.0
11 | # HELP python_gc_collections_total Number of times this generation was collected
12 | # TYPE python_gc_collections_total counter
13 | python_gc_collections_total{generation="0"} 97.0
14 | python_gc_collections_total{generation="1"} 8.0
15 | python_gc_collections_total{generation="2"} 0.0
16 | # HELP python_info Python platform information
17 | # TYPE python_info gauge
18 | python_info{implementation="CPython",major="3",minor="10",patchlevel="12",version="3.10.12"} 1.0
19 | # HELP process_virtual_memory_bytes Virtual memory size in bytes.
20 | # TYPE process_virtual_memory_bytes gauge
21 | process_virtual_memory_bytes 3.43359488e+08
22 | # HELP process_resident_memory_bytes Resident memory size in bytes.
23 | # TYPE process_resident_memory_bytes gauge
24 | process_resident_memory_bytes 3.6716544e+07
25 | # HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
26 | # TYPE process_start_time_seconds gauge
27 | process_start_time_seconds 1.72253151306e+09
28 | # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
29 | # TYPE process_cpu_seconds_total counter
30 | process_cpu_seconds_total 4.34
31 | # HELP process_open_fds Number of open file descriptors.
32 | # TYPE process_open_fds gauge
33 | process_open_fds 8.0
34 | # HELP process_max_fds Maximum number of open file descriptors.
35 | # TYPE process_max_fds gauge
36 | process_max_fds 1.048576e+06
37 | # HELP proxy_watch_total Total number of connection made
38 | # TYPE proxy_watch_total counter
39 | proxy_watch_total{application="proxy_watch",host="proxy.internal.q6cyber.com",ip="8003"} 1211.0
40 |
--------------------------------------------------------------------------------
/src/test/resources/data/label_timestamp/metrics-future:
--------------------------------------------------------------------------------
1 | # HELP python_gc_objects_collected_seconds Objects collected during gc
2 | # TYPE python_gc_objects_collected_seconds gauge
3 | python_gc_objects_collected_seconds{generation="0",_metrics_accumulator_latest="3992623250"} 4911.0
4 |
--------------------------------------------------------------------------------
/src/test/resources/data/label_timestamp/metrics-old:
--------------------------------------------------------------------------------
1 | # HELP python_gc_objects_collected_seconds Objects collected during gc
2 | # TYPE python_gc_objects_collected_seconds gauge
3 | python_gc_objects_collected_seconds{generation="0",_metrics_accumulator_latest="1722620529"} 5.0
4 |
--------------------------------------------------------------------------------
/src/test/resources/metric-groups.yml:
--------------------------------------------------------------------------------
1 | ---
2 | defaults:
3 | restartCronExpression: "0 0 0 ? * *"
4 | metricGroups:
5 | default:
6 | displayMetrics: false
7 | name: default
8 | maxTimeSeries: 2500
9 |
--------------------------------------------------------------------------------
