├── .github ├── CODEOWNERS └── workflows │ └── coverage.yml ├── codecov.yml ├── .gitignore ├── component ├── publisher-client │ └── src │ │ ├── test │ │ ├── resources │ │ │ ├── log4j2.xml │ │ │ └── testng.xml │ │ └── java │ │ │ └── org │ │ │ └── wso2 │ │ │ └── am │ │ │ └── analytics │ │ │ └── publisher │ │ │ ├── util │ │ │ ├── AuthAPIMockService.java │ │ │ ├── UnitTestAppender.java │ │ │ └── TestUtils.java │ │ │ ├── AuthAPIClientTestCase.java │ │ │ ├── LogMetricReporterTestCase.java │ │ │ ├── ErrorHandlingTestCase.java │ │ │ ├── MetricReporterTestCase.java │ │ │ └── DefaultFaultMetricBuilderTestCase.java │ │ └── main │ │ └── java │ │ └── org │ │ └── wso2 │ │ └── am │ │ └── analytics │ │ └── publisher │ │ ├── reporter │ │ ├── TimerMetric.java │ │ ├── MetricType.java │ │ ├── MetricSchema.java │ │ ├── cloud │ │ │ ├── DefaultChoreoFaultMetricEventBuilder.java │ │ │ ├── DefaultChoreoResponseMetricEventBuilder.java │ │ │ ├── QueueFlusher.java │ │ │ ├── DefaultAnalyticsThreadFactory.java │ │ │ ├── ParallelQueueWorker.java │ │ │ ├── QueueWorker.java │ │ │ ├── DefaultCounterMetric.java │ │ │ ├── EventQueue.java │ │ │ └── DefaultFaultMetricEventBuilder.java │ │ ├── CounterMetric.java │ │ ├── moesif │ │ │ ├── util │ │ │ │ ├── MoesifKeyEntry.java │ │ │ │ └── MoesifMicroserviceConstants.java │ │ │ ├── MissedEventHandler.java │ │ │ ├── MoesifCounterMetric.java │ │ │ ├── MoesifMetricEventBuilder.java │ │ │ ├── EventQueue.java │ │ │ ├── MoesifReporter.java │ │ │ └── ParallelQueueWorker.java │ │ ├── Metric.java │ │ ├── AbstractMetricEventBuilder.java │ │ ├── log │ │ │ ├── LogMetricEventBuilder.java │ │ │ ├── LogMetricReporter.java │ │ │ └── LogCounterMetric.java │ │ ├── MetricReporter.java │ │ ├── elk │ │ │ ├── ELKMetricReporter.java │ │ │ ├── ELKCounterMetric.java │ │ │ └── ELKMetricEventBuilder.java │ │ ├── MetricEventBuilder.java │ │ ├── AbstractMetricReporter.java │ │ └── MetricReporterFactory.java │ │ ├── client │ │ ├── ClientStatus.java │ │ ├── MoesifClientContextHolder.java │ │ ├── DataHolder.java │ │ ├── AbstractMoesifClient.java │ │ ├── WSO2TokenCredential.java │ │ └── EventHubProducerClientFactory.java │ │ ├── util │ │ ├── LogSanitizer.java │ │ ├── EventMapAttributeFilter.java │ │ ├── BackoffRetryCounter.java │ │ ├── UserAgentParser.java │ │ └── HttpStatusHelper.java │ │ ├── exception │ │ ├── HttpClientException.java │ │ ├── AuthenticationException.java │ │ ├── MetricReportingException.java │ │ ├── MetricCreationException.java │ │ ├── ConnectionRecoverableException.java │ │ ├── APICallException.java │ │ └── ConnectionUnrecoverableException.java │ │ ├── auth │ │ ├── DefaultApi.java │ │ ├── TokenDetailsDTO.java │ │ ├── AuthClient.java │ │ └── AuthProxyUtils.java │ │ └── properties │ │ ├── AIMetadata.java │ │ ├── Properties.java │ │ └── AITokenUsage.java └── pom.xml ├── issue_template.md ├── README.md ├── features ├── pom.xml └── publisher-client │ └── pom.xml ├── suppressions.xml ├── pull_request_template.md └── findbugs-exclude.xml /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in the repo. 2 | * @AnuGayan @nimsara66 3 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | comment: 2 | layout: "reach, diff, flags, files" 3 | behavior: default 4 | require_changes: false 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | 25 | #idea files 26 | .idea 27 | *.iml 28 | .vscode 29 | 30 | .DS_Store 31 | .settings 32 | target -------------------------------------------------------------------------------- /component/publisher-client/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Test Coverage Uploader 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | workflow_dispatch: 11 | branches: 12 | - main 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout main 19 | uses: actions/checkout@v2 20 | with: 21 | fetch-depth: 2 22 | - name: Set up JDK 11 23 | uses: actions/setup-java@v2 24 | with: 25 | java-version: 11 26 | distribution: 'adopt' 27 | - name: Build with Tests 28 | run: mvn clean install --file pom.xml 29 | - name: Upload unit test coverage to Codecov 30 | uses: codecov/codecov-action@v1.2.1 31 | with: 32 | flags: unit_tests 33 | -------------------------------------------------------------------------------- /issue_template.md: -------------------------------------------------------------------------------- 1 | **Description:** 2 | 3 | 4 | **Suggested Labels:** 5 | 6 | 7 | **Suggested Assignees:** 8 | 9 | 10 | **Affected Product Version:** 11 | 12 | **OS, DB, other environment details and versions:** 13 | 14 | **Steps to reproduce:** 15 | 16 | 17 | **Related Issues:** 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # apim-analytics-publisher 2 | This repo contains the data publisher for Choreo Analytics cloud. Data publisher is responsible for sending analytics 3 | data into Azure cloud for processing and visualization. 4 | 5 | ## About this repository 6 | 7 | | Branch | Build Status(Jenkins) | 8 | | :------------ |:------------- 9 | | master | [![Build Status](https://wso2.org/jenkins/job/analytics-products/job/apim-analytics-publisher/badge/icon)](https://wso2.org/jenkins/job/analytics-products/job/apim-analytics-publisher) | 10 | 11 | 12 | ## Install 13 | ### Install from source 14 | 15 | - Clone this repository 16 | - Build the source using ```mvn clean install``` 17 | - You can find the executable at ```/component/data-publisher/target/org.wso2.am.analytics.publisher 18 | .client-x.x.x-jar-with-dependencies.jar``` 19 | ### Download from releases 20 | 21 | - Navigate to release section of this repository and download the desired release 22 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/TimerMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | /** 22 | * Interface for Timer Metric. 23 | */ 24 | public interface TimerMetric extends Metric { 25 | } 26 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/MetricType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | /** 22 | * Enum to represent metric types supported by the reporting framework. 23 | */ 24 | public enum MetricType { 25 | COUNTER, 26 | TIMER 27 | } 28 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/client/ClientStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.client; 20 | 21 | /** 22 | * Enum to represent the status of Eventhub publisher. 23 | */ 24 | public enum ClientStatus { 25 | CONNECTED, 26 | NOT_CONNECTED, 27 | FLUSHING_FAILED, 28 | RETRYING 29 | } 30 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/util/LogSanitizer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.util; 19 | 20 | /** 21 | * Utility class for sanitizing log messages by removing carriage return and newline characters. 22 | */ 23 | public class LogSanitizer { 24 | public static String sanitize(String input) { 25 | return input != null ? input.replaceAll("[\r\n]", "") : null; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/exception/HttpClientException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | 20 | package org.wso2.am.analytics.publisher.exception; 21 | 22 | /** 23 | * Exception class for HTTP client related exceptions. 24 | */ 25 | public class HttpClientException extends Exception { 26 | public HttpClientException(String message, Throwable cause) { 27 | super(message, cause); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/exception/AuthenticationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | 20 | package org.wso2.am.analytics.publisher.exception; 21 | 22 | /** 23 | * Exception class for authentication related exceptions. 24 | */ 25 | public class AuthenticationException extends Exception { 26 | public AuthenticationException(String message, Throwable cause) { 27 | super(message, cause); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/auth/DefaultApi.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.auth; 20 | 21 | import feign.RequestLine; 22 | 23 | /** 24 | * Analytics Auth API. 25 | * 26 | *

This is a RESTFul API for authenication information to communcate with event hub. 27 | * 28 | */ 29 | public interface DefaultApi { 30 | @RequestLine("GET /token") 31 | TokenDetailsDTO tokenGet(); 32 | } 33 | 34 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/MetricSchema.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | /** 22 | * Enum to represent supported metric schema types by the reporting framework. 23 | */ 24 | public enum MetricSchema { 25 | RESPONSE, 26 | ERROR, 27 | CHOREO_RESPONSE, 28 | CHOREO_ERROR, 29 | ELK_RESPONSE, 30 | ELK_ERROR, 31 | LATENCY, 32 | PAYLOAD, 33 | MOESIF_RESPONSE, 34 | MOESIF_ERROR, 35 | } 36 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/exception/MetricReportingException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.exception; 20 | 21 | /** 22 | * This exception will be thrown if any error happens when reporting a metric. 23 | */ 24 | public class MetricReportingException extends Exception { 25 | public MetricReportingException(String msg) { 26 | super(msg); 27 | } 28 | 29 | public MetricReportingException(String msg, Throwable e) { 30 | super(msg, e); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/exception/MetricCreationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.exception; 20 | 21 | /** 22 | * This exception will be thrown if any error happens during Metric Reporter Creation. 23 | */ 24 | public class MetricCreationException extends Exception { 25 | public MetricCreationException(String msg) { 26 | super(msg); 27 | } 28 | 29 | public MetricCreationException(String msg, Throwable e) { 30 | super(msg, e); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/auth/TokenDetailsDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.auth; 20 | 21 | import com.google.gson.annotations.SerializedName; 22 | 23 | /** 24 | * TokenDetailsDTO of the Auth API response. 25 | */ 26 | public class TokenDetailsDTO { 27 | 28 | @SerializedName("token") 29 | private String token; 30 | 31 | public String getToken() { 32 | return token; 33 | } 34 | 35 | public void setToken(String token) { 36 | this.token = token; 37 | } 38 | 39 | } 40 | 41 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/exception/ConnectionRecoverableException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.exception; 20 | 21 | /** 22 | * Exception to represent any recoverable errors event publishing client encounter. 23 | */ 24 | public class ConnectionRecoverableException extends Exception { 25 | public ConnectionRecoverableException(String msg) { 26 | super(msg); 27 | } 28 | 29 | public ConnectionRecoverableException(String msg, Throwable e) { 30 | super(msg, e); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/exception/APICallException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.exception; 20 | 21 | /** 22 | * Exception class for API calls related exceptions. 23 | * Could be used in scenarios where the issue is not strictly due to HTTP Client. 24 | */ 25 | public class APICallException extends Exception { 26 | public APICallException(String msg) { 27 | super(msg); 28 | } 29 | 30 | public APICallException(String msg, Throwable e) { 31 | super(msg, e); 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/exception/ConnectionUnrecoverableException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.exception; 20 | 21 | /** 22 | * Exception to represent any unrecoverable errors event publishing client encounter. 23 | */ 24 | public class ConnectionUnrecoverableException extends Exception { 25 | public ConnectionUnrecoverableException(String msg) { 26 | super(msg); 27 | } 28 | 29 | public ConnectionUnrecoverableException(String msg, Throwable e) { 30 | super(msg, e); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/DefaultChoreoFaultMetricEventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.cloud; 20 | 21 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 22 | 23 | /** 24 | * Builder class for fault events. 25 | */ 26 | public class DefaultChoreoFaultMetricEventBuilder extends DefaultFaultMetricEventBuilder { 27 | 28 | public DefaultChoreoFaultMetricEventBuilder() { 29 | super(DefaultInputValidator.getInstance().getEventProperties(MetricSchema.CHOREO_ERROR)); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /component/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | org.wso2.am.analytics.publisher 23 | apim-analytics-publisher-parent 24 | 1.2.34-SNAPSHOT 25 | ../pom.xml 26 | 27 | 28 | apim-analytics-publisher-component 29 | 4.0.0 30 | pom 31 | API Analytics Publisher Component 32 | 33 | 34 | publisher-client 35 | 36 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/DefaultChoreoResponseMetricEventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.cloud; 20 | 21 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 22 | 23 | /** 24 | * Default builder for response metric type. Restrictions are set on the key names that uses can set to the builder. 25 | * Allows keys and their validity will be checked when populating and availability of all required properties will be 26 | * checked when building. 27 | */ 28 | public class DefaultChoreoResponseMetricEventBuilder extends DefaultResponseMetricEventBuilder { 29 | 30 | public DefaultChoreoResponseMetricEventBuilder() { 31 | super(DefaultInputValidator.getInstance().getEventProperties(MetricSchema.CHOREO_RESPONSE)); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/CounterMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 22 | 23 | /** 24 | * Interface for Counter Metric. 25 | */ 26 | public interface CounterMetric extends Metric { 27 | /** 28 | * method to increment the count of the metric. Associated properties should be passed along with the method 29 | * invocation. 30 | * 31 | * @param builder {@link MetricEventBuilder} of the this Metric 32 | * @return current counter value 33 | * @throws MetricReportingException Exception will be thrown if expected properties are not present 34 | */ 35 | public int incrementCount(MetricEventBuilder builder) throws MetricReportingException; 36 | } 37 | -------------------------------------------------------------------------------- /features/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | org.wso2.am.analytics.publisher 23 | apim-analytics-publisher-parent 24 | 1.2.34-SNAPSHOT 25 | ../pom.xml 26 | 27 | 28 | 4.0.0 29 | 30 | API Analytics Publisher Feature Aggregate 31 | apim-analytics-publisher-features 32 | pom 33 | 34 | 35 | publisher-client 36 | 37 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/client/MoesifClientContextHolder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.client; 19 | 20 | import org.wso2.am.analytics.publisher.reporter.moesif.util.MoesifMicroserviceConstants; 21 | 22 | /** 23 | * Holds context of the Moesif client retry mechanism. 24 | */ 25 | public class MoesifClientContextHolder { 26 | public static final ThreadLocal PUBLISH_ATTEMPTS = new ThreadLocal() { 27 | @Override 28 | protected Integer initialValue() { 29 | return Integer.valueOf(MoesifMicroserviceConstants.NUM_RETRY_ATTEMPTS_PUBLISH); 30 | } 31 | 32 | @Override 33 | public Integer get() { 34 | return super.get(); 35 | } 36 | 37 | @Override 38 | public void set(Integer value) { 39 | super.set(value); 40 | } 41 | }; 42 | } 43 | 44 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/moesif/util/MoesifKeyEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.moesif.util; 19 | 20 | import com.google.gson.annotations.SerializedName; 21 | 22 | /** 23 | * POJO to parse the JSON. 24 | */ 25 | public class MoesifKeyEntry { 26 | private String uuid; 27 | @SerializedName("organization_id") 28 | private String organizationID; 29 | @SerializedName("moesif_key") 30 | private String moesifKey; 31 | private String env; 32 | 33 | public MoesifKeyEntry() { 34 | } 35 | 36 | public String getMoesif_key() { 37 | return moesifKey; 38 | } 39 | 40 | public String getOrganization_id() { 41 | return organizationID; 42 | } 43 | 44 | public String getUuid() { 45 | return uuid; 46 | } 47 | 48 | public String getEnv() { 49 | return env; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/util/EventMapAttributeFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | package org.wso2.am.analytics.publisher.util; 19 | 20 | 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | import java.util.Set; 24 | 25 | /** 26 | * Utility to filter only the required attributes. 27 | */ 28 | public class EventMapAttributeFilter { 29 | private static final EventMapAttributeFilter INSTANCE = new EventMapAttributeFilter(); 30 | 31 | public static EventMapAttributeFilter getInstance() { 32 | return INSTANCE; 33 | } 34 | 35 | public Map filter(Map source, Map requiredAttributes) { 36 | 37 | Set targetKeys = requiredAttributes.keySet(); 38 | Map filteredEventMap = new HashMap<>(); 39 | 40 | for (String key : targetKeys) { 41 | filteredEventMap.put(key, source.get(key)); 42 | } 43 | 44 | return filteredEventMap; 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/Metric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | /** 22 | * Base class for {@link CounterMetric} and {@link TimerMetric}. 23 | */ 24 | public interface Metric { 25 | /** 26 | * Returns name of the metric. Name is unique per given Reporter 27 | * 28 | * @return Name of the metric 29 | */ 30 | public String getName(); 31 | 32 | /** 33 | * Method to get schema name. 34 | * 35 | * @return Schema name of this Metric 36 | */ 37 | public MetricSchema getSchema(); 38 | 39 | /** 40 | * Returns the Event Builder which can produce an event conforming to the schema of the {@link Metric}. Users 41 | * should get a builder instance though here, populate the relevant fields and return back to Metric class. 42 | * @return MetricEventBuilder conforming to schema of Metric 43 | */ 44 | public MetricEventBuilder getEventBuilder(); 45 | } 46 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/AbstractMetricEventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 22 | 23 | import java.util.Map; 24 | 25 | /** 26 | * Abstract implementation of {@link MetricEventBuilder}. Ensures that subclasses perform validations when adding 27 | * elements. 28 | */ 29 | public abstract class AbstractMetricEventBuilder implements MetricEventBuilder { 30 | @Override 31 | public Map build() throws MetricReportingException { 32 | if (validate()) { 33 | return buildEvent(); 34 | } 35 | throw new MetricReportingException("Validation failure occurred when building the event"); 36 | } 37 | 38 | /** 39 | * Process the added data and return as a flat {@link Map}. 40 | * 41 | * @return Map representing attributes of Metric Event 42 | */ 43 | protected abstract Map buildEvent(); 44 | } 45 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/QueueFlusher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.cloud; 20 | 21 | import org.wso2.am.analytics.publisher.client.EventHubClient; 22 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 23 | 24 | import java.util.concurrent.BlockingQueue; 25 | 26 | /** 27 | * Class to periodically flush event batch. 28 | */ 29 | public class QueueFlusher implements Runnable { 30 | 31 | private final BlockingQueue queue; 32 | private final EventHubClient client; 33 | 34 | public QueueFlusher(BlockingQueue queue, EventHubClient client) { 35 | this.queue = queue; 36 | this.client = client; 37 | } 38 | 39 | @Override public void run() { 40 | if (queue.isEmpty()) { 41 | //For scenarios where no API invocation is happening additional check in the EventHubClient will stop 42 | // empty batch flushing 43 | client.flushEvents(); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/util/BackoffRetryCounter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.util; 20 | 21 | /** 22 | * Util class to implement backoff retrying. 23 | */ 24 | public class BackoffRetryCounter { 25 | private final String[] timeIntervalNames = new String[]{"5 sec", "10 sec", "15 sec", "30 sec", "1 min", "1 min", 26 | "2 min", "5 min"}; 27 | private final long[] timeIntervals = new long[]{5000, 10000, 15000, 30000, 60000, 60000, 120000, 300000}; 28 | 29 | private int intervalIndex = 0; 30 | 31 | public synchronized void reset() { 32 | intervalIndex = 0; 33 | } 34 | 35 | public synchronized void increment() { 36 | if (intervalIndex < timeIntervals.length - 1) { 37 | intervalIndex++; 38 | } 39 | } 40 | 41 | public long getTimeIntervalMillis() { 42 | return timeIntervals[intervalIndex]; 43 | } 44 | 45 | 46 | public String getTimeInterval() { 47 | return timeIntervalNames[intervalIndex]; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/moesif/MissedEventHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.moesif; 19 | 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | import org.wso2.am.analytics.publisher.retriever.MoesifKeyRetriever; 23 | 24 | import java.util.TimerTask; 25 | 26 | /** 27 | * Responsible for periodically calling the Moesif microservice and 28 | * refreshing the internal map. 29 | */ 30 | public class MissedEventHandler extends TimerTask { 31 | private static final Logger log = LogManager.getLogger(MissedEventHandler.class); 32 | private final MoesifKeyRetriever keyRetriever; 33 | 34 | public MissedEventHandler(MoesifKeyRetriever keyRetriever) { 35 | this.keyRetriever = keyRetriever; 36 | } 37 | 38 | @Override 39 | public void run() { 40 | // clear the internal map of orgID-MoesifKey 41 | keyRetriever.clearMoesifKeyMap(); 42 | // clear the environment map 43 | keyRetriever.clearEnvMap(); 44 | // refresh the internal map of orgID-MoesifKey 45 | keyRetriever.initOrRefreshOrgIDMoesifKeyMap(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/moesif/util/MoesifMicroserviceConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.moesif.util; 19 | 20 | /** 21 | * Class for constants related to external Moesif microservice. 22 | */ 23 | public class MoesifMicroserviceConstants { 24 | public static final String MOESIF_PROTOCOL_WITH_FQDN_KEY = "moesifProtocolWithFQDN"; 25 | public static final String MOESIF_EP_COMMON_PATH = "moesif/moesif_key"; 26 | public static final String MOESIF_MS_VERSIONING_KEY = "moesifMSVersioning"; 27 | public static final String MS_USERNAME_CONFIG_KEY = "moesifMSAuthUsername"; 28 | public static final String MS_PWD_CONFIG_KEY = "moesifMSAuthPwd"; 29 | public static final String CONTENT_TYPE = "application/json"; 30 | public static final String QUERY_PARAM = "org_id"; 31 | public static final int NUM_RETRY_ATTEMPTS = 3; 32 | public static final long TIME_TO_WAIT = 10000; 33 | public static final int NUM_RETRY_ATTEMPTS_PUBLISH = 3; 34 | public static final long TIME_TO_WAIT_PUBLISH = 10000; 35 | public static final int REQUEST_READ_TIMEOUT = 10000; 36 | public static final long PERIODIC_CALL_DELAY = 300000; 37 | 38 | } 39 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/log/LogMetricEventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.log; 20 | 21 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 22 | import org.wso2.am.analytics.publisher.reporter.AbstractMetricEventBuilder; 23 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 24 | 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | 28 | /** 29 | * Event builder for log Metric Reporter. 30 | */ 31 | public class LogMetricEventBuilder extends AbstractMetricEventBuilder { 32 | private Map eventMap = new HashMap<>(); 33 | 34 | @Override 35 | protected Map buildEvent() { 36 | return eventMap; 37 | } 38 | 39 | @Override 40 | public boolean validate() throws MetricReportingException { 41 | for (Object value : eventMap.values()) { 42 | if (!(value instanceof String)) { 43 | throw new MetricReportingException("Only attributes of type String is supported"); 44 | } 45 | } 46 | return true; 47 | } 48 | 49 | @Override public MetricEventBuilder addAttribute(String key, Object value) throws MetricReportingException { 50 | eventMap.put(key, value); 51 | return this; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/DefaultAnalyticsThreadFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.wso2.am.analytics.publisher.reporter.cloud; 17 | 18 | import java.util.concurrent.ThreadFactory; 19 | import java.util.concurrent.atomic.AtomicInteger; 20 | 21 | /** 22 | * Custom Thread Factory for Analytics publisher impl. 23 | */ 24 | public class DefaultAnalyticsThreadFactory implements ThreadFactory { 25 | private static final AtomicInteger poolNumber = new AtomicInteger(1); 26 | final ThreadGroup group; 27 | final AtomicInteger threadNumber = new AtomicInteger(1); 28 | final String namePrefix; 29 | 30 | public DefaultAnalyticsThreadFactory(String threadPoolExecutorName) { 31 | SecurityManager s = System.getSecurityManager(); 32 | group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); 33 | namePrefix = "Cloud-Analytics-" + threadPoolExecutorName + "-pool-" + poolNumber.getAndIncrement() + 34 | "-thread-"; 35 | } 36 | 37 | public Thread newThread(Runnable r) { 38 | Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); 39 | if (t.isDaemon()) { 40 | t.setDaemon(false); 41 | } 42 | if (t.getPriority() != Thread.NORM_PRIORITY) { 43 | t.setPriority(Thread.NORM_PRIORITY); 44 | } 45 | return t; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /component/publisher-client/src/test/resources/testng.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/MetricReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter; 19 | 20 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 21 | 22 | import java.util.Map; 23 | 24 | /** 25 | * Base interface for all Metric Reporter implementations. {@link AbstractMetricReporter} will implement this 26 | * interface and any concrete implementations should extend {@link AbstractMetricReporter} 27 | */ 28 | public interface MetricReporter { 29 | 30 | /** 31 | * Create and return {@link CounterMetric} to instrument collected metrics. 32 | * 33 | * @param name Name of the metric 34 | * @param schema Metric schema 35 | * @return {@link CounterMetric} 36 | * @throws MetricCreationException if error occurred when creating CounterMetric 37 | */ 38 | CounterMetric createCounterMetric(String name, MetricSchema schema) throws MetricCreationException; 39 | 40 | /** 41 | * Create and return {@link TimerMetric} to instrument collected metrics. 42 | * 43 | * @param name Name of the metric 44 | * @return {@link TimerMetric} 45 | */ 46 | TimerMetric createTimerMetric(String name); 47 | 48 | /** 49 | * Returns the currently set configurations. Setting configurations will be mandated through constructor 50 | * 51 | * @return {@link Map} representing current configurations 52 | */ 53 | Map getConfiguration(); 54 | 55 | } 56 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/elk/ELKMetricReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.elk; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 24 | import org.wso2.am.analytics.publisher.reporter.AbstractMetricReporter; 25 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 26 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 27 | import org.wso2.am.analytics.publisher.reporter.TimerMetric; 28 | 29 | import java.util.Map; 30 | 31 | /** 32 | * Log Metric Reporter class. Intended for testing only. 33 | */ 34 | public class ELKMetricReporter extends AbstractMetricReporter { 35 | private static final Logger log = LogManager.getLogger(ELKMetricReporter.class); 36 | 37 | public ELKMetricReporter(Map properties) throws MetricCreationException { 38 | super(properties); 39 | log.info("LogMetricReporter successfully initialized"); 40 | } 41 | 42 | @Override 43 | protected void validateConfigProperties(Map properties) throws MetricCreationException { 44 | //nothing to validate 45 | } 46 | 47 | @Override 48 | protected CounterMetric createCounter(String name, MetricSchema schema) { 49 | return new ELKCounterMetric(name, schema); 50 | } 51 | 52 | @Override 53 | protected TimerMetric createTimer(String name) { 54 | return null; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/log/LogMetricReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.log; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 24 | import org.wso2.am.analytics.publisher.reporter.AbstractMetricReporter; 25 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 26 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 27 | import org.wso2.am.analytics.publisher.reporter.TimerMetric; 28 | 29 | import java.util.Map; 30 | 31 | /** 32 | * Log Metric Reporter class. Intended for testing only. 33 | */ 34 | public class LogMetricReporter extends AbstractMetricReporter { 35 | private static final Logger log = LogManager.getLogger(LogMetricReporter.class); 36 | 37 | 38 | public LogMetricReporter(Map properties) throws MetricCreationException { 39 | super(properties); 40 | log.info("LogMetricReporter successfully initialized"); 41 | } 42 | 43 | @Override 44 | protected void validateConfigProperties(Map properties) throws MetricCreationException { 45 | //nothing to validate 46 | } 47 | 48 | @Override 49 | protected CounterMetric createCounter(String name, MetricSchema schema) { 50 | return new LogCounterMetric(name); 51 | } 52 | 53 | @Override 54 | protected TimerMetric createTimer(String name) { 55 | return null; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/log/LogCounterMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.log; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 24 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 25 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 26 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 27 | 28 | import java.util.Map; 29 | 30 | /** 31 | * Log Counter Metrics class, This class can be used to log analytics event to a separate log file. 32 | */ 33 | public class LogCounterMetric implements CounterMetric { 34 | private static final Logger log = LogManager.getLogger(LogCounterMetric.class); 35 | private final String name; 36 | 37 | protected LogCounterMetric(String name) { 38 | this.name = name; 39 | } 40 | 41 | @Override 42 | public int incrementCount(MetricEventBuilder builder) throws MetricReportingException { 43 | Map properties = builder.build(); 44 | log.info("Metric Name: " + name.replaceAll("[\r\n]", "") + " Metric Value: " 45 | + properties.toString().replaceAll("[\r\n]", "")); 46 | return 0; 47 | } 48 | 49 | @Override 50 | public String getName() { 51 | return name; 52 | } 53 | 54 | @Override 55 | public MetricSchema getSchema() { 56 | return null; 57 | } 58 | 59 | @Override 60 | public MetricEventBuilder getEventBuilder() { 61 | return new LogMetricEventBuilder(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /component/publisher-client/src/test/java/org/wso2/am/analytics/publisher/util/AuthAPIMockService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.util; 20 | 21 | import org.mockserver.integration.ClientAndServer; 22 | import org.mockserver.model.JsonBody; 23 | import org.testng.annotations.AfterClass; 24 | import org.testng.annotations.BeforeClass; 25 | import org.wso2.am.analytics.publisher.auth.TokenDetailsDTO; 26 | 27 | import static org.mockserver.model.HttpRequest.request; 28 | import static org.mockserver.model.HttpResponse.response; 29 | 30 | /** 31 | * Mocking auth-api service. 32 | */ 33 | public class AuthAPIMockService { 34 | 35 | protected static final String SAS_TOKEN = "SharedAccessSignature sr=sb://localhost/incoming-hub/publishers" 36 | + "/pub1&sig=signature&se=1641892957&skn=send-policy"; 37 | private static final int TEST_PORT = 9191; 38 | private ClientAndServer mockServer; 39 | protected String authApiEndpoint; 40 | 41 | @BeforeClass 42 | public void startServer() { 43 | mockServer = ClientAndServer.startClientAndServer(TEST_PORT); 44 | authApiEndpoint = "http://localhost:" + TEST_PORT + "/auth-api"; 45 | } 46 | 47 | @AfterClass 48 | public void stopServer() { 49 | mockServer.stop(); 50 | } 51 | 52 | protected void mock(int responseCode, String token) { 53 | 54 | TokenDetailsDTO dto = new TokenDetailsDTO(); 55 | dto.setToken(SAS_TOKEN); 56 | mockServer.when( 57 | request() 58 | .withMethod("GET") 59 | .withPath("/auth-api/token") 60 | .withHeader("Authorization", "Bearer " + token) 61 | ) 62 | .respond( 63 | response() 64 | .withStatusCode(responseCode) 65 | .withBody(JsonBody.json(dto)) 66 | ); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | > Describe the problems, issues, or needs driving this feature/fix and include links to related issues in the following format: Resolves issue1, issue2, etc. 3 | 4 | ## Goals 5 | > Describe the solutions that this feature/fix will introduce to resolve the problems described above 6 | 7 | ## Approach 8 | > Describe how you are implementing the solutions. Include an animated GIF or screenshot if the change affects the UI (email documentation@wso2.com to review all UI text). Include a link to a Markdown file or Google doc if the feature write-up is too long to paste here. 9 | 10 | ## User stories 11 | > Summary of user stories addressed by this change> 12 | 13 | ## Release note 14 | > Brief description of the new feature or bug fix as it will appear in the release notes 15 | 16 | ## Documentation 17 | > Link(s) to product documentation that addresses the changes of this PR. If no doc impact, enter “N/A” plus brief explanation of why there’s no doc impact 18 | 19 | ## Training 20 | > Link to the PR for changes to the training content in https://github.com/wso2/WSO2-Training, if applicable 21 | 22 | ## Certification 23 | > Type “Sent” when you have provided new/updated certification questions, plus four answers for each question (correct answer highlighted in bold), based on this change. Certification questions/answers should be sent to certification@wso2.com and NOT pasted in this PR. If there is no impact on certification exams, type “N/A” and explain why. 24 | 25 | ## Marketing 26 | > Link to drafts of marketing content that will describe and promote this feature, including product page changes, technical articles, blog posts, videos, etc., if applicable 27 | 28 | ## Automation tests 29 | - Unit tests 30 | > Code coverage information 31 | - Integration tests 32 | > Details about the test cases and coverage 33 | 34 | ## Security checks 35 | - Followed secure coding standards in http://wso2.com/technical-reports/wso2-secure-engineering-guidelines? yes/no 36 | - Ran FindSecurityBugs plugin and verified report? yes/no 37 | - Confirmed that this PR doesn't commit any keys, passwords, tokens, usernames, or other secrets? yes/no 38 | 39 | ## Samples 40 | > Provide high-level details about the samples related to this feature 41 | 42 | ## Related PRs 43 | > List any other related PRs 44 | 45 | ## Migrations (if applicable) 46 | > Describe migration steps and platforms on which migration has been tested 47 | 48 | ## Test environment 49 | > List all JDK versions, operating systems, databases, and browser/versions on which this feature/fix was tested 50 | 51 | ## Learning 52 | > Describe the research phase and any blog posts, patterns, libraries, or add-ons you used to solve the problem. -------------------------------------------------------------------------------- /component/publisher-client/src/test/java/org/wso2/am/analytics/publisher/util/UnitTestAppender.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.util; 20 | 21 | import org.apache.logging.log4j.core.Filter; 22 | import org.apache.logging.log4j.core.LogEvent; 23 | import org.apache.logging.log4j.core.appender.AbstractAppender; 24 | import org.apache.logging.log4j.core.config.plugins.Plugin; 25 | import org.apache.logging.log4j.core.config.plugins.PluginAttribute; 26 | import org.apache.logging.log4j.core.config.plugins.PluginElement; 27 | import org.apache.logging.log4j.core.config.plugins.PluginFactory; 28 | 29 | import java.util.ArrayList; 30 | import java.util.Collections; 31 | import java.util.List; 32 | 33 | @Plugin(name = "UnitTestAppender", category = "Core", elementType = "appender", printObject = true) 34 | public class UnitTestAppender extends AbstractAppender { 35 | private List messages = Collections.synchronizedList(new ArrayList<>()); 36 | 37 | protected UnitTestAppender(String name, Filter filter) { 38 | super(name, filter, null, false, null); 39 | } 40 | 41 | @PluginFactory 42 | public static UnitTestAppender createAppender( 43 | @PluginAttribute("name") String name, 44 | @PluginElement("Filter") Filter filter) { 45 | if (name == null) { 46 | LOGGER.error("No name provided for UnitTestAppender"); 47 | return null; 48 | } 49 | return new UnitTestAppender(name, filter); 50 | } 51 | 52 | public List getMessages() { 53 | return messages; 54 | } 55 | 56 | public boolean checkContains(String message) { 57 | for (String log : messages) { 58 | if (log.contains(message)) { 59 | return true; 60 | } 61 | } 62 | return false; 63 | } 64 | 65 | @Override 66 | public void append(LogEvent event) { 67 | messages.add(event.getMessage().getFormattedMessage()); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/MetricEventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 22 | 23 | import java.util.Map; 24 | 25 | /** 26 | * Main interface class for Metric Event Builders. Metric Event Builders are responsible of collecting metrics, 27 | * validating them and later returning them as a Map<String, Object>. Default builders will be implemented and 28 | * for any custom message building new builders have to be introduced 29 | */ 30 | public interface MetricEventBuilder { 31 | 32 | /** 33 | * Validates the provided attributes and build a flat {@link Map}. Any validation failures will cause 34 | * {@link org.wso2.am.analytics.publisher.exception.MetricReportingException}. 35 | * 36 | * @return Map containing all attributes related to Metric Event 37 | * @throws MetricReportingException if validation failed 38 | */ 39 | public Map build() throws MetricReportingException; 40 | 41 | /** 42 | * Checks the validity of the added attributes. If all required attributes are present true will be returned. 43 | * Else {@link MetricReportingException} will be thrown. 44 | * 45 | * @return Validity state of the added data 46 | * @throws MetricReportingException if validation failed 47 | */ 48 | public boolean validate() throws MetricReportingException; 49 | 50 | /** 51 | * Method to add any attribute to the builder. Each concrete implementation can implement validations 52 | * based on the key. 53 | * 54 | * @param key Key of the attribute 55 | * @param number Value of the attribute 56 | * @return Returns itself to support chaining 57 | * @throws MetricReportingException if validation failed 58 | */ 59 | public MetricEventBuilder addAttribute(String key, Object number) throws MetricReportingException; 60 | } 61 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/util/UserAgentParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.util; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import ua_parser.Client; 24 | import ua_parser.Parser; 25 | 26 | import java.util.LinkedHashMap; 27 | import java.util.Map; 28 | 29 | /** 30 | * User agent parser util class. 31 | */ 32 | public class UserAgentParser { 33 | private static final Logger log = LogManager.getLogger(UserAgentParser.class); 34 | private static final UserAgentParser INSTANCE = new UserAgentParser(); 35 | private boolean isInitialized = false; 36 | private Parser uaParser; 37 | private Map clientCache; 38 | 39 | private UserAgentParser() { 40 | uaParser = new Parser(); 41 | isInitialized = true; 42 | clientCache = new LinkedHashMap(Constants.USER_AGENT_DEFAULT_CACHE_SIZE 43 | + Constants.DEFAULT_WORKER_THREADS) { 44 | static final int MAX = Constants.USER_AGENT_DEFAULT_CACHE_SIZE; 45 | @Override 46 | protected boolean removeEldestEntry(Map.Entry eldest) { 47 | return size() > MAX; 48 | } 49 | }; 50 | } 51 | 52 | public static UserAgentParser getInstance() { 53 | return INSTANCE; 54 | } 55 | 56 | public Client parseUserAgent(String userAgentHeader) { 57 | if (isInitialized) { 58 | Client client = clientCache.get(userAgentHeader); 59 | if (client == null) { 60 | client = uaParser.parse(userAgentHeader); 61 | clientCache.put(userAgentHeader, client); 62 | log.debug("user agent added to cache. Current cache size is " + clientCache.size()); 63 | return client; 64 | } else { 65 | log.debug("User agent fetched from cache"); 66 | return client; 67 | } 68 | } else { 69 | return null; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /findbugs-exclude.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/elk/ELKCounterMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.elk; 20 | 21 | import com.google.gson.Gson; 22 | import org.apache.logging.log4j.LogManager; 23 | import org.apache.logging.log4j.Logger; 24 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 25 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 26 | import org.wso2.am.analytics.publisher.reporter.GenericInputValidator; 27 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 28 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 29 | 30 | import java.util.Map; 31 | 32 | /** 33 | * Log Counter Metrics class, This class can be used to log analytics event to a separate log file. 34 | */ 35 | public class ELKCounterMetric implements CounterMetric { 36 | private static final Logger log = LogManager.getLogger(ELKCounterMetric.class); 37 | private final String name; 38 | private final Gson gson; 39 | private MetricSchema schema; 40 | 41 | protected ELKCounterMetric(String name, MetricSchema schema) { 42 | this.name = name; 43 | this.gson = new Gson(); 44 | this.schema = schema; 45 | } 46 | 47 | @Override 48 | public int incrementCount(MetricEventBuilder builder) throws MetricReportingException { 49 | Map event = builder.build(); 50 | String jsonString = gson.toJson(event); 51 | 52 | log.info("apimMetrics: " + name.replaceAll("[\r\n]", "") + ", properties :" + 53 | jsonString.replaceAll("[\r\n]", "")); 54 | return 0; 55 | } 56 | 57 | @Override 58 | public String getName() { 59 | return name; 60 | } 61 | 62 | @Override 63 | public MetricSchema getSchema() { 64 | return schema; 65 | } 66 | 67 | @Override 68 | public MetricEventBuilder getEventBuilder() { 69 | 70 | switch (schema) { 71 | case RESPONSE: 72 | default: 73 | return new ELKMetricEventBuilder( 74 | GenericInputValidator.getInstance().getEventProperties(MetricSchema.ELK_RESPONSE)); 75 | case ERROR: 76 | return new ELKMetricEventBuilder( 77 | GenericInputValidator.getInstance().getEventProperties(MetricSchema.ELK_ERROR)); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/properties/AIMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, WSO2 LLC. (http://www.wso2.com). 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.properties; 20 | 21 | import org.wso2.am.analytics.publisher.util.Constants; 22 | 23 | import java.util.Map; 24 | 25 | /** 26 | * Represents metadata information about the AI model used in API processing. 27 | * This includes the vendor details, model name, and version. 28 | */ 29 | public class AIMetadata { 30 | private String vendorName; 31 | private String model; 32 | private String vendorVersion; 33 | 34 | /** 35 | * Default constructor. 36 | */ 37 | public AIMetadata() { 38 | } 39 | 40 | /** 41 | * Constructor to initialize from a Map 42 | */ 43 | public AIMetadata(Map map) { 44 | this.vendorName = (String) map.get(Constants.AI_VENDOR_NAME); 45 | this.model = (String) map.get(Constants.AI_MODEL); 46 | this.vendorVersion = (String) map.get(Constants.AI_VENDOR_VERSION); 47 | } 48 | 49 | /** 50 | * Gets the name of the AI model vendor. 51 | * 52 | * @return The AI vendor name. 53 | */ 54 | public String getVendorName() { 55 | return vendorName; 56 | } 57 | 58 | /** 59 | * Sets the name of the AI model vendor. 60 | * 61 | * @param vendorName The AI vendor name. 62 | */ 63 | public void setVendorName(String vendorName) { 64 | this.vendorName = vendorName; 65 | } 66 | 67 | /** 68 | * Gets the AI model name. 69 | * 70 | * @return The AI model name. 71 | */ 72 | public String getModel() { 73 | return model; 74 | } 75 | 76 | /** 77 | * Sets the AI model name. 78 | * 79 | * @param model The AI model name. 80 | */ 81 | public void setModel(String model) { 82 | this.model = model; 83 | } 84 | 85 | /** 86 | * Gets the version of the AI model provided by the vendor. 87 | * 88 | * @return The AI model vendor version. 89 | */ 90 | public String getVendorVersion() { 91 | return vendorVersion; 92 | } 93 | 94 | /** 95 | * Sets the version of the AI model provided by the vendor. 96 | * 97 | * @param vendorVersion The AI model vendor version. 98 | */ 99 | public void setVendorVersion(String vendorVersion) { 100 | this.vendorVersion = vendorVersion; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/properties/Properties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, WSO2 LLC. (http://www.wso2.com). 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.properties; 20 | 21 | /** 22 | * Represents the properties associated with an API request, 23 | * including routing information, AI-related metadata, and AI token usage. 24 | */ 25 | public class Properties { 26 | private boolean isEgress; 27 | private String subType; 28 | private AIMetadata aiMetadata; 29 | private AITokenUsage aiTokenUsage; 30 | 31 | /** 32 | * Default constructor. 33 | */ 34 | public Properties() { 35 | } 36 | 37 | /** 38 | * Checks if the request is an egress request. 39 | * 40 | * @return {@code true} if the request is egress, {@code false} otherwise. 41 | */ 42 | public boolean isEgress() { 43 | return isEgress; 44 | } 45 | 46 | /** 47 | * Sets whether the request is an egress request. 48 | * 49 | * @param egress {@code true} if the request is egress, {@code false} otherwise. 50 | */ 51 | public void setEgress(boolean egress) { 52 | isEgress = egress; 53 | } 54 | 55 | /** 56 | * Gets the subtype of the API request. 57 | * 58 | * @return The API request subtype. 59 | */ 60 | public String getSubType() { 61 | return subType; 62 | } 63 | 64 | /** 65 | * Sets the subtype of the API request. 66 | * 67 | * @param subType The API request subtype. 68 | */ 69 | public void setSubType(String subType) { 70 | this.subType = subType; 71 | } 72 | 73 | /** 74 | * Gets the AI metadata associated with this request. 75 | * 76 | * @return The {@link AIMetadata} instance. 77 | */ 78 | public AIMetadata getAiMetadata() { 79 | return aiMetadata; 80 | } 81 | 82 | /** 83 | * Sets the AI metadata for this request. 84 | * 85 | * @param aiMetadata The {@link AIMetadata} instance. 86 | */ 87 | public void setAiMetadata(AIMetadata aiMetadata) { 88 | this.aiMetadata = aiMetadata; 89 | } 90 | 91 | /** 92 | * Gets the AI token usage details for this request. 93 | * 94 | * @return The {@link AITokenUsage} instance. 95 | */ 96 | public AITokenUsage getAiTokenUsage() { 97 | return aiTokenUsage; 98 | } 99 | 100 | /** 101 | * Sets the AI token usage details for this request. 102 | * 103 | * @param aiTokenUsage The {@link AITokenUsage} instance. 104 | */ 105 | public void setAiTokenUsage(AITokenUsage aiTokenUsage) { 106 | this.aiTokenUsage = aiTokenUsage; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/properties/AITokenUsage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2025, WSO2 LLC. (http://www.wso2.com). 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.properties; 20 | 21 | import org.wso2.am.analytics.publisher.util.Constants; 22 | 23 | import java.util.Map; 24 | 25 | /** 26 | * Represents the AI token usage details, including prompt tokens, 27 | * total tokens, and completion tokens. 28 | * This helps in tracking API usage costs and resource consumption. 29 | */ 30 | public class AITokenUsage { 31 | private int promptTokens; 32 | private int totalTokens; 33 | private int completionTokens; 34 | 35 | /** 36 | * Default constructor. 37 | */ 38 | public AITokenUsage() { 39 | } 40 | 41 | /** 42 | * Constructor to initialize from a Map 43 | */ 44 | public AITokenUsage(Map map) { 45 | this.promptTokens = (int) map.get(Constants.AI_PROMPT_TOKEN_USAGE); 46 | this.totalTokens = (int) map.get(Constants.AI_TOTAL_TOKEN_USAGE); 47 | this.completionTokens = (int) map.get(Constants.AI_COMPLETION_TOKEN_USAGE); 48 | } 49 | 50 | 51 | /** 52 | * Gets the number of tokens used in the prompt. 53 | * 54 | * @return The number of prompt tokens. 55 | */ 56 | public int getPromptTokens() { 57 | return promptTokens; 58 | } 59 | 60 | /** 61 | * Sets the number of tokens used in the prompt. 62 | * 63 | * @param promptTokens The number of prompt tokens. 64 | */ 65 | public void setPromptTokens(int promptTokens) { 66 | this.promptTokens = promptTokens; 67 | } 68 | 69 | /** 70 | * Gets the total number of tokens used in the request. 71 | * 72 | * @return The total number of tokens. 73 | */ 74 | public int getTotalTokens() { 75 | return totalTokens; 76 | } 77 | 78 | /** 79 | * Sets the total number of tokens used in the request. 80 | * 81 | * @param totalTokens The total number of tokens. 82 | */ 83 | public void setTotalTokens(int totalTokens) { 84 | this.totalTokens = totalTokens; 85 | } 86 | 87 | /** 88 | * Gets the number of tokens used in the AI model's response. 89 | * 90 | * @return The number of completion tokens. 91 | */ 92 | public int getCompletionTokens() { 93 | return completionTokens; 94 | } 95 | 96 | /** 97 | * Sets the number of tokens used in the AI model's response. 98 | * 99 | * @param completionTokens The number of completion tokens. 100 | */ 101 | public void setCompletionTokens(int completionTokens) { 102 | this.completionTokens = completionTokens; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/ParallelQueueWorker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.cloud; 19 | 20 | import com.google.gson.Gson; 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.wso2.am.analytics.publisher.client.EventHubClient; 24 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 25 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 26 | 27 | import java.util.Map; 28 | import java.util.concurrent.BlockingQueue; 29 | 30 | /** 31 | * Will removes the events from queues and send then to the endpoints. 32 | */ 33 | public class ParallelQueueWorker implements Runnable { 34 | 35 | private static final Logger log = LogManager.getLogger(ParallelQueueWorker.class); 36 | private BlockingQueue eventQueue; 37 | private EventHubClient client; 38 | 39 | public ParallelQueueWorker(BlockingQueue queue, EventHubClient client) { 40 | this.client = client; 41 | this.eventQueue = queue; 42 | } 43 | 44 | public void run() { 45 | if (log.isDebugEnabled()) { 46 | log.debug(eventQueue.size() + " messages in queue before " + 47 | Thread.currentThread().getName().replaceAll("[\r\n]", "") 48 | + " worker has polled queue"); 49 | } 50 | while (true) { 51 | MetricEventBuilder eventBuilder; 52 | String event; 53 | try { 54 | eventBuilder = eventQueue.take(); 55 | if (eventBuilder != null) { 56 | Map eventMap = eventBuilder.build(); 57 | event = new Gson().toJson(eventMap); 58 | client.sendEvent(event); 59 | } 60 | } catch (MetricReportingException e) { 61 | log.error("Builder instance is not duly filled. Event building failed", e); 62 | continue; 63 | } catch (InterruptedException e) { 64 | Thread.currentThread().interrupt(); 65 | } catch (Exception e) { 66 | log.error("Analytics event sending failed. Event will be dropped", e); 67 | } 68 | 69 | if (log.isDebugEnabled()) { 70 | log.debug(eventQueue.size() + " messages in queue after " + 71 | Thread.currentThread().getName().replaceAll("[\r\n]", "") 72 | + " worker has finished work"); 73 | } 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/moesif/MoesifCounterMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.moesif; 19 | 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 23 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 24 | import org.wso2.am.analytics.publisher.reporter.GenericInputValidator; 25 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 26 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 27 | 28 | /** 29 | * Implementation of {@link CounterMetric} for Moesif Metric Reporter. 30 | */ 31 | public class MoesifCounterMetric implements CounterMetric { 32 | private static final Logger log = LogManager.getLogger(MoesifCounterMetric.class); 33 | private EventQueue queue; 34 | private String name; 35 | private MetricSchema schema; 36 | 37 | public MoesifCounterMetric(String name, EventQueue queue, MetricSchema schema) { 38 | this.name = name; 39 | this.schema = schema; 40 | this.queue = queue; 41 | } 42 | 43 | 44 | @Override 45 | public int incrementCount(MetricEventBuilder metricEventBuilder) throws MetricReportingException { 46 | queue.put(metricEventBuilder); 47 | return 0; 48 | } 49 | 50 | @Override 51 | public String getName() { 52 | return this.name; 53 | } 54 | 55 | @Override 56 | public MetricSchema getSchema() { 57 | return this.schema; 58 | } 59 | 60 | /** 61 | * Returns Event Builder used for this CounterMetric. Depending on the schema different types of builders will be 62 | * returned. 63 | * 64 | * @return {@link MetricEventBuilder} for this {@link CounterMetric} 65 | */ 66 | @Override 67 | public MetricEventBuilder getEventBuilder() { 68 | switch (schema) { 69 | case RESPONSE: 70 | default: 71 | return new MoesifMetricEventBuilder( 72 | GenericInputValidator.getInstance().getEventProperties(MetricSchema.MOESIF_RESPONSE)); 73 | case ERROR: 74 | return new MoesifMetricEventBuilder( 75 | GenericInputValidator.getInstance().getEventProperties(MetricSchema.MOESIF_ERROR)); 76 | case CHOREO_RESPONSE: 77 | return new MoesifMetricEventBuilder( 78 | GenericInputValidator.getInstance().getEventProperties(MetricSchema.CHOREO_RESPONSE)); 79 | case CHOREO_ERROR: 80 | return new MoesifMetricEventBuilder( 81 | GenericInputValidator.getInstance().getEventProperties(MetricSchema.CHOREO_ERROR)); 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/util/HttpStatusHelper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.util; 19 | 20 | import java.util.Arrays; 21 | import java.util.Collections; 22 | import java.util.HashSet; 23 | import java.util.Set; 24 | /** 25 | * Utility class for handling HTTP status codes. 26 | * Provides methods to determine the type of HTTP response (success, client error, server error) 27 | * and whether a request should be retried based on the status code. 28 | */ 29 | public class HttpStatusHelper { 30 | // HTTP Status Code Ranges 31 | public static final int SUCCESS_RANGE_START = 200; 32 | public static final int SUCCESS_RANGE_END = 299; 33 | public static final int CLIENT_ERROR_RANGE_START = 400; 34 | public static final int CLIENT_ERROR_RANGE_END = 499; 35 | public static final int SERVER_ERROR_RANGE_START = 500; 36 | public static final int SERVER_ERROR_RANGE_END = 599; 37 | 38 | private static final Set RETRYABLE_CLIENT_ERRORS = 39 | Collections.unmodifiableSet(new HashSet<>(Arrays.asList(408, 429))); 40 | /** 41 | * Checks if the HTTP status code indicates a successful response. 42 | * Success range: 200-299 43 | * 44 | * @param statusCode HTTP status code to check 45 | * @return true if status code is in success range (2xx) 46 | */ 47 | public static boolean isSuccess(int statusCode) { 48 | return statusCode >= SUCCESS_RANGE_START && 49 | statusCode <= SUCCESS_RANGE_END; 50 | } 51 | 52 | /** 53 | * Checks if the HTTP status code indicates a client error. 54 | * Client error range: 400-499 55 | * 56 | * @param statusCode HTTP status code to check 57 | * @return true if status code is in client error range (4xx) 58 | */ 59 | public static boolean isClientError(int statusCode) { 60 | return statusCode >= CLIENT_ERROR_RANGE_START && 61 | statusCode < CLIENT_ERROR_RANGE_END; 62 | } 63 | 64 | /** 65 | * Checks if the HTTP status code indicates a server error. 66 | * Server error range: 500-599 67 | * 68 | * @param statusCode HTTP status code to check 69 | * @return true if status code is in server error range (5xx) 70 | */ 71 | public static boolean isServerError(int statusCode) { 72 | return statusCode >= SERVER_ERROR_RANGE_START && 73 | statusCode < SERVER_ERROR_RANGE_END; 74 | } 75 | 76 | /** 77 | * Determine if the request should be retried based on status code. 78 | * Retryable conditions: server errors (5xx) and specific client errors (408, 429). 79 | * @param statusCode HTTP status code 80 | * @return true if request should be retried 81 | */ 82 | public static boolean shouldRetry(int statusCode) { 83 | return isServerError(statusCode) || RETRYABLE_CLIENT_ERRORS.contains(statusCode); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/client/DataHolder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.client; 19 | 20 | /** 21 | * Data holder class to generate mock analytics events. 22 | */ 23 | public class DataHolder { 24 | static final String[] NODE_ID = new String[]{"1", "2", "3", "4", "5"}; 25 | static final String[] DEPLOYMENT_ID = 26 | new String[]{"prod", "prod", "prod", "prod", "prod"}; 27 | static final String[] API_UUID = new String[]{"apiUUID1", "apiUUID2", "apiUUID3", "apiUUID4", "apiUUID5"}; 28 | static final String[] REGION_ID = new String[]{"region1", "region2", "region3", "region4", "region5"}; 29 | static final String[] GATEWAY_TYPE = new String[]{"type1", "type2", "type3", "type4", "type5"}; 30 | static final String[] DESTINATION = 31 | new String[]{"destination1", "destination2", "destination3", "destination4", "destination5"}; 32 | static final String[] REQUEST_MED_LATENCY = new String[]{"100", "200", "300", "400", "500"}; 33 | static final String[] RESPONSE_MED_LATENCY = new String[]{"500", "400", "300", "200", "100"}; 34 | static final String[] RESPONSE_LATENCY = new String[]{"100", "200", "300", "400", "500"}; 35 | static final String[] RESPONSE_CODE = new String[]{"100", "200", "300", "400", "500"}; 36 | static final String[] RESPONSE_SIZE = new String[]{"100", "200", "300", "400", "500"}; 37 | static final String[] 38 | API_CREATOR = new String[]{"creator1", "creator2", "creator3", "creator4", "creator5"}; 39 | static final String[] API_METHOD = new String[]{"POST", "GET", "PUT", "DELETE", "PATCH"}; 40 | static final String[] API_RESOURCE_TEMPLATE = new String[]{"/{id}", "/{name}", "/{age}", "/{gender}", "/{country}"}; 41 | static final String[] API_VERSION = new String[]{"1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0"}; 42 | static final String[] API_NAME = new String[]{"api1", "api2", "api3", "api4", "api5"}; 43 | static final String[] 44 | API_CONTEXT = new String[]{"/context1", "/context2", "/context3", "/context4", "/context5"}; 45 | static final String[] APPLICATION_NAME = new String[]{"app1", "app2", "app3", "app4", "app5"}; 46 | static final String[] KEY_TYPE = new String[]{"production", "sandbox"}; 47 | static final String[] 48 | API_CREATOR_TENANT_DOMAIN = new String[]{"carbon.super", "carbon.super", "carbon.super", "carbon.super", 49 | "carbon.super"}; 50 | static final String[] APPLICATION_CONSUMER_KEY = new String[]{"key1", "key2", "key3", "key4", "key5"}; 51 | static final String[] APPLICATION_OWNER = new String[]{"owner1", "owner2", "owner3", "owner4", "owner5"}; 52 | static final String[] USER_AGENT = new String[]{"agent1", "agent2", "agent3", "agent4", "agent5"}; 53 | static final String[] EVENT_TYPE = new String[]{"response", "response", "response", "response", "response"}; 54 | } 55 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/moesif/MoesifMetricEventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.moesif; 19 | 20 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 21 | import org.wso2.am.analytics.publisher.reporter.AbstractMetricEventBuilder; 22 | import org.wso2.am.analytics.publisher.reporter.GenericInputValidator; 23 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 24 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 25 | import org.wso2.am.analytics.publisher.util.EventMapAttributeFilter; 26 | 27 | import java.util.HashMap; 28 | import java.util.Map; 29 | 30 | /** 31 | * Event builder for Moesif Metric Reporter. Default Event builder has response schema type. 32 | */ 33 | public class MoesifMetricEventBuilder extends AbstractMetricEventBuilder { 34 | protected Map requiredAttributes; 35 | private Map eventMap; 36 | private Boolean isBuilt = false; 37 | 38 | public MoesifMetricEventBuilder() { 39 | requiredAttributes = GenericInputValidator.getInstance().getEventProperties(MetricSchema.MOESIF_RESPONSE); 40 | eventMap = new HashMap<>(); 41 | } 42 | 43 | public MoesifMetricEventBuilder(Map requiredAttributes) { 44 | this.requiredAttributes = requiredAttributes; 45 | eventMap = new HashMap<>(); 46 | } 47 | 48 | @Override 49 | protected Map buildEvent() { 50 | if (!isBuilt) { 51 | // util function to filter required attributes 52 | eventMap = EventMapAttributeFilter.getInstance().filter(eventMap, requiredAttributes); 53 | 54 | isBuilt = true; 55 | } 56 | return eventMap; 57 | } 58 | 59 | @Override 60 | public boolean validate() throws MetricReportingException { 61 | if (!isBuilt) { 62 | for (Map.Entry entry : requiredAttributes.entrySet()) { 63 | Object attribute = eventMap.get(entry.getKey()); 64 | if (attribute == null) { 65 | throw new MetricReportingException(entry.getKey() + " is missing in metric data. This metric event " 66 | + "will not be processed further."); 67 | } else if (!attribute.getClass().equals(entry.getValue())) { 68 | throw new MetricReportingException(entry.getKey() + " is expecting a " + entry.getValue() + " type " 69 | + "attribute while attribute of type " 70 | + attribute.getClass() + " is present."); 71 | } 72 | } 73 | } 74 | return true; 75 | } 76 | 77 | @Override 78 | public MetricEventBuilder addAttribute(String key, Object value) throws MetricReportingException { 79 | eventMap.put(key, value); 80 | return this; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/QueueWorker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.cloud; 19 | 20 | import com.google.gson.Gson; 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.wso2.am.analytics.publisher.client.EventHubClient; 24 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 25 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 26 | 27 | import java.util.Map; 28 | import java.util.concurrent.BlockingQueue; 29 | import java.util.concurrent.ExecutorService; 30 | import java.util.concurrent.ThreadPoolExecutor; 31 | 32 | /** 33 | * Will removes the events from queues and send then to the endpoints. 34 | */ 35 | public class QueueWorker implements Runnable { 36 | 37 | private static final Logger log = LogManager.getLogger(QueueWorker.class); 38 | private BlockingQueue eventQueue; 39 | private ExecutorService executorService; 40 | private EventHubClient client; 41 | 42 | public QueueWorker(BlockingQueue queue, EventHubClient client, 43 | ExecutorService executorService) { 44 | this.client = client; 45 | this.eventQueue = queue; 46 | this.executorService = executorService; 47 | } 48 | 49 | public void run() { 50 | try { 51 | if (log.isDebugEnabled()) { 52 | log.debug(eventQueue.size() + " messages in queue before " + 53 | Thread.currentThread().getName().replaceAll("[\r\n]", "") 54 | + " worker has polled queue"); 55 | } 56 | ThreadPoolExecutor threadPoolExecutor = ((ThreadPoolExecutor) executorService); 57 | do { 58 | MetricEventBuilder eventBuilder = eventQueue.poll(); 59 | if (eventBuilder != null) { 60 | String event; 61 | try { 62 | Map eventMap = eventBuilder.build(); 63 | event = new Gson().toJson(eventMap); 64 | } catch (MetricReportingException e) { 65 | log.error("Builder instance is not duly filled. Event building failed", e); 66 | continue; 67 | } 68 | client.sendEvent(event); 69 | } else { 70 | break; 71 | } 72 | } while (threadPoolExecutor.getActiveCount() == 1 && eventQueue.size() != 0); 73 | //while condition to handle possible task rejections 74 | if (log.isDebugEnabled()) { 75 | log.debug(eventQueue.size() + " messages in queue after " + 76 | Thread.currentThread().getName().replaceAll("[\r\n]", "") 77 | + " worker has finished work"); 78 | } 79 | } catch (Throwable e) { 80 | log.error("Error in passing events to Event Hub client. Events dropped", e); 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /component/publisher-client/src/test/java/org/wso2/am/analytics/publisher/util/TestUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.util; 20 | 21 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 22 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 23 | 24 | import java.time.Clock; 25 | import java.time.OffsetDateTime; 26 | import java.util.HashMap; 27 | import java.util.List; 28 | 29 | /** 30 | * Util class containing helper methods for test 31 | */ 32 | public class TestUtils { 33 | public static void populateBuilder(MetricEventBuilder builder) throws MetricReportingException { 34 | String uaString = "Mozilla/5.0 (iPhone; CPU iPhone OS 5_1_1 like Mac OS X) AppleWebKit/534.46 (KHTML, " 35 | + "like Gecko) Version/5.1 Mobile/9B206 Safari/7534.48.3"; 36 | 37 | builder.addAttribute(Constants.REQUEST_TIMESTAMP, OffsetDateTime.now(Clock.systemUTC()).toString()) 38 | .addAttribute(Constants.CORRELATION_ID, "1234-4567") 39 | .addAttribute(Constants.KEY_TYPE, "prod") 40 | .addAttribute(Constants.API_ID, "9876-54f1") 41 | .addAttribute(Constants.API_TYPE, "HTTP") 42 | .addAttribute(Constants.API_NAME, "PizzaShack") 43 | .addAttribute(Constants.API_VERSION, "1.0.0") 44 | .addAttribute(Constants.API_CREATION, "admin") 45 | .addAttribute(Constants.API_METHOD, "POST") 46 | .addAttribute(Constants.API_CONTEXT, "/v1/") 47 | .addAttribute(Constants.API_RESOURCE_TEMPLATE, "/resource/{value}") 48 | .addAttribute(Constants.API_CREATOR_TENANT_DOMAIN, "carbon.super") 49 | .addAttribute(Constants.DESTINATION, "localhost:8080") 50 | .addAttribute(Constants.APPLICATION_ID, "3445-6778") 51 | .addAttribute(Constants.APPLICATION_NAME, "default") 52 | .addAttribute(Constants.APPLICATION_OWNER, "admin") 53 | .addAttribute(Constants.REGION_ID, "NA") 54 | .addAttribute(Constants.GATEWAY_TYPE, "Synapse") 55 | .addAttribute(Constants.USER_AGENT_HEADER, uaString) 56 | .addAttribute(Constants.USER_NAME, "admin") 57 | .addAttribute(Constants.PROXY_RESPONSE_CODE, 401) 58 | .addAttribute(Constants.TARGET_RESPONSE_CODE, 401) 59 | .addAttribute(Constants.RESPONSE_CACHE_HIT, true) 60 | .addAttribute(Constants.RESPONSE_LATENCY, 2000L) 61 | .addAttribute(Constants.BACKEND_LATENCY, 3000L) 62 | .addAttribute(Constants.REQUEST_MEDIATION_LATENCY, 1000L) 63 | .addAttribute(Constants.RESPONSE_MEDIATION_LATENCY, 1000L) 64 | .addAttribute(Constants.USER_IP, "127.0.0.1") 65 | .addAttribute(Constants.PROPERTIES, new HashMap()); 66 | } 67 | 68 | public static boolean isContains(List messages, String message) { 69 | for (String log : messages) { 70 | if (log.contains(message)) { 71 | return true; 72 | } 73 | } 74 | return false; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/auth/AuthClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.auth; 20 | 21 | import feign.Feign; 22 | import feign.FeignException; 23 | import feign.RetryableException; 24 | import feign.gson.GsonDecoder; 25 | import feign.gson.GsonEncoder; 26 | import feign.slf4j.Slf4jLogger; 27 | import org.wso2.am.analytics.publisher.exception.ConnectionRecoverableException; 28 | import org.wso2.am.analytics.publisher.exception.ConnectionUnrecoverableException; 29 | import org.wso2.am.analytics.publisher.util.Constants; 30 | 31 | import java.util.Map; 32 | 33 | /** 34 | * Auth client to generate SAS token that can use to authenticate with event hub. 35 | */ 36 | public class AuthClient { 37 | public static final String AUTH_HEADER = "Authorization"; 38 | 39 | public static String getSASToken(String authEndpoint, String token, Map properties) 40 | throws ConnectionRecoverableException, ConnectionUnrecoverableException { 41 | 42 | String isProxyEnabled = properties.get(Constants.PROXY_ENABLE); 43 | DefaultApi defaultApi; 44 | 45 | if (Boolean.parseBoolean(isProxyEnabled)) { 46 | defaultApi = Feign.builder().client(AuthProxyUtils.getClient(properties)) 47 | .encoder(new GsonEncoder()) 48 | .decoder(new GsonDecoder()) 49 | .logger(new Slf4jLogger()) 50 | .requestInterceptor(requestTemplate -> requestTemplate.header(AUTH_HEADER, "Bearer " + token)) 51 | .target(DefaultApi.class, authEndpoint); 52 | } else { 53 | defaultApi = Feign.builder() 54 | .encoder(new GsonEncoder()) 55 | .decoder(new GsonDecoder()) 56 | .logger(new Slf4jLogger()) 57 | .requestInterceptor(requestTemplate -> requestTemplate.header(AUTH_HEADER, "Bearer " + token)) 58 | .target(DefaultApi.class, authEndpoint); 59 | } 60 | try { 61 | TokenDetailsDTO dto = defaultApi.tokenGet(); 62 | return dto.getToken(); 63 | } catch (FeignException.Unauthorized e) { 64 | throw new ConnectionUnrecoverableException( 65 | "Invalid/expired user token. Please update apim.analytics" 66 | + ".auth_token in configuration and restart the instance", e); 67 | } catch (RetryableException e) { 68 | throw new ConnectionRecoverableException("Provided authentication endpoint " + authEndpoint + " is not " 69 | + "reachable."); 70 | } catch (IllegalArgumentException e) { 71 | throw new ConnectionUnrecoverableException("Invalid apim.analytics configurations provided. Please update " 72 | + "configurations and restart the instance."); 73 | } catch (FeignException.Forbidden e) { 74 | throw new ConnectionRecoverableException("Publisher has been temporarily revoked."); 75 | } catch (Exception e) { 76 | //we will retry for any other exception 77 | throw new ConnectionRecoverableException("Exception " + e.getClass() + " occurred."); 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/moesif/EventQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.moesif; 19 | 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | import org.wso2.am.analytics.publisher.client.AbstractMoesifClient; 23 | import org.wso2.am.analytics.publisher.client.MoesifClient; 24 | import org.wso2.am.analytics.publisher.client.SimpleMoesifClient; 25 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 26 | import org.wso2.am.analytics.publisher.reporter.cloud.DefaultAnalyticsThreadFactory; 27 | import org.wso2.am.analytics.publisher.retriever.MoesifKeyRetriever; 28 | 29 | import java.util.concurrent.BlockingQueue; 30 | import java.util.concurrent.ExecutorService; 31 | import java.util.concurrent.Executors; 32 | import java.util.concurrent.LinkedBlockingQueue; 33 | import java.util.concurrent.RejectedExecutionException; 34 | import java.util.concurrent.atomic.AtomicInteger; 35 | 36 | /** 37 | * Bounded concurrent queue wrapping for Moesif reporter{@link java.util.concurrent.ArrayBlockingQueue}. 38 | */ 39 | public class EventQueue { 40 | private static final Logger log = LogManager.getLogger(EventQueue.class); 41 | private final BlockingQueue eventQueue; 42 | private final ExecutorService publisherExecutorService; 43 | private final AtomicInteger failureCount; 44 | 45 | public EventQueue(int queueSize, int workerThreadCount, String key) { 46 | this(queueSize, workerThreadCount, new SimpleMoesifClient(key)); 47 | } 48 | 49 | public EventQueue(int queueSize, int workerThreadCount, String key, String baseUrl) { 50 | this(queueSize, workerThreadCount, new SimpleMoesifClient(key, baseUrl)); 51 | } 52 | 53 | public EventQueue(int queueSize, int workerThreadCount, MoesifKeyRetriever moesifKeyRetriever) { 54 | this(queueSize, workerThreadCount, new MoesifClient(moesifKeyRetriever)); 55 | } 56 | 57 | public EventQueue(int queueSize, int workerThreadCount, AbstractMoesifClient moesifClient) { 58 | publisherExecutorService = Executors.newFixedThreadPool(workerThreadCount, 59 | new DefaultAnalyticsThreadFactory("Queue-Worker")); 60 | eventQueue = new LinkedBlockingQueue<>(queueSize); 61 | failureCount = new AtomicInteger(0); 62 | for (int i = 0; i < workerThreadCount; i++) { 63 | publisherExecutorService.submit(new ParallelQueueWorker(eventQueue, moesifClient)); 64 | } 65 | } 66 | 67 | public void put(MetricEventBuilder builder) { 68 | try { 69 | if (!eventQueue.offer(builder)) { 70 | int count = failureCount.incrementAndGet(); 71 | if (count == 1) { 72 | log.error("Event queue is full. Starting to drop analytics events."); 73 | } else if (count % 1000 == 0) { 74 | log.error("Event queue is full. {} events dropped so far", count); 75 | } 76 | } 77 | } catch (RejectedExecutionException e) { 78 | log.warn("Task submission failed. Task queue might be full", e); 79 | } 80 | } 81 | 82 | @Override 83 | protected void finalize() throws Throwable { 84 | publisherExecutorService.shutdown(); 85 | super.finalize(); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/client/AbstractMoesifClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2025, WSO2 LLC. (https://www.wso2.com). 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.client; 19 | 20 | import com.moesif.api.models.EventModel; 21 | 22 | import org.apache.logging.log4j.LogManager; 23 | import org.apache.logging.log4j.Logger; 24 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 25 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 26 | import org.wso2.am.analytics.publisher.util.Constants; 27 | 28 | import java.io.IOException; 29 | import java.util.ArrayList; 30 | import java.util.List; 31 | import java.util.Map; 32 | 33 | /** 34 | * Abstract class representing a Moesif client for publishing events and building event responses. 35 | */ 36 | public abstract class AbstractMoesifClient { 37 | protected final Logger log = LogManager.getLogger(AbstractMoesifClient.class); 38 | 39 | /** 40 | * Publish method is responsible for publishing events to Moesif. 41 | */ 42 | public abstract void publish(MetricEventBuilder builder) throws MetricReportingException; 43 | public abstract void publishBatch(List builders); 44 | 45 | public abstract EventModel buildEventResponse(Map data) 46 | throws IOException, MetricReportingException; 47 | 48 | protected static void populateHeaders(Map data, Map reqHeaders, 49 | Map rspHeaders) { 50 | reqHeaders.put(Constants.MOESIF_USER_AGENT_KEY, 51 | (String) data.getOrDefault(Constants.USER_AGENT_HEADER, Constants.UNKNOWN_VALUE)); 52 | reqHeaders.put(Constants.MOESIF_CONTENT_TYPE_KEY, Constants.MOESIF_CONTENT_TYPE_HEADER); 53 | Map properties = (Map) data.get(Constants.PROPERTIES); 54 | if (properties == null) { 55 | return; 56 | } 57 | if (properties.containsKey(Constants.REQUEST_HEADERS) && properties.get(Constants.REQUEST_HEADERS) != null) { 58 | Map headers = (Map) properties.get(Constants.REQUEST_HEADERS); 59 | reqHeaders.putAll(headers); 60 | } 61 | 62 | rspHeaders.put(Constants.VARY_HEADER, Constants.ACCEPT_ENCODING_VALUE); 63 | rspHeaders.put(Constants.PRAGMA_HEADER, Constants.NO_CACHE_VALUE); 64 | rspHeaders.put(Constants.EXPIRES_HEADER, Constants.EXPIRES_VALUE); 65 | rspHeaders.put(Constants.MOESIF_CONTENT_TYPE_KEY, Constants.APPLICATION_JSON_UTF8_VALUE); 66 | rspHeaders.put(Constants.CACHE_CONTROL_HEADER, Constants.NO_CACHE_VALUE); 67 | 68 | if (properties.containsKey(Constants.RESPONSE_HEADERS) && properties.get(Constants.RESPONSE_HEADERS) != null) { 69 | Map headers = (Map) properties.get(Constants.RESPONSE_HEADERS); 70 | rspHeaders.putAll(headers); 71 | } 72 | } 73 | 74 | protected List buildEventsFromBuilders(List builders) { 75 | List events = new ArrayList<>(); 76 | for (MetricEventBuilder builder : builders) { 77 | try { 78 | Map event = builder.build(); 79 | events.add(this.buildEventResponse(event)); 80 | } catch (MetricReportingException | IOException e) { 81 | log.error("Builder instance is not duly filled. Event building failed", e); 82 | } 83 | } 84 | return events; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /component/publisher-client/src/test/java/org/wso2/am/analytics/publisher/AuthAPIClientTestCase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher; 20 | 21 | import org.testng.Assert; 22 | import org.testng.annotations.Test; 23 | import org.wso2.am.analytics.publisher.auth.AuthClient; 24 | import org.wso2.am.analytics.publisher.exception.ConnectionRecoverableException; 25 | import org.wso2.am.analytics.publisher.exception.ConnectionUnrecoverableException; 26 | import org.wso2.am.analytics.publisher.util.AuthAPIMockService; 27 | 28 | import java.util.HashMap; 29 | import java.util.UUID; 30 | 31 | public class AuthAPIClientTestCase extends AuthAPIMockService { 32 | 33 | @Test(expectedExceptions = { ConnectionUnrecoverableException.class }, 34 | expectedExceptionsMessageRegExp = "Invalid/expired user token.*") 35 | public void testAuthClientWithAInvalidToken() throws Exception { 36 | 37 | String authToken = UUID.randomUUID().toString(); 38 | mock(401, authToken); 39 | 40 | AuthClient.getSASToken(authApiEndpoint, authToken, new HashMap<>()); 41 | } 42 | 43 | @Test 44 | public void testAuthClientWithAValidToken() throws Exception { 45 | 46 | String authToken = UUID.randomUUID().toString(); 47 | mock(200, authToken); 48 | 49 | String sasToken = AuthClient.getSASToken(authApiEndpoint, authToken, new HashMap<>()); 50 | Assert.assertEquals(sasToken, SAS_TOKEN); 51 | } 52 | 53 | @Test(expectedExceptions = { ConnectionRecoverableException.class }) 54 | public void testAuthClientWithFor500Response() throws Exception { 55 | 56 | String authToken = UUID.randomUUID().toString(); 57 | mock(500, authToken); 58 | 59 | AuthClient.getSASToken(authApiEndpoint, authToken, new HashMap<>()); 60 | } 61 | 62 | @Test(expectedExceptions = { ConnectionRecoverableException.class }) 63 | public void testAuthClientWithFor400Response() throws Exception { 64 | 65 | String authToken = UUID.randomUUID().toString(); 66 | mock(400, authToken); 67 | 68 | AuthClient.getSASToken(authApiEndpoint, authToken, new HashMap<>()); 69 | } 70 | 71 | @Test(expectedExceptions = { ConnectionRecoverableException.class }, 72 | expectedExceptionsMessageRegExp = "Publisher has been temporarily revoked.") 73 | public void testAuthClientWithFor403Response() throws Exception { 74 | 75 | String authToken = UUID.randomUUID().toString(); 76 | mock(403, authToken); 77 | 78 | AuthClient.getSASToken(authApiEndpoint, authToken, new HashMap<>()); 79 | } 80 | 81 | @Test(expectedExceptions = { ConnectionUnrecoverableException.class }, 82 | expectedExceptionsMessageRegExp = "Invalid apim.analytics configurations provided.*") 83 | public void testAuthClientWithForInvalidAuthUrl() throws Exception { 84 | 85 | String authToken = UUID.randomUUID().toString(); 86 | mock(200, authToken); 87 | 88 | String authEndpoint = "invalid/host/auth-api"; 89 | AuthClient.getSASToken(authEndpoint, authToken, new HashMap<>()); 90 | } 91 | 92 | @Test(expectedExceptions = { ConnectionRecoverableException.class }, 93 | expectedExceptionsMessageRegExp = "Provided authentication endpoint.*") 94 | public void testAuthClientWithForNonExistAuthUrl() throws Exception { 95 | 96 | String authToken = UUID.randomUUID().toString(); 97 | mock(200, authToken); 98 | 99 | String authEndpoint = "https://no.such.host/auth-api"; 100 | AuthClient.getSASToken(authEndpoint, authToken, new HashMap<>()); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/DefaultCounterMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.cloud; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.wso2.am.analytics.publisher.client.ClientStatus; 24 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 25 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 26 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 27 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 28 | 29 | import java.util.concurrent.atomic.AtomicInteger; 30 | 31 | /** 32 | * Implementation of {@link CounterMetric} for Choroe Metric Reporter. 33 | */ 34 | public class DefaultCounterMetric implements CounterMetric { 35 | private static final Logger log = LogManager.getLogger(DefaultCounterMetric.class); 36 | private String name; 37 | private EventQueue queue; 38 | private MetricSchema schema; 39 | private ClientStatus status; 40 | private final AtomicInteger failureCount; 41 | 42 | public DefaultCounterMetric(String name, EventQueue queue, MetricSchema schema) throws MetricCreationException { 43 | //Constructor should be made protected. Keeping public till testing plan is finalized 44 | this.name = name; 45 | this.queue = queue; 46 | if (schema == MetricSchema.ERROR || schema == MetricSchema.RESPONSE 47 | || schema == MetricSchema.CHOREO_ERROR || schema == MetricSchema.CHOREO_RESPONSE) { 48 | this.schema = schema; 49 | } else { 50 | throw new MetricCreationException("Default Counter Metric only supports " + MetricSchema.RESPONSE + ", " 51 | + ", " + MetricSchema.ERROR + ", " + MetricSchema.CHOREO_RESPONSE + " and " 52 | + MetricSchema.CHOREO_RESPONSE + " types."); 53 | } 54 | this.status = queue.getClient().getStatus(); 55 | this.failureCount = new AtomicInteger(0); 56 | } 57 | 58 | @Override 59 | public String getName() { 60 | return name; 61 | } 62 | 63 | @Override public MetricSchema getSchema() { 64 | return schema; 65 | } 66 | 67 | @Override 68 | public int incrementCount(MetricEventBuilder builder) { 69 | if (!(status == ClientStatus.NOT_CONNECTED)) { 70 | queue.put(builder); 71 | } else { 72 | if (failureCount.incrementAndGet() % 1000 == 0) { 73 | log.error("Eventhub client is not connected. " + failureCount.incrementAndGet() + " events dropped so " 74 | + "far. Please correct your configuration and restart the instance."); 75 | } 76 | } 77 | return 0; 78 | } 79 | 80 | /** 81 | * Returns Event Builder used for this CounterMetric. Depending on the schema different types of builders will be 82 | * returned. 83 | * 84 | * @return {@link MetricEventBuilder} for this {@link CounterMetric} 85 | */ 86 | @Override 87 | public MetricEventBuilder getEventBuilder() { 88 | switch (schema) { 89 | case RESPONSE: 90 | return new DefaultResponseMetricEventBuilder(); 91 | case ERROR: 92 | return new DefaultFaultMetricEventBuilder(); 93 | case CHOREO_RESPONSE: 94 | return new DefaultChoreoResponseMetricEventBuilder(); 95 | case CHOREO_ERROR: 96 | return new DefaultChoreoFaultMetricEventBuilder(); 97 | default: 98 | // will not happen 99 | return null; 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /component/publisher-client/src/test/java/org/wso2/am/analytics/publisher/LogMetricReporterTestCase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.apache.logging.log4j.core.LoggerContext; 24 | import org.apache.logging.log4j.core.config.Configuration; 25 | import org.testng.Assert; 26 | import org.testng.annotations.Test; 27 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 28 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 29 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 30 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 31 | import org.wso2.am.analytics.publisher.reporter.MetricReporter; 32 | import org.wso2.am.analytics.publisher.reporter.MetricReporterFactory; 33 | import org.wso2.am.analytics.publisher.reporter.log.LogCounterMetric; 34 | import org.wso2.am.analytics.publisher.util.TestUtils; 35 | import org.wso2.am.analytics.publisher.util.UnitTestAppender; 36 | 37 | import java.util.List; 38 | 39 | public class LogMetricReporterTestCase { 40 | private static final Logger log = LogManager.getLogger(LogMetricReporterTestCase.class); 41 | 42 | 43 | @Test 44 | public void testLogMetricReporter() throws MetricCreationException, MetricReportingException { 45 | log.info("Running log metric test case"); 46 | Logger log = LogManager.getLogger(LogCounterMetric.class); 47 | LoggerContext context = LoggerContext.getContext(false); 48 | Configuration config = context.getConfiguration(); 49 | UnitTestAppender appender = config.getAppender("UnitTestAppender"); 50 | 51 | MetricReporter metricReporter = MetricReporterFactory.getInstance().createMetricReporter( 52 | "org.wso2.am.analytics.publisher.reporter.log.LogMetricReporter", null); 53 | CounterMetric metric = metricReporter.createCounterMetric("testCounter", null); 54 | MetricEventBuilder builder = metric.getEventBuilder(); 55 | builder.addAttribute("attribute1", "value1").addAttribute("attribute2", "value2").addAttribute("attribute3", 56 | "value3"); 57 | metric.incrementCount(builder); 58 | 59 | List messages = appender.getMessages(); 60 | 61 | Assert.assertTrue(TestUtils.isContains(messages, "testCounter"), "Metric name is not properly logged"); 62 | Assert.assertTrue(TestUtils.isContains(messages, "attribute1=value1"), "Metric attribute is not properly " 63 | + "logged"); 64 | Assert.assertTrue(TestUtils.isContains(messages, "attribute2=value2"), 65 | "Metric attribute is not properly logged"); 66 | Assert.assertTrue(TestUtils.isContains(messages, "attribute3=value3"), 67 | "Metric attribute is not properly logged"); 68 | } 69 | 70 | @Test(expectedExceptions = MetricReportingException.class, dependsOnMethods = {"testLogMetricReporter"}) 71 | public void testLogMetricReporterWithInvalidAttributes() throws MetricCreationException, MetricReportingException { 72 | MetricReporter metricReporter = MetricReporterFactory.getInstance().createMetricReporter( 73 | "org.wso2.am.analytics.publisher.reporter.log.LogMetricReporter", null); 74 | CounterMetric metric = metricReporter.createCounterMetric("testCounter", null); 75 | MetricEventBuilder builder = metric.getEventBuilder(); 76 | builder.addAttribute("attribute1", 123).addAttribute("attribute2", "value2").addAttribute("attribute3", 77 | "value3"); 78 | metric.incrementCount(builder); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/EventQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.cloud; 19 | 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | import org.wso2.am.analytics.publisher.client.EventHubClient; 23 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 24 | 25 | import java.util.concurrent.BlockingQueue; 26 | import java.util.concurrent.ExecutorService; 27 | import java.util.concurrent.Executors; 28 | import java.util.concurrent.LinkedBlockingQueue; 29 | import java.util.concurrent.RejectedExecutionException; 30 | import java.util.concurrent.ScheduledExecutorService; 31 | import java.util.concurrent.TimeUnit; 32 | import java.util.concurrent.atomic.AtomicInteger; 33 | 34 | /** 35 | * Bounded concurrent queue wrapping {@link java.util.concurrent.ArrayBlockingQueue}. 36 | */ 37 | public class EventQueue { 38 | 39 | private static final Logger log = LogManager.getLogger(EventQueue.class); 40 | private final BlockingQueue eventQueue; 41 | private final ExecutorService publisherExecutorService; 42 | private final EventHubClient client; 43 | private final AtomicInteger failureCount; 44 | private final ScheduledExecutorService flushingExecutorService; 45 | 46 | public EventQueue(int queueSize, int workerThreadCount, EventHubClient client, int flushingDelay) { 47 | this.client = client; 48 | // Note : Using a fixed worker thread pool and a bounded queue to control the load on the server 49 | publisherExecutorService = Executors.newFixedThreadPool(workerThreadCount, 50 | new DefaultAnalyticsThreadFactory("Queue-Worker")); 51 | flushingExecutorService = Executors.newScheduledThreadPool(workerThreadCount, 52 | new DefaultAnalyticsThreadFactory("Queue-Flusher")); 53 | eventQueue = new LinkedBlockingQueue<>(queueSize); 54 | failureCount = new AtomicInteger(0); 55 | for (int i = 0; i < workerThreadCount; i++) { 56 | if (i == 0) { 57 | publisherExecutorService.submit(new ParallelQueueWorker(eventQueue, client)); 58 | flushingExecutorService.scheduleWithFixedDelay(new QueueFlusher(eventQueue, client), flushingDelay, 59 | flushingDelay, TimeUnit.SECONDS); 60 | } else { 61 | EventHubClient clonedClient = client.clone(); 62 | publisherExecutorService.submit(new ParallelQueueWorker(eventQueue, clonedClient)); 63 | flushingExecutorService.scheduleWithFixedDelay(new QueueFlusher(eventQueue, clonedClient), 64 | flushingDelay, flushingDelay, TimeUnit.SECONDS); 65 | } 66 | } 67 | } 68 | 69 | public void put(MetricEventBuilder builder) { 70 | try { 71 | if (!eventQueue.offer(builder)) { 72 | int count = failureCount.incrementAndGet(); 73 | if (count == 1) { 74 | log.error("Event queue is full. Starting to drop analytics events."); 75 | } else if (count % 1000 == 0) { 76 | log.error("Event queue is full. " + count + " events dropped so far"); 77 | } 78 | } 79 | } catch (RejectedExecutionException e) { 80 | log.warn("Task submission failed. Task queue might be full", e); 81 | } 82 | 83 | } 84 | 85 | @Override 86 | protected void finalize() throws Throwable { 87 | publisherExecutorService.shutdown(); 88 | super.finalize(); 89 | } 90 | 91 | protected EventHubClient getClient() { 92 | return this.client; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/moesif/MoesifReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.moesif; 19 | 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 23 | import org.wso2.am.analytics.publisher.reporter.AbstractMetricReporter; 24 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 25 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 26 | import org.wso2.am.analytics.publisher.reporter.TimerMetric; 27 | import org.wso2.am.analytics.publisher.reporter.moesif.util.MoesifMicroserviceConstants; 28 | import org.wso2.am.analytics.publisher.retriever.MoesifKeyRetriever; 29 | import org.wso2.am.analytics.publisher.util.Constants; 30 | 31 | import java.util.Map; 32 | import java.util.Timer; 33 | 34 | /** 35 | * Moesif Metric Reporter Implementation. This implementation is responsible for sending analytics data into Moesif 36 | * dashboard in a secure and reliable way. 37 | */ 38 | public class MoesifReporter extends AbstractMetricReporter { 39 | private static final Logger log = LogManager.getLogger(MoesifReporter.class); 40 | private final EventQueue eventQueue; 41 | 42 | public MoesifReporter(Map properties) throws MetricCreationException { 43 | super(properties); 44 | int queueSize = Constants.DEFAULT_QUEUE_SIZE; 45 | int workerThreads = Constants.DEFAULT_WORKER_THREADS; 46 | if (properties.get(Constants.QUEUE_SIZE) != null) { 47 | queueSize = Integer.parseInt(properties.get(Constants.QUEUE_SIZE)); 48 | } 49 | if (properties.get(Constants.WORKER_THREAD_COUNT) != null) { 50 | workerThreads = Integer.parseInt(properties.get(Constants.WORKER_THREAD_COUNT)); 51 | } 52 | if (properties.get(Constants.TYPE).contains(Constants.MOESIF)) { 53 | String moesifKey = properties.get(Constants.MOESIF_KEY); 54 | String moesifBasePath = properties.get(Constants.MOESIF_BASE_URL); 55 | if (moesifBasePath == null || moesifBasePath.isEmpty()) { 56 | this.eventQueue = new EventQueue(queueSize, workerThreads, moesifKey); 57 | } else { 58 | this.eventQueue = new EventQueue(queueSize, workerThreads, moesifKey, moesifBasePath); 59 | } 60 | } else { 61 | String moesifBasePath = properties.get( 62 | MoesifMicroserviceConstants.MOESIF_PROTOCOL_WITH_FQDN_KEY) + properties.get( 63 | MoesifMicroserviceConstants.MOESIF_MS_VERSIONING_KEY); 64 | MoesifKeyRetriever keyRetriever = MoesifKeyRetriever.getInstance( 65 | properties.get(MoesifMicroserviceConstants.MS_USERNAME_CONFIG_KEY), 66 | properties.get(MoesifMicroserviceConstants.MS_PWD_CONFIG_KEY), moesifBasePath); 67 | 68 | this.eventQueue = new EventQueue(queueSize, workerThreads, keyRetriever); 69 | 70 | MissedEventHandler missedEventHandler = new MissedEventHandler(keyRetriever); 71 | // execute MissedEventHandler periodically. 72 | Timer timer = new Timer(); 73 | timer.schedule(missedEventHandler, 0, MoesifMicroserviceConstants.PERIODIC_CALL_DELAY); 74 | } 75 | } 76 | 77 | @Override 78 | protected void validateConfigProperties(Map map) throws MetricCreationException { 79 | 80 | } 81 | 82 | @Override 83 | public CounterMetric createCounter(String name, MetricSchema metricSchema) throws MetricCreationException { 84 | MoesifCounterMetric counterMetric = new MoesifCounterMetric(name, eventQueue, metricSchema); 85 | return counterMetric; 86 | } 87 | 88 | @Override 89 | protected TimerMetric createTimer(String s) { 90 | return null; 91 | } 92 | } 93 | 94 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/AbstractMetricReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 24 | 25 | import java.util.HashMap; 26 | import java.util.Map; 27 | 28 | /** 29 | * Implementation of {@link MetricReporter}. All concrete implementations should extend this class. Validations and 30 | * metric type creation is enforced using this abstract class 31 | */ 32 | public abstract class AbstractMetricReporter implements MetricReporter { 33 | private static final Logger log = LogManager.getLogger(AbstractMetricReporter.class); 34 | private final Map properties; 35 | private Map metricRegistry; 36 | 37 | protected AbstractMetricReporter(Map properties) throws MetricCreationException { 38 | this.properties = properties; 39 | metricRegistry = new HashMap<>(); 40 | validateConfigProperties(properties); 41 | } 42 | 43 | /** 44 | * Method to validate the configuration properties. Config properties are accepted as a map to increase 45 | * extendability. Hence this method is responsible to sanitize the map 46 | * 47 | * @param properties Configuration properties needed by the implementation 48 | * @throws MetricCreationException Exception will be throw is any of the required fields are missing or no in the 49 | * expected format 50 | */ 51 | protected abstract void validateConfigProperties(Map properties) throws MetricCreationException; 52 | 53 | public Map getConfiguration() { 54 | return properties; 55 | } 56 | 57 | @Override 58 | public CounterMetric createCounterMetric(String name, MetricSchema schema) throws MetricCreationException { 59 | Metric metric = metricRegistry.get(name); 60 | if (metric == null) { 61 | synchronized (this) { 62 | if (metricRegistry.get(name) == null) { 63 | metric = createCounter(name, schema); 64 | metricRegistry.put(name, metric); 65 | } else { 66 | metric = metricRegistry.get(name); 67 | } 68 | } 69 | } else if (!(metric instanceof CounterMetric)) { 70 | throw new MetricCreationException("Timer Metric with the same name already exists. Please use a different" 71 | + " name"); 72 | } else if (metric.getSchema() != schema) { 73 | throw new MetricCreationException("Counter Metric with the same name but different schema already exists." 74 | + " Please use a different name"); 75 | } 76 | return (CounterMetric) metric; 77 | } 78 | 79 | protected abstract CounterMetric createCounter(String name, MetricSchema schema) throws MetricCreationException; 80 | 81 | @Override 82 | public TimerMetric createTimerMetric(String name) { 83 | Metric metric = metricRegistry.get(name); 84 | if (metric == null) { 85 | synchronized (this) { 86 | if (metricRegistry.get(name) == null) { 87 | metric = createTimer(name); 88 | metricRegistry.put(name, metric); 89 | } else { 90 | metric = metricRegistry.get(name); 91 | } 92 | } 93 | } 94 | if (!(metric instanceof TimerMetric)) { 95 | log.error("Counter Metric with the same name already exists. Please use a different name"); 96 | return null; 97 | } else { 98 | return (TimerMetric) metric; 99 | } 100 | } 101 | 102 | protected abstract TimerMetric createTimer(String name); 103 | 104 | } 105 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/client/WSO2TokenCredential.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.client; 20 | 21 | import com.azure.core.credential.AccessToken; 22 | import com.azure.core.credential.TokenCredential; 23 | import com.azure.core.credential.TokenRequestContext; 24 | import org.apache.logging.log4j.LogManager; 25 | import org.apache.logging.log4j.Logger; 26 | import org.wso2.am.analytics.publisher.auth.AuthClient; 27 | import org.wso2.am.analytics.publisher.exception.ConnectionRecoverableException; 28 | import org.wso2.am.analytics.publisher.exception.ConnectionUnrecoverableException; 29 | import org.wso2.am.analytics.publisher.util.BackoffRetryCounter; 30 | import reactor.core.publisher.Mono; 31 | 32 | import java.time.Instant; 33 | import java.time.OffsetDateTime; 34 | import java.time.ZoneOffset; 35 | import java.util.Arrays; 36 | import java.util.Map; 37 | 38 | /** 39 | * WSO2 SAS token refresh implementation for TokenCredential. 40 | */ 41 | class WSO2TokenCredential implements TokenCredential { 42 | private static final Logger log = LogManager.getLogger(WSO2TokenCredential.class); 43 | private final String authEndpoint; 44 | private final String authToken; 45 | private final Map properties; 46 | private BackoffRetryCounter backoffRetryCounter; 47 | 48 | public WSO2TokenCredential(String authEndpoint, String authToken, Map properties) { 49 | this.authEndpoint = authEndpoint; 50 | this.authToken = authToken; 51 | this.properties = properties; 52 | backoffRetryCounter = new BackoffRetryCounter(); 53 | } 54 | 55 | @Override 56 | public Mono getToken(TokenRequestContext tokenRequestContext) { 57 | log.debug("Trying to retrieving a new SAS token."); 58 | try { 59 | String sasToken = AuthClient.getSASToken(this.authEndpoint, this.authToken, this.properties); 60 | backoffRetryCounter.reset(); 61 | log.debug("New SAS token retrieved."); 62 | // Using lower duration than actual. 63 | OffsetDateTime time = getExpirationTime(sasToken); 64 | return Mono.fromCallable(() -> new AccessToken(sasToken, time)); 65 | } catch (ConnectionRecoverableException e) { 66 | log.error("Error occurred when retrieving SAS token. Connection will be retried in " 67 | + backoffRetryCounter.getTimeInterval().replaceAll("[\r\n]", ""), e); 68 | try { 69 | Thread.sleep(backoffRetryCounter.getTimeIntervalMillis()); 70 | } catch (InterruptedException interruptedException) { 71 | Thread.currentThread().interrupt(); 72 | } 73 | backoffRetryCounter.increment(); 74 | return getToken(tokenRequestContext); 75 | } catch (ConnectionUnrecoverableException e) { 76 | //Do not pass the exception. Publishing threads will encounter authentication issue and then try to 77 | // reinitialize publisher. 78 | log.error("Error occurred when retrieving SAS token.", e); 79 | backoffRetryCounter.reset(); 80 | return Mono.error(e); 81 | } 82 | } 83 | 84 | private OffsetDateTime getExpirationTime(String sharedAccessSignature) { 85 | String[] parts = sharedAccessSignature.split("&"); 86 | return Arrays.stream(parts).map(part -> part.split("=")) 87 | .filter(pair -> pair.length == 2 && pair[0].equalsIgnoreCase("se")) 88 | .findFirst().map(pair -> pair[1]) 89 | .map((expirationTimeStr) -> { 90 | try { 91 | long epochSeconds = Long.parseLong(expirationTimeStr); 92 | return Instant.ofEpochSecond(epochSeconds).atOffset(ZoneOffset.UTC); 93 | } catch (NumberFormatException e) { 94 | log.error("Invalid expiration time format in the SAS token.", e); 95 | return OffsetDateTime.MAX; 96 | } 97 | }) 98 | .orElse(OffsetDateTime.MAX); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/moesif/ParallelQueueWorker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 LLC. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | package org.wso2.am.analytics.publisher.reporter.moesif; 19 | 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | 23 | import org.wso2.am.analytics.publisher.client.AbstractMoesifClient; 24 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 25 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 26 | 27 | import java.util.ArrayList; 28 | import java.util.List; 29 | import java.util.concurrent.BlockingQueue; 30 | 31 | /** 32 | * Will dequeue the events from queues and send then to the moesif client. 33 | */ 34 | public class ParallelQueueWorker implements Runnable { 35 | private static final Logger log = LogManager.getLogger(ParallelQueueWorker.class); 36 | private BlockingQueue eventQueue; 37 | private AbstractMoesifClient client; 38 | private final int batchSize = 50; 39 | private final long batchTimeoutMs = 2000; 40 | 41 | public ParallelQueueWorker(BlockingQueue queue, AbstractMoesifClient moesifClient) { 42 | this.eventQueue = queue; 43 | this.client = moesifClient; 44 | } 45 | 46 | /** 47 | * Continuously runs a worker thread that processes analytics events in batches from a blocking queue. 48 | *

49 | * The method polls the {@code eventQueue} at fixed intervals (100ms) for new {@code MetricEventBuilder} events. 50 | * Events are collected into a batch and processed when either: 51 | *

55 | * Once either condition is met, the collected batch is sent using {@code processBatch}, and the timer is reset. 56 | *

57 | * If the thread is interrupted, it exits gracefully by setting the interrupt flag. Any other exception 58 | * during processing is logged, and the affected events will be dropped. 59 | * 60 | * This method is intended to be executed in a dedicated thread via an {@code ExecutorService} or similar mechanism. 61 | */ 62 | 63 | public void run() { 64 | List batch = new ArrayList<>(batchSize); 65 | long lastBatchTime = System.currentTimeMillis(); 66 | while (true) { 67 | MetricEventBuilder eventBuilder; 68 | try { 69 | 70 | eventBuilder = eventQueue.poll(100, java.util.concurrent.TimeUnit.MILLISECONDS); 71 | if (eventBuilder != null) { 72 | batch.add(eventBuilder); 73 | } 74 | 75 | long currentBatchTime = System.currentTimeMillis(); 76 | boolean shouldSendBatch = batch.size() >= batchSize || 77 | (currentBatchTime - lastBatchTime) >= batchTimeoutMs; 78 | 79 | if (shouldSendBatch) { 80 | if (!batch.isEmpty()) { 81 | log.debug("Sending batch of {} events", batch.size()); 82 | processBatch(new ArrayList<>(batch)); 83 | batch.clear(); 84 | } 85 | lastBatchTime = currentBatchTime; 86 | } 87 | } catch (InterruptedException e) { 88 | Thread.currentThread().interrupt(); 89 | } catch (Exception e) { 90 | log.error("Analytics event sending failed. Event will be dropped", e); 91 | } 92 | } 93 | } 94 | private void processBatch(List batch) { 95 | try { 96 | if (batch.size() == 1) { 97 | client.publish(batch.get(0)); 98 | } else { 99 | client.publishBatch(batch); 100 | } 101 | log.debug("Successfully processed batch of {} events", batch.size()); 102 | } catch (MetricReportingException e) { 103 | log.error("Failed to process batch of {} events", batch.size(), e); 104 | } catch (Exception e) { 105 | log.error("Unexpected error processing batch", e); 106 | } 107 | } 108 | 109 | 110 | 111 | } 112 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/cloud/DefaultFaultMetricEventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.cloud; 20 | 21 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 22 | import org.wso2.am.analytics.publisher.properties.AIMetadata; 23 | import org.wso2.am.analytics.publisher.properties.AITokenUsage; 24 | import org.wso2.am.analytics.publisher.properties.Properties; 25 | import org.wso2.am.analytics.publisher.reporter.AbstractMetricEventBuilder; 26 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 27 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 28 | import org.wso2.am.analytics.publisher.util.Constants; 29 | 30 | import java.util.HashMap; 31 | import java.util.Map; 32 | 33 | /** 34 | * Builder class for fault events. 35 | */ 36 | public class DefaultFaultMetricEventBuilder extends AbstractMetricEventBuilder { 37 | protected final Map requiredAttributes; 38 | protected final Map eventMap; 39 | 40 | public DefaultFaultMetricEventBuilder() { 41 | requiredAttributes = DefaultInputValidator.getInstance().getEventProperties(MetricSchema.ERROR); 42 | eventMap = new HashMap<>(); 43 | } 44 | 45 | protected DefaultFaultMetricEventBuilder(Map requiredAttributes) { 46 | this.requiredAttributes = requiredAttributes; 47 | eventMap = new HashMap<>(); 48 | } 49 | 50 | @Override 51 | public boolean validate() throws MetricReportingException { 52 | Map propertyMap = (Map) eventMap.remove(Constants.PROPERTIES); 53 | if (propertyMap != null) { 54 | extractPropertyObject(propertyMap); 55 | } 56 | for (Map.Entry entry : requiredAttributes.entrySet()) { 57 | Object attribute = eventMap.get(entry.getKey()); 58 | if (attribute == null) { 59 | throw new MetricReportingException(entry.getKey() + " is missing in metric data. This metric event " 60 | + "will not be processed further."); 61 | } else if (!attribute.getClass().equals(entry.getValue())) { 62 | throw new MetricReportingException(entry.getKey() + " is expecting a " + entry.getValue() + " type " 63 | + "attribute while attribute of type " + attribute.getClass() 64 | + " is present"); 65 | } 66 | } 67 | return true; 68 | } 69 | 70 | private void extractPropertyObject(Map properties) { 71 | Properties propertyObject = new Properties(); 72 | if (properties.get(Constants.AI_METADATA) != null) { 73 | Map aiMetadata = (Map) properties.remove(Constants.AI_METADATA); 74 | propertyObject.setAiMetadata(new AIMetadata(aiMetadata)); 75 | } 76 | if (properties.get(Constants.AI_TOKEN_USAGE) != null) { 77 | Map aiTokenUsage = (Map) properties.remove(Constants.AI_TOKEN_USAGE); 78 | propertyObject.setAiTokenUsage(new AITokenUsage(aiTokenUsage)); 79 | } 80 | if (properties.get(Constants.IS_EGRESS) != null) { 81 | boolean isEgress = (boolean) properties.remove(Constants.IS_EGRESS); 82 | propertyObject.setEgress(isEgress); 83 | } 84 | if (properties.get(Constants.SUBTYPE) != null) { 85 | String subType = (String) properties.remove(Constants.SUBTYPE); 86 | propertyObject.setSubType(subType); 87 | } 88 | eventMap.put(Constants.PROPERTIES, propertyObject); 89 | } 90 | 91 | @Override 92 | public MetricEventBuilder addAttribute(String key, Object value) throws MetricReportingException { 93 | //all validation is moved to validate method to reduce analytics data processing latency 94 | eventMap.put(key, value); 95 | return this; 96 | } 97 | 98 | @Override 99 | protected Map buildEvent() { 100 | eventMap.put(Constants.EVENT_TYPE, Constants.FAULT_EVENT_TYPE); 101 | return eventMap; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /component/publisher-client/src/test/java/org/wso2/am/analytics/publisher/ErrorHandlingTestCase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher; 20 | 21 | import org.apache.logging.log4j.Level; 22 | import org.apache.logging.log4j.LogManager; 23 | import org.apache.logging.log4j.Logger; 24 | import org.apache.logging.log4j.core.LoggerContext; 25 | import org.apache.logging.log4j.core.config.Configuration; 26 | import org.testng.Assert; 27 | import org.testng.annotations.Test; 28 | import org.wso2.am.analytics.publisher.client.EventHubClient; 29 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 30 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 31 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 32 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 33 | import org.wso2.am.analytics.publisher.reporter.MetricReporter; 34 | import org.wso2.am.analytics.publisher.reporter.MetricReporterFactory; 35 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 36 | import org.wso2.am.analytics.publisher.util.Constants; 37 | import org.wso2.am.analytics.publisher.util.TestUtils; 38 | import org.wso2.am.analytics.publisher.util.UnitTestAppender; 39 | 40 | import java.util.ArrayList; 41 | import java.util.HashMap; 42 | import java.util.List; 43 | import java.util.Map; 44 | 45 | public class ErrorHandlingTestCase { 46 | 47 | @Test 48 | public void testConnectionInvalidURL() throws MetricCreationException, MetricReportingException { 49 | Logger log = LogManager.getLogger(EventHubClient.class); 50 | LoggerContext context = LoggerContext.getContext(false); 51 | Configuration config = context.getConfiguration(); 52 | UnitTestAppender appender = config.getAppender("UnitTestAppender"); 53 | 54 | Map configMap = new HashMap<>(); 55 | configMap.put(Constants.AUTH_API_URL, "some_url"); 56 | configMap.put(Constants.AUTH_API_TOKEN, "some_token"); 57 | MetricReporter metricReporter = MetricReporterFactory.getInstance().createMetricReporter(configMap); 58 | CounterMetric metric = metricReporter.createCounterMetric("test-connection-counter", MetricSchema.RESPONSE); 59 | List messages = new ArrayList(appender.getMessages()); 60 | Assert.assertTrue(TestUtils.isContains(messages, "Unrecoverable error occurred when creating Eventhub " 61 | + "Client"), "Expected error hasn't logged in the " 62 | + "EventHubClientClass"); 63 | } 64 | 65 | @Test(dependsOnMethods = {"testConnectionInvalidURL"}) 66 | public void testConnectionUnavailability() throws Exception { 67 | Logger log = LogManager.getLogger(EventHubClient.class); 68 | LoggerContext context = LoggerContext.getContext(false); 69 | Configuration config = context.getConfiguration(); 70 | UnitTestAppender appender = config.getAppender("UnitTestAppender"); 71 | log.atLevel(Level.DEBUG); 72 | 73 | Map configMap = new HashMap<>(); 74 | configMap.put(Constants.AUTH_API_URL, "https://localhost:1234/non-existance"); 75 | configMap.put(Constants.AUTH_API_TOKEN, "some_token"); 76 | MetricReporterFactory factory = MetricReporterFactory.getInstance(); 77 | factory.reset(); 78 | MetricReporter metricReporter = factory.createMetricReporter(configMap); 79 | CounterMetric metric = metricReporter.createCounterMetric("test-connection-counter", MetricSchema.RESPONSE); 80 | List messages = new ArrayList(appender.getMessages()); 81 | 82 | Thread.sleep(1000); 83 | Assert.assertTrue(TestUtils.isContains(messages, "Recoverable error occurred when creating Eventhub Client. " 84 | + "Retry attempts will be made")); 85 | Assert.assertTrue(TestUtils.isContains(messages, "Provided authentication endpoint " 86 | + "https://localhost:1234/non-existance is not " 87 | + "reachable.")); 88 | MetricEventBuilder builder = metric.getEventBuilder(); 89 | TestUtils.populateBuilder(builder); 90 | metric.incrementCount(builder); 91 | Thread.sleep(1000); 92 | messages = new ArrayList(appender.getMessages()); 93 | Assert.assertTrue(TestUtils.isContains(messages, "will be parked as EventHub Client is inactive."), "Thread " 94 | + "waiting log entry has not printed."); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/elk/ELKMetricEventBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter.elk; 20 | 21 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 22 | import org.wso2.am.analytics.publisher.reporter.AbstractMetricEventBuilder; 23 | import org.wso2.am.analytics.publisher.reporter.GenericInputValidator; 24 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 25 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 26 | import org.wso2.am.analytics.publisher.util.Constants; 27 | import org.wso2.am.analytics.publisher.util.EventMapAttributeFilter; 28 | import org.wso2.am.analytics.publisher.util.UserAgentParser; 29 | import ua_parser.Client; 30 | 31 | import java.util.HashMap; 32 | import java.util.Map; 33 | 34 | /** 35 | * Event builder for log Metric Reporter. 36 | */ 37 | public class ELKMetricEventBuilder extends AbstractMetricEventBuilder { 38 | 39 | protected Map requiredAttributes; 40 | private Map eventMap; 41 | private Boolean isBuilt = false; 42 | 43 | public ELKMetricEventBuilder() { 44 | requiredAttributes = GenericInputValidator.getInstance().getEventProperties(MetricSchema.RESPONSE); 45 | eventMap = new HashMap<>(); 46 | } 47 | 48 | public ELKMetricEventBuilder(Map requiredAttributes) { 49 | this.requiredAttributes = requiredAttributes; 50 | eventMap = new HashMap<>(); 51 | } 52 | 53 | @Override 54 | protected Map buildEvent() { 55 | if (!isBuilt) { 56 | // util function to filter required attributes 57 | eventMap = EventMapAttributeFilter.getInstance().filter(eventMap, requiredAttributes); 58 | 59 | // userAgent raw string is not required and removing 60 | String userAgentHeader = (String) eventMap.remove(Constants.USER_AGENT_HEADER); 61 | if (userAgentHeader != null) { 62 | setUserAgentProperties(userAgentHeader); 63 | } 64 | isBuilt = true; 65 | } 66 | return eventMap; 67 | } 68 | 69 | @Override 70 | public boolean validate() throws MetricReportingException { 71 | if (!isBuilt) { 72 | Map propertyMap = (Map) eventMap.get(Constants.PROPERTIES); 73 | if (propertyMap != null) { 74 | copyDefaultPropertiesToRootLevel(propertyMap); 75 | } 76 | for (Map.Entry entry : requiredAttributes.entrySet()) { 77 | Object attribute = eventMap.get(entry.getKey()); 78 | if (attribute == null) { 79 | throw new MetricReportingException(entry.getKey() + " is missing in metric data. This metric event " 80 | + "will not be processed further."); 81 | } else if (!attribute.getClass().equals(entry.getValue())) { 82 | throw new MetricReportingException(entry.getKey() + " is expecting a " + entry.getValue() + " type " 83 | + "attribute while attribute of type " 84 | + attribute.getClass() + " is present."); 85 | } 86 | } 87 | } 88 | return true; 89 | } 90 | 91 | @Override 92 | public MetricEventBuilder addAttribute(String key, Object value) throws MetricReportingException { 93 | eventMap.put(key, value); 94 | return this; 95 | } 96 | 97 | private void setUserAgentProperties(String userAgentHeader) { 98 | String browser = null; 99 | String platform = null; 100 | Client client = UserAgentParser.getInstance().parseUserAgent(userAgentHeader); 101 | if (client != null) { 102 | browser = client.userAgent.family; 103 | platform = client.os.family; 104 | } 105 | 106 | if (browser == null || browser.isEmpty()) { 107 | browser = Constants.UNKNOWN_VALUE; 108 | } 109 | if (platform == null || platform.isEmpty()) { 110 | platform = Constants.UNKNOWN_VALUE; 111 | } 112 | eventMap.put(Constants.USER_AGENT, browser); 113 | eventMap.put(Constants.PLATFORM, platform); 114 | } 115 | 116 | private void copyDefaultPropertiesToRootLevel(Map properties) { 117 | 118 | if (properties.get(Constants.API_CONTEXT) != null) { 119 | eventMap.put(Constants.API_CONTEXT, properties.get(Constants.API_CONTEXT)); 120 | } 121 | if (properties.get(Constants.USER_NAME) != null) { 122 | eventMap.put(Constants.USER_NAME, properties.get(Constants.USER_NAME)); 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/client/EventHubProducerClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.client; 20 | 21 | import com.azure.core.amqp.AmqpRetryOptions; 22 | import com.azure.core.amqp.AmqpTransportType; 23 | import com.azure.core.amqp.ProxyAuthenticationType; 24 | import com.azure.core.amqp.ProxyOptions; 25 | import com.azure.core.credential.TokenCredential; 26 | import com.azure.messaging.eventhubs.EventHubClientBuilder; 27 | import com.azure.messaging.eventhubs.EventHubProducerClient; 28 | import org.apache.commons.lang3.StringUtils; 29 | import org.apache.logging.log4j.LogManager; 30 | import org.apache.logging.log4j.Logger; 31 | import org.wso2.am.analytics.publisher.auth.AuthClient; 32 | import org.wso2.am.analytics.publisher.exception.ConnectionRecoverableException; 33 | import org.wso2.am.analytics.publisher.exception.ConnectionUnrecoverableException; 34 | import org.wso2.am.analytics.publisher.util.Constants; 35 | 36 | import java.io.UnsupportedEncodingException; 37 | import java.net.InetSocketAddress; 38 | import java.net.Proxy; 39 | import java.net.SocketAddress; 40 | import java.net.URLDecoder; 41 | import java.util.Map; 42 | 43 | /** 44 | * Factory class to create EventHubProducerClient instance. 45 | */ 46 | public class EventHubProducerClientFactory { 47 | private static final Logger log = LogManager.getLogger(EventHubClient.class); 48 | 49 | public static EventHubProducerClient create(String authEndpoint, String authToken, AmqpRetryOptions retryOptions, 50 | Map properties) 51 | throws ConnectionRecoverableException, ConnectionUnrecoverableException { 52 | TokenCredential tokenCredential = new WSO2TokenCredential(authEndpoint, authToken, properties); 53 | String tempSASToken; 54 | // generate SAS token to get eventhub meta data 55 | tempSASToken = getSASToken(authEndpoint, authToken, properties); 56 | 57 | String resourceURI = getResourceURI(tempSASToken); 58 | String fullyQualifiedNamespace = getNamespace(resourceURI); 59 | String eventhubName = getEventHubName(resourceURI); 60 | 61 | String isProxyEnabled = properties.get(Constants.PROXY_ENABLE); 62 | if (Boolean.parseBoolean(isProxyEnabled)) { 63 | String proxyHost = properties.get(Constants.PROXY_HOST); 64 | int proxyPort = Integer.parseInt(properties.get(Constants.PROXY_PORT)); 65 | String proxyUsername = properties.get(Constants.PROXY_USERNAME); 66 | String proxyPassword = properties.get(Constants.PROXY_PASSWORD); 67 | 68 | SocketAddress address = new InetSocketAddress(proxyHost, proxyPort); 69 | Proxy proxyAddress = new Proxy(Proxy.Type.HTTP, address); 70 | ProxyOptions proxyOptions; 71 | if (!StringUtils.isBlank(proxyUsername) && !StringUtils.isBlank(proxyPassword)) { 72 | proxyOptions = 73 | new ProxyOptions(ProxyAuthenticationType.BASIC, proxyAddress, proxyUsername, proxyPassword); 74 | } else { 75 | proxyOptions = 76 | new ProxyOptions(ProxyAuthenticationType.NONE, proxyAddress, null, null); 77 | } 78 | 79 | return new EventHubClientBuilder() 80 | .credential(fullyQualifiedNamespace, eventhubName, tokenCredential) 81 | .proxyOptions(proxyOptions) 82 | .retry(retryOptions) 83 | .transportType(AmqpTransportType.AMQP_WEB_SOCKETS) 84 | .buildProducerClient(); 85 | } else { 86 | return new EventHubClientBuilder() 87 | .credential(fullyQualifiedNamespace, eventhubName, tokenCredential) 88 | .retry(retryOptions) 89 | .buildProducerClient(); 90 | } 91 | } 92 | 93 | private static String getSASToken(String authEndpoint, String authToken, Map properties) 94 | throws ConnectionRecoverableException, ConnectionUnrecoverableException { 95 | return AuthClient.getSASToken(authEndpoint, authToken, properties); 96 | } 97 | 98 | /** 99 | * Extracts the resource URI from the SAS Token. 100 | * 101 | * @param sasToken SAS token of the user 102 | * @return decoded resource URI from the token 103 | */ 104 | private static String getResourceURI(String sasToken) { 105 | String[] sasAttributes = sasToken.split("&"); 106 | String[] resource = sasAttributes[0].split("="); 107 | String resourceURI = ""; 108 | try { 109 | resourceURI = URLDecoder.decode(resource[1], "UTF-8"); 110 | } catch (UnsupportedEncodingException e) { 111 | //never happens 112 | } 113 | //remove protocol append 114 | return resourceURI.replace("sb://", ""); 115 | } 116 | 117 | private static String getNamespace(String resourceURI) { 118 | return resourceURI.split("/")[0]; 119 | } 120 | 121 | private static String getEventHubName(String resourceURI) { 122 | return resourceURI.split("/", 2)[1]; 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /component/publisher-client/src/test/java/org/wso2/am/analytics/publisher/MetricReporterTestCase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.testng.annotations.Test; 24 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 25 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 26 | import org.wso2.am.analytics.publisher.reporter.CounterMetric; 27 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 28 | import org.wso2.am.analytics.publisher.reporter.MetricReporter; 29 | import org.wso2.am.analytics.publisher.reporter.MetricReporterFactory; 30 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 31 | import org.wso2.am.analytics.publisher.util.Constants; 32 | import org.wso2.am.analytics.publisher.util.TestUtils; 33 | 34 | import java.util.HashMap; 35 | import java.util.Map; 36 | 37 | public class MetricReporterTestCase { 38 | private static final Logger log = LogManager.getLogger(MetricReporterTestCase.class); 39 | 40 | @Test(enabled = false, expectedExceptions = MetricCreationException.class) 41 | public void testMetricReporterCreationWithoutConfigs() throws MetricCreationException, MetricReportingException { 42 | try { 43 | createAndPublish(null, null); 44 | } catch (MetricCreationException e) { 45 | log.error(e); 46 | throw e; 47 | } 48 | } 49 | 50 | @Test(enabled = false, expectedExceptions = MetricCreationException.class, dependsOnMethods = 51 | {"testMetricReporterCreationWithoutConfigs"}) 52 | public void testMetricReporterCreationWithMissingConfigs() throws MetricCreationException, 53 | MetricReportingException { 54 | Map configs = new HashMap<>(); 55 | configs.put("token.api.url", "localhost/token-api"); 56 | configs.put("auth.api.url", "localhost/auth-api"); 57 | configs.put("consumer.secret", "some_secret"); 58 | createAndPublish(configs, null); 59 | } 60 | 61 | @Test(enabled = false, expectedExceptions = MetricCreationException.class, dependsOnMethods = 62 | {"testMetricReporterCreationWithMissingConfigs"}) 63 | public void testMetricReporterCreationWithNullConfigs() throws MetricCreationException, 64 | MetricReportingException { 65 | Map configs = new HashMap<>(); 66 | configs.put("token.api.url", "localhost/token-api"); 67 | configs.put("auth.api.url", "localhost/auth-api"); 68 | configs.put("consumer.secret", "some_secret"); 69 | configs.put("sas.token", "some_token"); 70 | configs.put("consumer.key", null); 71 | createAndPublish(configs, null); 72 | } 73 | 74 | @Test(enabled = false, expectedExceptions = MetricCreationException.class, dependsOnMethods = 75 | {"testMetricReporterCreationWithNullConfigs"}) 76 | public void testMetricReporterCreationWithEmptyConfigs() throws MetricCreationException, 77 | MetricReportingException { 78 | Map configs = new HashMap<>(); 79 | configs.put("token.api.url", "localhost/token-api"); 80 | configs.put("auth.api.url", "localhost/auth-api"); 81 | configs.put("sas.token", "some_token"); 82 | configs.put("consumer.secret", "some_secret"); 83 | configs.put("consumer.key", ""); 84 | createAndPublish(configs, null); 85 | } 86 | 87 | @Test 88 | public void testCompleteFlow() throws MetricCreationException, MetricReportingException, InterruptedException { 89 | Map configMap = new HashMap<>(); 90 | String authURL = System.getenv(Constants.AUTH_API_URL); 91 | String authToken = System.getenv(Constants.AUTH_API_TOKEN); 92 | if (authToken != null && authURL != null) { 93 | configMap.put(Constants.AUTH_API_URL, authURL); 94 | configMap.put(Constants.AUTH_API_TOKEN, authToken); 95 | } else { 96 | return; 97 | } 98 | MetricReporter metricReporter = MetricReporterFactory.getInstance().createMetricReporter(configMap); 99 | CounterMetric metric = metricReporter.createCounterMetric("test-connection-counter", MetricSchema.RESPONSE); 100 | for (int i = 0; i < 5; i++) { 101 | MetricEventBuilder builder = metric.getEventBuilder(); 102 | TestUtils.populateBuilder(builder); 103 | metric.incrementCount(builder); 104 | } 105 | Thread.sleep(2000); 106 | //Assertions will be done after mocking eventhub client 107 | } 108 | 109 | 110 | /** 111 | * Helper method to create and publish metrics for testing purposes 112 | * 113 | * @param configs Config map 114 | * @param event Event map 115 | * @throws MetricCreationException Thrown if config properties are missing 116 | * @throws MetricReportingException Thrown if event properties are missing 117 | */ 118 | private void createAndPublish(Map configs, Map event) 119 | throws MetricCreationException, MetricReportingException { 120 | MetricReporter metricReporter = MetricReporterFactory.getInstance().createMetricReporter(configs); 121 | CounterMetric counterMetric = metricReporter.createCounterMetric("apim.response", MetricSchema.RESPONSE); 122 | MetricEventBuilder builder = counterMetric.getEventBuilder(); 123 | 124 | counterMetric.incrementCount(builder); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /component/publisher-client/src/test/java/org/wso2/am/analytics/publisher/DefaultFaultMetricBuilderTestCase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher; 20 | 21 | import com.azure.core.amqp.AmqpRetryMode; 22 | import com.azure.core.amqp.AmqpRetryOptions; 23 | import org.apache.logging.log4j.LogManager; 24 | import org.apache.logging.log4j.Logger; 25 | import org.testng.Assert; 26 | import org.testng.annotations.BeforeMethod; 27 | import org.testng.annotations.Test; 28 | import org.wso2.am.analytics.publisher.client.EventHubClient; 29 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 30 | import org.wso2.am.analytics.publisher.exception.MetricReportingException; 31 | import org.wso2.am.analytics.publisher.reporter.MetricEventBuilder; 32 | import org.wso2.am.analytics.publisher.reporter.MetricSchema; 33 | import org.wso2.am.analytics.publisher.reporter.cloud.DefaultCounterMetric; 34 | import org.wso2.am.analytics.publisher.reporter.cloud.EventQueue; 35 | import org.wso2.am.analytics.publisher.util.Constants; 36 | 37 | import java.time.Clock; 38 | import java.time.Duration; 39 | import java.time.OffsetDateTime; 40 | import java.util.HashMap; 41 | import java.util.Map; 42 | 43 | public class DefaultFaultMetricBuilderTestCase { 44 | private static final Logger log = LogManager.getLogger(DefaultFaultMetricBuilderTestCase.class); 45 | private MetricEventBuilder builder; 46 | 47 | @BeforeMethod 48 | public void createBuilder() throws MetricCreationException { 49 | AmqpRetryOptions retryOptions = new AmqpRetryOptions() 50 | .setDelay(Duration.ofSeconds(30)) 51 | .setMaxRetries(2) 52 | .setMaxDelay(Duration.ofSeconds(120)) 53 | .setTryTimeout(Duration.ofSeconds(30)) 54 | .setMode(AmqpRetryMode.FIXED); 55 | EventHubClient client = new EventHubClient("some_endpoint", "some_token", retryOptions, 56 | new HashMap<>()); 57 | EventQueue queue = new EventQueue(100, 1, client, 10); 58 | DefaultCounterMetric metric = new DefaultCounterMetric("test.builder.metric", queue, MetricSchema.ERROR); 59 | builder = metric.getEventBuilder(); 60 | } 61 | 62 | @Test(expectedExceptions = MetricReportingException.class) 63 | public void testMissingAttributes() throws MetricCreationException, MetricReportingException { 64 | builder.addAttribute("apiName", "PizzaShack"); 65 | builder.build(); 66 | } 67 | 68 | @Test(expectedExceptions = MetricReportingException.class, dependsOnMethods = {"testMissingAttributes"}) 69 | public void testAttributesWithInvalidTypes() throws MetricCreationException, MetricReportingException { 70 | builder.addAttribute(Constants.REQUEST_TIMESTAMP, System.currentTimeMillis()) 71 | .addAttribute(Constants.CORRELATION_ID, "1234-4567") 72 | .addAttribute(Constants.KEY_TYPE, "prod") 73 | .addAttribute(Constants.ERROR_TYPE, "backend") 74 | .addAttribute(Constants.ERROR_CODE, 401) 75 | .addAttribute(Constants.ERROR_MESSAGE, "Authentication Error") 76 | .addAttribute(Constants.API_ID, "9876-54f1") 77 | .addAttribute(Constants.API_NAME, "PizzaShack") 78 | .addAttribute(Constants.API_VERSION, "1.0.0") 79 | .addAttribute(Constants.API_CREATION, "admin") 80 | .addAttribute(Constants.API_METHOD, "POST") 81 | .addAttribute(Constants.API_CREATOR_TENANT_DOMAIN, "carbon.super") 82 | .addAttribute(Constants.APPLICATION_ID, "3445-6778") 83 | .addAttribute(Constants.APPLICATION_NAME, "default") 84 | .addAttribute(Constants.APPLICATION_OWNER, "admin") 85 | .addAttribute(Constants.REGION_ID, "NA") 86 | .addAttribute(Constants.GATEWAY_TYPE, "Synapse") 87 | .addAttribute(Constants.PROXY_RESPONSE_CODE, 401) 88 | .addAttribute(Constants.TARGET_RESPONSE_CODE, "someString") 89 | .build(); 90 | } 91 | 92 | @Test(dependsOnMethods = {"testAttributesWithInvalidTypes"}) 93 | public void testMetricBuilder() throws MetricCreationException, MetricReportingException { 94 | Map eventMap = builder 95 | .addAttribute(Constants.REQUEST_TIMESTAMP, OffsetDateTime.now(Clock.systemUTC()).toString()) 96 | .addAttribute(Constants.CORRELATION_ID, "1234-4567") 97 | .addAttribute(Constants.KEY_TYPE, "prod") 98 | .addAttribute(Constants.ERROR_TYPE, "backend") 99 | .addAttribute(Constants.ERROR_CODE, 401) 100 | .addAttribute(Constants.ERROR_MESSAGE, "Authentication Error") 101 | .addAttribute(Constants.API_ID, "9876-54f1") 102 | .addAttribute(Constants.API_TYPE, "HTTP") 103 | .addAttribute(Constants.API_NAME, "PizzaShack") 104 | .addAttribute(Constants.API_VERSION, "1.0.0") 105 | .addAttribute(Constants.API_CREATION, "admin") 106 | .addAttribute(Constants.API_METHOD, "POST") 107 | .addAttribute(Constants.API_CREATOR_TENANT_DOMAIN, "carbon.super") 108 | .addAttribute(Constants.APPLICATION_ID, "3445-6778") 109 | .addAttribute(Constants.APPLICATION_NAME, "default") 110 | .addAttribute(Constants.APPLICATION_OWNER, "admin") 111 | .addAttribute(Constants.REGION_ID, "NA") 112 | .addAttribute(Constants.GATEWAY_TYPE, "Synapse") 113 | .addAttribute(Constants.PROXY_RESPONSE_CODE, 401) 114 | .addAttribute(Constants.TARGET_RESPONSE_CODE, 401) 115 | .addAttribute(Constants.PROPERTIES, new HashMap()) 116 | .build(); 117 | 118 | Assert.assertFalse(eventMap.isEmpty()); 119 | Assert.assertEquals(eventMap.size(), 22, "Some attributes are missing from the resulting event map"); 120 | Assert.assertEquals(eventMap.get(Constants.EVENT_TYPE), "fault", "Event type should be set to fault"); 121 | Assert.assertEquals(eventMap.get(Constants.API_TYPE), "HTTP", "API type should be set to HTTP"); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/auth/AuthProxyUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.auth; 20 | 21 | import feign.Client; 22 | import feign.httpclient.ApacheHttpClient; 23 | import org.apache.commons.lang3.StringUtils; 24 | import org.apache.http.HttpHost; 25 | import org.apache.http.auth.AuthScope; 26 | import org.apache.http.auth.UsernamePasswordCredentials; 27 | import org.apache.http.client.CredentialsProvider; 28 | import org.apache.http.config.RegistryBuilder; 29 | import org.apache.http.conn.socket.ConnectionSocketFactory; 30 | import org.apache.http.conn.socket.PlainConnectionSocketFactory; 31 | import org.apache.http.conn.ssl.DefaultHostnameVerifier; 32 | import org.apache.http.conn.ssl.SSLConnectionSocketFactory; 33 | import org.apache.http.conn.ssl.SSLContexts; 34 | import org.apache.http.impl.client.BasicCredentialsProvider; 35 | import org.apache.http.impl.client.HttpClientBuilder; 36 | import org.apache.http.impl.client.ProxyAuthenticationStrategy; 37 | import org.apache.http.impl.conn.DefaultProxyRoutePlanner; 38 | import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; 39 | import org.apache.logging.log4j.LogManager; 40 | import org.apache.logging.log4j.Logger; 41 | import org.wso2.am.analytics.publisher.exception.HttpClientException; 42 | import org.wso2.am.analytics.publisher.util.Constants; 43 | 44 | import java.io.IOException; 45 | import java.nio.file.Files; 46 | import java.nio.file.Paths; 47 | import java.security.KeyManagementException; 48 | import java.security.KeyStore; 49 | import java.security.KeyStoreException; 50 | import java.security.NoSuchAlgorithmException; 51 | import java.security.cert.CertificateException; 52 | import java.util.Map; 53 | 54 | import javax.net.ssl.SSLContext; 55 | 56 | import static org.wso2.am.analytics.publisher.util.Constants.HTTPS_PROTOCOL; 57 | import static org.wso2.am.analytics.publisher.util.Constants.HTTP_PROTOCOL; 58 | import static org.wso2.am.analytics.publisher.util.Constants.KEYSTORE_TYPE; 59 | 60 | /** 61 | * Util class to generate http client with proxy configurations. 62 | */ 63 | public class AuthProxyUtils { 64 | 65 | private static final Logger log = LogManager.getLogger(AuthProxyUtils.class); 66 | 67 | public static Client getClient(Map properties) { 68 | return getFeignHttpClient(properties); 69 | } 70 | 71 | private static ApacheHttpClient getFeignHttpClient(Map properties) { 72 | String proxyHost = properties.get(Constants.PROXY_HOST); 73 | int proxyPort = Integer.parseInt(properties.get(Constants.PROXY_PORT)); 74 | String proxyUsername = properties.get(Constants.PROXY_USERNAME); 75 | String proxyPassword = properties.get(Constants.PROXY_PASSWORD); 76 | String proxyProtocol = properties.get(Constants.PROXY_PROTOCOL); 77 | 78 | if (StringUtils.isEmpty(proxyProtocol)) { 79 | proxyProtocol = HTTP_PROTOCOL; 80 | } 81 | 82 | PoolingHttpClientConnectionManager pool = null; 83 | try { 84 | pool = getPoolingHttpClientConnectionManager(properties); 85 | } catch (HttpClientException e) { 86 | log.error("Error while getting http client connection manager", e); 87 | } 88 | 89 | HttpHost host = new HttpHost(proxyHost, proxyPort, proxyProtocol); 90 | DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(host); 91 | HttpClientBuilder clientBuilder = HttpClientBuilder.create() 92 | .setRoutePlanner(routePlanner) 93 | .setConnectionManager(pool); 94 | 95 | if (!StringUtils.isBlank(proxyUsername) && !StringUtils.isBlank(proxyPassword)) { 96 | CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); 97 | credentialsProvider.setCredentials(new AuthScope(proxyHost, proxyPort), 98 | new UsernamePasswordCredentials(proxyUsername, proxyPassword)); 99 | clientBuilder 100 | .setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy()) 101 | .setDefaultCredentialsProvider(credentialsProvider); 102 | } 103 | 104 | return new ApacheHttpClient(clientBuilder.build()); 105 | } 106 | 107 | private static PoolingHttpClientConnectionManager getPoolingHttpClientConnectionManager(Map properties) throws HttpClientException { 109 | SSLConnectionSocketFactory socketFactory = createSocketFactory(properties); 110 | ConnectionSocketFactory httpSocketFactory = new PlainConnectionSocketFactory(); 111 | org.apache.http.config.Registry socketFactoryRegistry = 112 | RegistryBuilder.create() 113 | .register(HTTP_PROTOCOL, httpSocketFactory) 114 | .register(HTTPS_PROTOCOL, socketFactory) 115 | .build(); 116 | return new PoolingHttpClientConnectionManager(socketFactoryRegistry); 117 | } 118 | 119 | private static SSLConnectionSocketFactory createSocketFactory(Map properties) 120 | throws HttpClientException { 121 | SSLContext sslContext; 122 | 123 | String keyStorePassword = properties.get(Constants.KEYSTORE_PASSWORD); 124 | String keyStorePath = properties.get(Constants.KEYSTORE_LOCATION); 125 | try { 126 | KeyStore trustStore = KeyStore.getInstance(KEYSTORE_TYPE); 127 | trustStore.load(Files.newInputStream(Paths.get(keyStorePath)), keyStorePassword.toCharArray()); 128 | sslContext = SSLContexts.custom().loadTrustMaterial(trustStore).build(); 129 | return new SSLConnectionSocketFactory(sslContext, new DefaultHostnameVerifier()); 130 | } catch (KeyStoreException e) { 131 | throw new HttpClientException("Failed to read from Key Store", e); 132 | } catch (IOException e) { 133 | throw new HttpClientException("Key Store not found in " + keyStorePath, e); 134 | } catch (CertificateException e) { 135 | throw new HttpClientException("Failed to read Certificate", e); 136 | } catch (NoSuchAlgorithmException e) { 137 | throw new HttpClientException("Failed to load Key Store from " + keyStorePath, e); 138 | } catch (KeyManagementException e) { 139 | throw new HttpClientException("Failed to load key from" + keyStorePath, e); 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /component/publisher-client/src/main/java/org/wso2/am/analytics/publisher/reporter/MetricReporterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2021, WSO2 Inc. (http://www.wso2.org) All Rights Reserved. 3 | * 4 | * WSO2 Inc. licenses this file to you under the Apache License, 5 | * Version 2.0 (the "License"); you may not use this file except 6 | * in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, 12 | * software distributed under the License is distributed on an 13 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | * KIND, either express or implied. See the License for the 15 | * specific language governing permissions and limitations 16 | * under the License. 17 | */ 18 | 19 | package org.wso2.am.analytics.publisher.reporter; 20 | 21 | import org.apache.logging.log4j.LogManager; 22 | import org.apache.logging.log4j.Logger; 23 | import org.wso2.am.analytics.publisher.exception.MetricCreationException; 24 | import org.wso2.am.analytics.publisher.reporter.cloud.DefaultAnalyticsMetricReporter; 25 | import org.wso2.am.analytics.publisher.reporter.elk.ELKMetricReporter; 26 | import org.wso2.am.analytics.publisher.reporter.moesif.MoesifReporter; 27 | import org.wso2.am.analytics.publisher.util.Constants; 28 | 29 | import java.lang.reflect.Constructor; 30 | import java.lang.reflect.InvocationTargetException; 31 | import java.util.HashMap; 32 | import java.util.Map; 33 | 34 | /** 35 | * Factory class to create {@link MetricReporter}. Based on the passed argument relevant concrete implementation will 36 | * be created and returned. Factory will behave in Singleton manner and if same type of instance is requested again 37 | * Factory will return earlier requested instance. 38 | */ 39 | public class MetricReporterFactory { 40 | private static final Logger log = LogManager.getLogger(MetricReporterFactory.class); 41 | private static final MetricReporterFactory instance = new MetricReporterFactory(); 42 | private static Map reporterRegistry = new HashMap<>(); 43 | 44 | private MetricReporterFactory() { 45 | //private constructor 46 | } 47 | 48 | public MetricReporter createMetricReporter(Map properties) 49 | throws MetricCreationException { 50 | if (reporterRegistry.get(Constants.DEFAULT_REPORTER) == null) { 51 | synchronized (this) { 52 | if (reporterRegistry.get(Constants.DEFAULT_REPORTER) == null) { 53 | MetricReporter reporterInstance = new DefaultAnalyticsMetricReporter(properties); 54 | reporterRegistry.put(Constants.DEFAULT_REPORTER, reporterInstance); 55 | return reporterInstance; 56 | } 57 | } 58 | } 59 | MetricReporter reporterInstance = reporterRegistry.get(Constants.DEFAULT_REPORTER); 60 | log.info("Metric Reporter of type " + reporterInstance.getClass().toString().replaceAll("[\r\n]", "") + 61 | " is already created. Hence returning same instance"); 62 | return reporterInstance; 63 | } 64 | 65 | public MetricReporter createLogMetricReporter(Map properties) throws MetricCreationException { 66 | if (reporterRegistry.get(Constants.ELK_REPORTER) == null) { 67 | synchronized (this) { 68 | if (reporterRegistry.get(Constants.ELK_REPORTER) == null) { 69 | MetricReporter reporterInstance = new ELKMetricReporter(properties); 70 | reporterRegistry.put(Constants.ELK_REPORTER, reporterInstance); 71 | return reporterInstance; 72 | } 73 | } 74 | } 75 | 76 | MetricReporter reporterInstance = reporterRegistry.get(Constants.ELK_REPORTER); 77 | log.info("Metric Reporter of type " + reporterInstance.getClass().toString().replaceAll("[\r\n]", "") + 78 | " is already created. Hence returning same instance"); 79 | return reporterInstance; 80 | } 81 | 82 | public MetricReporter createMoesifMetricReporter(Map properties) throws MetricCreationException { 83 | if (reporterRegistry.get(Constants.MOESIF) == null) { 84 | synchronized (this) { 85 | if (reporterRegistry.get(Constants.MOESIF) == null) { 86 | MetricReporter reporterInstance = new MoesifReporter(properties); 87 | reporterRegistry.put(Constants.MOESIF, reporterInstance); 88 | } 89 | } 90 | } 91 | MetricReporter reporterInstance = reporterRegistry.get(Constants.MOESIF); 92 | log.info("Metric Reporter of type " + reporterInstance.getClass().toString().replaceAll("[\r\n]", "") + 93 | " is already created. Hence returning same instance"); 94 | return reporterInstance; 95 | } 96 | 97 | public MetricReporter createMetricReporter(String fullyQualifiedClassName, Map properties) 98 | throws MetricCreationException { 99 | if (reporterRegistry.get(fullyQualifiedClassName) == null) { 100 | synchronized (this) { 101 | if (reporterRegistry.get(fullyQualifiedClassName) == null) { 102 | if (fullyQualifiedClassName != null && !fullyQualifiedClassName.isEmpty()) { 103 | try { 104 | Class clazz = 105 | (Class) Class.forName(fullyQualifiedClassName); 106 | Constructor constructor = 107 | clazz.getConstructor(Map.class); 108 | MetricReporter reporterInstance = constructor.newInstance(properties); 109 | reporterRegistry.put(fullyQualifiedClassName, reporterInstance); 110 | return reporterInstance; 111 | } catch (InstantiationException | IllegalAccessException | ClassNotFoundException 112 | | NoSuchMethodException | InvocationTargetException e) { 113 | throw new MetricCreationException("Error occurred while creating a Metric Reporter of type" 114 | + " " + fullyQualifiedClassName, e); 115 | } 116 | } else { 117 | throw new MetricCreationException("Provided class name is either empty or null. Hence cannot " 118 | + "create the Reporter."); 119 | } 120 | } 121 | } 122 | } 123 | MetricReporter reporterInstance = reporterRegistry.get(fullyQualifiedClassName); 124 | log.info("Metric Reporter of type " + reporterInstance.getClass().toString().replaceAll("[\r\n]", "") + 125 | " is already created. Hence returning same instance"); 126 | return reporterInstance; 127 | } 128 | 129 | /** 130 | * Reset the MetricReporterFactory registry. Only intended to be used in testing 131 | */ 132 | public void reset() { 133 | reporterRegistry.clear(); 134 | } 135 | 136 | public static MetricReporterFactory getInstance() { 137 | return instance; 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /features/publisher-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 19 | 20 | 21 | 22 | org.wso2.am.analytics.publisher 23 | apim-analytics-publisher-features 24 | 1.2.34-SNAPSHOT 25 | ../pom.xml 26 | 27 | 28 | 4.0.0 29 | 30 | org.wso2.am.analytics.publisher.client.feature 31 | carbon-feature 32 | WSO2 APIM Analytics Publisher Feature 33 | This feature contains the all bundles related to APIM Analytics publisher. By installing 34 | this feature APIM servers will get ability to publish analytics to APIM cloud 35 | 36 | 37 | 38 | org.wso2.am.analytics.publisher 39 | org.wso2.am.analytics.publisher.client 40 | 41 | 42 | com.azure 43 | azure-messaging-eventhubs 44 | 45 | 46 | com.google.code.gson 47 | gson 48 | 49 | 50 | log4j 51 | log4j 52 | 53 | 54 | org.wso2.orbit.com.azure 55 | azure-core 56 | 57 | 58 | org.wso2.orbit.com.azure 59 | azure-core-amqp 60 | 61 | 62 | org.wso2.orbit.com.microsoft.azure 63 | qpid-proton-j-extensions 64 | 65 | 66 | org.wso2.orbit.com.azure 67 | azure-messaging-eventhubs 68 | 69 | 70 | org.wso2.orbit.io.projectreactor 71 | reactor-core 72 | 73 | 74 | org.apache.qpid 75 | proton-j 76 | 77 | 78 | ua.parser.wso2 79 | ua-parser 80 | 81 | 82 | 83 | 84 | 85 | 86 | org.wso2.carbon.maven 87 | carbon-feature-plugin 88 | ${carbon.feature.plugin.version} 89 | true 90 | 91 | 92 | 1-p2-feature-generation 93 | 94 | generate 95 | 96 | 97 | ../etc/feature.properties 98 | 99 | 100 | org.wso2.carbon.p2.category.type 101 | server 102 | 103 | 104 | org.eclipse.equinox.p2.type.group 105 | true 106 | 107 | 108 | 109 | 110 | org.wso2.am.analytics.publisher.client 111 | ${project.version} 112 | 113 | 114 | com.google.gson 115 | ${gson.version} 116 | 117 | 118 | azure-core 119 | ${azure-core.version} 120 | 121 | 122 | azure-core-amqp 123 | ${azure-core-amqp.version} 124 | 125 | 126 | qpid-proton-j-extensions 127 | ${azure-qpid-proton-j-extensions.version} 128 | 129 | 130 | azure-messaging-eventhubs 131 | ${azure-messaging-eventhubs.version} 132 | 133 | 134 | reactor-core 135 | ${reactor-core.version} 136 | 137 | 138 | org.apache.qpid.proton-j 139 | ${proton-j.version} 140 | 141 | 142 | ua-parser 143 | ${ua_parser.version} 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | --------------------------------------------------------------------------------