├── src ├── main │ └── java │ │ └── com │ │ └── blibli │ │ └── oss │ │ └── qa │ │ └── util │ │ ├── model │ │ ├── Constant.java │ │ ├── HarModel.java │ │ ├── RequestResponsePair.java │ │ └── RequestResponseStorage.java │ │ ├── har │ │ └── Pages.java │ │ ├── ResponseModel.java │ │ ├── json │ │ └── JSONUtil.java │ │ └── services │ │ ├── DevToolsServices.java │ │ ├── HarEntryConverter.java │ │ └── NetworkListener.java └── test │ └── java │ └── com │ └── blibli │ └── oss │ └── qa │ └── util │ ├── BaseTest.java │ ├── AppTestWithHarFilter.java │ ├── UsingCdpTest.java │ ├── UnicodeTest.java │ ├── AppTest.java │ └── ThreadSafeHarCreationTest.java ├── .gitignore ├── .github └── workflows │ ├── maven.yml │ └── maven-publish.yml ├── settings.xml ├── README.md ├── pom.xml └── LICENSE /src/main/java/com/blibli/oss/qa/util/model/Constant.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.model; 2 | 3 | public class Constant { 4 | public static String DEFAULT_UNICODE="UTF-8"; 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/har/Pages.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.har; 2 | 3 | import java.util.Date; 4 | 5 | public class Pages { 6 | private Date startedDate; 7 | private Date endedDate; 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | pom.xml.tag 3 | pom.xml.releaseBackup 4 | pom.xml.versionsBackup 5 | pom.xml.next 6 | release.properties 7 | dependency-reduced-pom.xml 8 | buildNumber.properties 9 | .mvn/timing.properties 10 | # https://github.com/takari/maven-wrapper#usage-without-binary-jar 11 | .mvn/wrapper/maven-wrapper.jar 12 | .idea/ 13 | *.har 14 | *.iml -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/model/HarModel.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.model; 2 | 3 | import lombok.Data; 4 | 5 | import org.openqa.selenium.remote.http.HttpRequest; 6 | import org.openqa.selenium.remote.http.HttpResponse; 7 | 8 | @Data 9 | public class HarModel { 10 | private HttpRequest httpRequest; 11 | private HttpResponse httpResponse; 12 | 13 | public HarModel(HttpRequest httpRequest, HttpResponse httpResponse) { 14 | this.httpRequest = httpRequest; 15 | this.httpResponse = httpResponse; 16 | } 17 | 18 | public HttpRequest getHttpRequest() { 19 | return this.httpRequest; 20 | } 21 | 22 | public HttpResponse getHttpResponse() { 23 | return this.httpResponse; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/test/java/com/blibli/oss/qa/util/BaseTest.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util; 2 | 3 | import com.blibli.oss.qa.util.services.NetworkListener; 4 | import io.github.bonigarcia.wdm.WebDriverManager; 5 | import org.openqa.selenium.chrome.ChromeDriver; 6 | import org.openqa.selenium.chrome.ChromeOptions; 7 | import org.openqa.selenium.remote.DesiredCapabilities; 8 | 9 | import java.io.File; 10 | import java.io.IOException; 11 | import java.nio.file.Files; 12 | import java.nio.file.Paths; 13 | import java.util.Optional; 14 | 15 | public class BaseTest { 16 | 17 | 18 | 19 | public String readHarData(String fileName) throws IOException { 20 | String harFile = Paths.get(".").toAbsolutePath().normalize().toString() + ""+File.separator +""+fileName; 21 | System.out.println("Read Har Data " + harFile); 22 | return new String(Files.readAllBytes(Paths.get(fileName))); 23 | } 24 | 25 | } 26 | 27 | -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/ResponseModel.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util; 2 | 3 | 4 | import org.openqa.selenium.devtools.v142.network.model.ResponseReceived; 5 | import org.openqa.selenium.devtools.v142.network.Network; 6 | 7 | public class ResponseModel { 8 | ResponseReceived responseReceived; 9 | Network.GetResponseBodyResponse getResponseBodyResponse; 10 | 11 | public ResponseReceived getResponseReceived() { 12 | return responseReceived; 13 | } 14 | 15 | public void setResponseReceived(ResponseReceived responseReceived) { 16 | this.responseReceived = responseReceived; 17 | } 18 | 19 | public Network.GetResponseBodyResponse getGetResponseBodyResponse() { 20 | return getResponseBodyResponse; 21 | } 22 | 23 | public void setGetResponseBodyResponse(Network.GetResponseBodyResponse getResponseBodyResponse) { 24 | this.getResponseBodyResponse = getResponseBodyResponse; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Java CI with Maven 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | env: 13 | ## Sets environment variable 14 | CHROME_MODE: HEADLESS 15 | 16 | jobs: 17 | build: 18 | 19 | runs-on: ubuntu-latest 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Set up JDK 11 24 | uses: actions/setup-java@v3 25 | with: 26 | java-version: '11' 27 | distribution: 'temurin' 28 | cache: maven 29 | server-id: github # Value of the distributionManagement/repository/id field of the pom.xml 30 | settings-path: ${{ github.workspace }} # location for the settings.xml file 31 | 32 | - uses: browser-actions/setup-chrome@latest 33 | - name: test chrome 34 | run: chrome --version 35 | - name: Build with Maven 36 | run: mvn -B clean verify --file pom.xml -s $GITHUB_WORKSPACE/settings.xml -Dfile.encoding=UTF-8 37 | -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/model/RequestResponsePair.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.model; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import org.openqa.selenium.devtools.v142.network.model.*; 6 | 7 | 8 | import java.util.Date; 9 | 10 | @Getter 11 | @Setter 12 | public class RequestResponsePair { 13 | private String requestId; 14 | private Request request; 15 | private RequestWillBeSent requestWillBeSent; 16 | private RequestWillBeSentExtraInfo requestWillBeSentExtraInfo; 17 | private Response response; 18 | private ResponseReceivedExtraInfo responseReceivedExtraInfo; 19 | private LoadingFailed loadingFailed; 20 | private LoadingFinished loadingFinished; 21 | private Date requestOn; 22 | private String responseBody; 23 | 24 | public RequestResponsePair(Request request, Date requestOn, String responseBody) { 25 | this.request = request; 26 | this.requestOn = requestOn; 27 | this.responseBody = responseBody; 28 | } 29 | 30 | public RequestResponsePair(String requestId, Request request, Date requestOn, String responseBody) { 31 | this.requestId = requestId; 32 | this.request = request; 33 | this.requestOn = requestOn; 34 | this.responseBody = responseBody; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 23 | 24 | 25 | github 26 | ${env.USERNAME} 27 | ${env.GITHUB_TOKEN} 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/model/RequestResponseStorage.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.model; 2 | 3 | import lombok.Getter; 4 | import org.openqa.selenium.devtools.v142.network.model.LoadingFailed; 5 | import org.openqa.selenium.devtools.v142.network.model.Request; 6 | import org.openqa.selenium.devtools.v142.network.model.Response; 7 | import org.openqa.selenium.devtools.v142.network.model.ResponseReceivedExtraInfo; 8 | 9 | import java.util.Collections; 10 | import java.util.Date; 11 | import java.util.List; 12 | import java.util.concurrent.CopyOnWriteArrayList; 13 | 14 | @Getter 15 | public class RequestResponseStorage { 16 | private final List requestResponsePairs; 17 | 18 | public RequestResponseStorage() { 19 | this.requestResponsePairs = new CopyOnWriteArrayList<>(); 20 | } 21 | 22 | public void addRequest(Request request, Date requestOn) { 23 | requestResponsePairs.add(new RequestResponsePair(request, requestOn, null)); 24 | } 25 | 26 | public void addRequest(String requestId, Request request, Date requestOn) { 27 | requestResponsePairs.add(new RequestResponsePair(requestId, request, requestOn, null)); 28 | } 29 | 30 | public void addResponse(Response response, String responseBody) { 31 | for (int i = 0; i < requestResponsePairs.size(); i++) { 32 | if (requestResponsePairs.get(i).getRequest().getUrl().equals(response.getUrl())) { 33 | requestResponsePairs.get(i).setResponse(response); 34 | requestResponsePairs.get(i).setResponseBody(responseBody); 35 | break; 36 | } 37 | } 38 | } 39 | 40 | public void addLoadingFailed(LoadingFailed loadingFailed){ 41 | for (int i = 0; i < requestResponsePairs.size(); i++) { 42 | if (requestResponsePairs.get(i).getRequestId().equals(loadingFailed.getRequestId().toString())) { 43 | requestResponsePairs.get(i).setLoadingFailed(loadingFailed); 44 | break; 45 | } 46 | } 47 | } 48 | 49 | public void addresponseReceivedExtraInfo(ResponseReceivedExtraInfo responseReceivedExtraInfoConsumer) { 50 | for (int i = 0; i < requestResponsePairs.size(); i++) { 51 | if (requestResponsePairs.get(i).getRequestId().equals(responseReceivedExtraInfoConsumer.getRequestId().toString())) { 52 | requestResponsePairs.get(i).setResponseReceivedExtraInfo(responseReceivedExtraInfoConsumer); 53 | break; 54 | } 55 | } 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/json/JSONUtil.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.json; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | 6 | import java.io.IOException; 7 | import java.nio.charset.StandardCharsets; 8 | import java.nio.file.Files; 9 | import java.nio.file.Path; 10 | import java.nio.file.Paths; 11 | import java.util.ArrayList; 12 | 13 | public class JSONUtil { 14 | 15 | // create function for writting json file with given path from object 16 | public static void appendJson(Object obj, String path) { 17 | // read json file from path 18 | String json = JSONUtil.readJson(path); 19 | ObjectMapper mapper = new ObjectMapper(); 20 | // convert json to json objectMapper 21 | ArrayList jsonObject = null; 22 | try { 23 | jsonObject = mapper.readValue(json, ArrayList.class); 24 | } catch (JsonProcessingException e) { 25 | e.printStackTrace(); 26 | jsonObject = new ArrayList<>(); 27 | } 28 | jsonObject.add(obj); 29 | 30 | try { 31 | json = mapper.writeValueAsString(jsonObject); 32 | // System.out.println("ResultingJSONstring = " + json); 33 | // write json into file from path parameter 34 | // mapper.writeValue(new java.io.File(path), obj); 35 | } catch (JsonProcessingException e) { 36 | e.printStackTrace(); 37 | } catch (IOException e) { 38 | e.printStackTrace(); 39 | } 40 | } 41 | 42 | private static String readJson(String path) { 43 | // check if path file is exist 44 | if (new java.io.File(path).exists()) { 45 | // read json file from path 46 | ObjectMapper mapper = new ObjectMapper(); 47 | try { 48 | return mapper.readValue(new java.io.File(path), String.class); 49 | } catch (IOException e) { 50 | e.printStackTrace(); 51 | } 52 | }else{ 53 | // create file from path 54 | try { 55 | // create empty json file from path 56 | Path p = Paths.get(path); 57 | Files.write(p,"{}".getBytes(StandardCharsets.UTF_8)); 58 | return "{}"; 59 | } catch (IOException e) { 60 | e.printStackTrace(); 61 | } 62 | } 63 | return null; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/services/DevToolsServices.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.services; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.openqa.selenium.MutableCapabilities; 5 | import org.openqa.selenium.WebDriver; 6 | import org.openqa.selenium.devtools.DevTools; 7 | import org.openqa.selenium.devtools.HasDevTools; 8 | import org.openqa.selenium.remote.Augmenter; 9 | import org.openqa.selenium.remote.RemoteWebDriver; 10 | 11 | import java.lang.reflect.Field; 12 | 13 | @Slf4j 14 | public class DevToolsServices { 15 | private String baseRemoteUrl; 16 | private WebDriver driver; 17 | 18 | private DevTools devTools; 19 | public DevToolsServices(WebDriver driver) { 20 | this.driver = driver; 21 | } 22 | public DevToolsServices(WebDriver driver , String baseRemoteUrl) { 23 | this.baseRemoteUrl = baseRemoteUrl; 24 | this.driver = driver; 25 | } 26 | 27 | public DevTools getDevTools(){ 28 | try { 29 | if (driver instanceof RemoteWebDriver) { 30 | this.devTools = getCdpUsingCustomurl(); 31 | } else { 32 | this.devTools = ((HasDevTools) driver).getDevTools(); 33 | } 34 | } catch (Exception e) { 35 | e.printStackTrace(); 36 | } 37 | return this.devTools; 38 | } 39 | 40 | private DevTools getCdpUsingCustomurl() { 41 | // Before proceeding, reach into the driver and manipulate the capabilities to 42 | // include the se:cdp and se:cdpVersion keys. 43 | try { 44 | Field capabilitiesField = RemoteWebDriver.class.getDeclaredField("capabilities"); 45 | capabilitiesField.setAccessible(true); 46 | String sessionId = ((RemoteWebDriver) driver).getSessionId().toString(); 47 | String devtoolsUrl = String.format("ws://%s/devtools/%s/page", baseRemoteUrl, sessionId); 48 | 49 | MutableCapabilities mutableCapabilities = (MutableCapabilities) capabilitiesField.get(driver); 50 | mutableCapabilities.setCapability("se:cdp", devtoolsUrl); 51 | mutableCapabilities.setCapability("se:cdpVersion", mutableCapabilities.getBrowserVersion()); 52 | } catch (Exception e) { 53 | log.info("Failed to spoof RemoteWebDriver capabilities :sadpanda:"); 54 | } 55 | 56 | // Proceed to "augment" the driver and get a dev tools client ... 57 | RemoteWebDriver augmenteDriver = (RemoteWebDriver) new Augmenter().augment(driver); 58 | DevTools devTools = ((HasDevTools) augmenteDriver).getDevTools(); 59 | this.driver = augmenteDriver; 60 | return devTools; 61 | } 62 | 63 | 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/com/blibli/oss/qa/util/AppTestWithHarFilter.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util; 2 | 3 | 4 | import com.blibli.oss.qa.util.services.NetworkListener; 5 | import io.github.bonigarcia.wdm.WebDriverManager; 6 | import org.junit.jupiter.api.AfterEach; 7 | import org.junit.jupiter.api.BeforeAll; 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | import org.openqa.selenium.By; 11 | import org.openqa.selenium.Keys; 12 | import org.openqa.selenium.WebDriver; 13 | import org.openqa.selenium.WebElement; 14 | import org.openqa.selenium.chrome.ChromeDriver; 15 | import org.openqa.selenium.chrome.ChromeOptions; 16 | import org.openqa.selenium.remote.DesiredCapabilities; 17 | import org.openqa.selenium.remote.RemoteWebDriver; 18 | 19 | import java.net.MalformedURLException; 20 | import java.net.URL; 21 | import java.util.ArrayList; 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | import java.util.Optional; 25 | 26 | 27 | /** 28 | * Unit test for simple App. 29 | */ 30 | public class AppTestWithHarFilter { 31 | /** 32 | * Rigorous Test :-) 33 | */ 34 | 35 | private WebDriver driver; 36 | private NetworkListener networkListener; 37 | ChromeOptions options; 38 | DesiredCapabilities capabilities; 39 | @BeforeAll 40 | static void setupClass() { 41 | 42 | } 43 | 44 | @BeforeEach 45 | public void setup() { 46 | options = new ChromeOptions(); 47 | } 48 | 49 | @Test 50 | public void testWithLocalDriver() { 51 | setupLocalDriver(); 52 | driver = new ChromeDriver(options); 53 | driver.manage().window().maximize(); 54 | networkListener = new NetworkListener(driver, "harFilter.har"); 55 | networkListener.start(); 56 | driver.get("https://en.wiktionary.org/wiki/Wiktionary:Main_Page"); 57 | WebElement element = driver.findElement(By.id("searchInput")); 58 | element.sendKeys("Kiwi/n"); 59 | } 60 | 61 | private void setupLocalDriver(){ 62 | options.addArguments("--no-sandbox"); 63 | options.addArguments("--remote-allow-origins=*"); 64 | options.addArguments("--disable-dev-shm-usage"); 65 | if(Optional.ofNullable(System.getenv("CHROME_MODE")).orElse("").equalsIgnoreCase("headless")){ 66 | options.addArguments("--headless"); 67 | System.out.println("Running With headless mode"); 68 | }else{ 69 | System.out.println("Running Without headless mode"); 70 | } 71 | WebDriverManager.chromedriver().setup(); 72 | } 73 | 74 | @AfterEach 75 | public void tearDownWithFilter() { 76 | driver.quit(); 77 | try { 78 | Thread.sleep(2000); 79 | } catch (InterruptedException e) { 80 | throw new RuntimeException(e); 81 | } 82 | //filter parameter for createHarFile() to only print har request that contains the string into the har file 83 | networkListener.createHarFile("en.wiktionary.org"); 84 | try { 85 | Thread.sleep(2000); 86 | } catch (InterruptedException e) { 87 | throw new RuntimeException(e); 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## HAR Util For Selenium 4 2 | [![](https://jitpack.io/v/blibli-badak/selenium-har-util.svg)](https://jitpack.io/#blibli-badak/selenium-har-util) 3 | 4 | ---- 5 | ### Background : 6 | Currently, Selenium 4 is the most popular webdriver for automation. and now they support to communicate with the CDP browser. that able to communicate with CDP. 7 | So we can use CDP to communicate with the browser. one of that feature is intercept network 8 | with this feature we can get the network request and response and create HAR files for the network request. 9 | 10 | This util is supposed to be get network request and write a HAR file. 11 | 12 | ### Requirement: 13 | - Selenium 4 that support CDP 14 | - ChromeDriver 15 | - Google Chrome or any browser that support CDPSession ( not yet tested in Geckodriver or safari driver) 16 | - Java 11 17 | - Maven 18 | 19 | ### Instalation 20 | 21 | Add Repository for this dependency 22 | ```xml 23 | 24 | jitpack.io 25 | https://jitpack.io 26 | 27 | ``` 28 | 29 | And then add the maven dependency same with the lattest version in jitpack [![](https://jitpack.io/v/blibli-badak/selenium-har-util.svg)](https://jitpack.io/#blibli-badak/selenium-har-util) 30 | 31 | ```xml 32 | 33 | com.github.blibli-badak 34 | selenium-har-util 35 | 1.1.1 36 | 37 | ``` 38 | 39 | if you are use another tools , you can check this link for the instruction https://jitpack.io/#blibli-badak/selenium-har-util 40 | 41 | 42 | Create your Driver 43 | ```java 44 | driver = new ChromeDriver(options); 45 | ``` 46 | Integrate with our network listener 47 | ```java 48 | NetworkListener networkListener = new NetworkListener(driver, "har.har"); 49 | ``` 50 | Or if you already have existing CDP session you can use this one (Starting from version 1.1.1) 51 | ```java 52 | NetworkListener networkListener = new NetworkListener(driver, "Your Existing CDP Sessions","har.har" ); 53 | ``` 54 | [Optional] Setup the Default Charset (since 1.3.0) 55 | 56 | This is used for unicode support like chinese or japanese character, reffer to this for the value https://docs.oracle.com/javase/8/docs/technotes/guides/intl/encoding.doc.html 57 | ```java 58 | networkListener.setCharset("UTF-8"); 59 | ``` 60 | 61 | Start Capture your network request 62 | ```java 63 | networkListener.start(); 64 | ``` 65 | And Run your automation. 66 | After finishing your automation , you can create HAR files by using this method 67 | 68 | ```java 69 | driver.quit(); 70 | networkListener.createHarFile(); 71 | ``` 72 | 73 | (Starting from version 1.0.12) 74 | If you want to filter the HAR to only contain certain requests, you can add a string parameter in the createHarFile() function. Example: 75 | 76 | ```java 77 | driver.quit(); 78 | networkListener.createHarFile("en.wiktionary.org"); 79 | ``` 80 | 81 | And voila , in your project will be have new file called har.har , and you can inspect it via your favourite HAR viewer or you can open it via inspect element -> Network tab in your browser 82 | 83 | ### HAR validator 84 | Using Chrome: 85 | - Open Inspect Tab 86 | - Network tab 87 | - Click on import HAR file 88 | 89 | Using free web analyzer 90 | - https://toolbox.googleapps.com/apps/har_analyzer/ 91 | - http://www.softwareishard.com/har/viewer/ 92 | 93 | -------------------------------------------------------------------------------- /src/test/java/com/blibli/oss/qa/util/UsingCdpTest.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util; 2 | 3 | import com.blibli.oss.qa.util.services.NetworkListener; 4 | import io.github.bonigarcia.wdm.WebDriverManager; 5 | import org.junit.jupiter.api.AfterEach; 6 | import org.junit.jupiter.api.BeforeEach; 7 | import org.junit.jupiter.api.Test; 8 | import org.openqa.selenium.By; 9 | import org.openqa.selenium.WebDriver; 10 | import org.openqa.selenium.WebElement; 11 | import org.openqa.selenium.chrome.ChromeDriver; 12 | import org.openqa.selenium.chrome.ChromeOptions; 13 | import org.openqa.selenium.remote.DesiredCapabilities; 14 | import org.openqa.selenium.support.ui.ExpectedConditions; 15 | import org.openqa.selenium.support.ui.WebDriverWait; 16 | 17 | import java.time.Duration; 18 | import java.util.Optional; 19 | 20 | public class UsingCdpTest extends BaseTest{ 21 | 22 | private NetworkListener networkListener; 23 | private ChromeDriver driver; 24 | private ChromeOptions options; 25 | private DesiredCapabilities capabilities; 26 | 27 | public void setupLocalDriver(){ 28 | options = new ChromeOptions(); 29 | options.addArguments("--no-sandbox"); 30 | options.addArguments("--remote-allow-origins=*"); 31 | options.addArguments("--disable-dev-shm-usage"); 32 | if(Optional.ofNullable(System.getenv("CHROME_MODE")).orElse("").equalsIgnoreCase("headless")){ 33 | options.addArguments("--headless"); 34 | System.out.println("Running With headless mode"); 35 | }else{ 36 | System.out.println("Running Without headless mode"); 37 | } 38 | WebDriverManager.chromedriver().setup(); 39 | } 40 | 41 | 42 | public void testWithLocalDriver() { 43 | setupLocalDriver(); 44 | driver = new ChromeDriver(options); 45 | driver.manage().window().maximize(); 46 | networkListener = new NetworkListener(driver , driver.getDevTools(), "har-with-cdp.har"); 47 | networkListener.start(); 48 | driver.get("https://en.wiktionary.org/wiki/Wiktionary:Main_Page"); 49 | WebDriverWait webDriverWait = new WebDriverWait(driver, Duration.ofSeconds(30)); 50 | WebElement element =driver.findElement(By.id("searchInput")); 51 | webDriverWait.until(webDriver -> element.isDisplayed()); 52 | element.sendKeys("Kiwi/n"); 53 | } 54 | 55 | public void testLoginFeature() { 56 | setupLocalDriver(); 57 | driver = new ChromeDriver(options); 58 | driver.manage().window().maximize(); 59 | networkListener = new NetworkListener(driver , driver.getDevTools(), "har-with-cdp.har"); 60 | networkListener.start(); 61 | driver.get("https://stackoverflow.com/users/login"); 62 | WebDriverWait webDriverWait = new WebDriverWait(driver, Duration.ofSeconds(10)); 63 | 64 | webDriverWait.until(ExpectedConditions.visibilityOfElementLocated(By.id("email"))).sendKeys("Kiwi"); 65 | webDriverWait.until(ExpectedConditions.visibilityOfElementLocated(By.id("password"))).sendKeys("Kiwi"); 66 | 67 | webDriverWait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//button[@id='submit-button']"))).click(); 68 | 69 | webDriverWait.until(ExpectedConditions.visibilityOfElementLocated(By.xpath("//p[@class='flex--item s-input-message js-error-message ']"))); 70 | } 71 | 72 | @AfterEach 73 | public void tearDown() { 74 | driver.quit(); 75 | try { 76 | Thread.sleep(2000); 77 | } catch (InterruptedException e) { 78 | throw new RuntimeException(e); 79 | } 80 | networkListener.createHarFile(); 81 | // in the github actions we need add some wait , because chrome exited too slow , 82 | // so when we create new session previous chrome is not closed completly 83 | try { 84 | Thread.sleep(2000); 85 | } catch (InterruptedException e) { 86 | throw new RuntimeException(e); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /.github/workflows/maven-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a package using Maven and then publish it to GitHub packages when a release is created 2 | # For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path 3 | 4 | name: Maven Package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | env: 10 | ## Sets environment variable 11 | CHROME_MODE: HEADLESS 12 | 13 | jobs: 14 | build: 15 | 16 | runs-on: ubuntu-latest 17 | permissions: 18 | contents: read 19 | packages: write 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | with: 24 | # Use a PAT to push commits back to the repo. Create a repository secret 25 | # named `PUBLISH_PAT` with a token that has repo write permissions. 26 | # If you prefer the built-in GITHUB_TOKEN, remove this `token:` line. 27 | token: ${{ secrets.PUBLISH_PAT }} 28 | # allow the action to use the token for pushing back to the repo 29 | persist-credentials: true 30 | # fetch full history so we can push safely and create commits 31 | fetch-depth: 0 32 | - name: Prepare version from release tag 33 | id: prepare_version 34 | run: | 35 | # Use the release tag as VERSION. Tags are expected like "1.3.0". 36 | TAG="${{ github.event.release.tag_name }}" 37 | # strip leading 'v' or 'V' if present (safe even if tags don't have 'v') 38 | VERSION="${TAG#v}" 39 | VERSION="${VERSION#V}" 40 | echo "Original tag: $TAG" 41 | 42 | # Validate semantic versioning (semver) format: MAJOR.MINOR.PATCH with optional pre-release/build 43 | semver_regex='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' 44 | if [[ "$VERSION" =~ $semver_regex ]]; then 45 | echo "VERSION=$VERSION" >> $GITHUB_ENV 46 | echo "Prepared version: $VERSION" 47 | else 48 | echo "Error: release tag '$TAG' is not a valid semantic version (semver)." 49 | echo "Expected format: MAJOR.MINOR.PATCH (for example: 1.3.0)" 50 | exit 1 51 | fi 52 | 53 | - name: Set Maven project version from tag 54 | run: | 55 | # Update pom.xml in workspace to use the release tag version (no backup pom) 56 | mvn -B -DskipTests org.codehaus.mojo:versions-maven-plugin:2.14.2:set -DnewVersion="$VERSION" -DgenerateBackupPoms=false 57 | - name: Commit pom.xml version change 58 | run: | 59 | # Configure git author for the automated commit 60 | git config user.name "github-actions[bot]" || true 61 | git config user.email "41898282+github-actions[bot]@users.noreply.github.com" || true 62 | 63 | # Only commit if there are changes (the versions plugin modified the pom) 64 | if git status --porcelain | grep -q .; then 65 | git add pom.xml 66 | git commit -m "ci: set project version to $VERSION [skip ci]" || echo "commit failed or no changes" 67 | # Push to master. Note: this may fail if branch protection prevents pushes from GITHUB_TOKEN. 68 | git push origin HEAD:master || echo "push failed (possibly protected branch)" 69 | else 70 | echo "No pom.xml changes to commit" 71 | fi 72 | - name: Set up JDK 11 73 | uses: actions/setup-java@v3 74 | with: 75 | java-version: '11' 76 | distribution: 'temurin' 77 | server-id: github # Value of the distributionManagement/repository/id field of the pom.xml 78 | settings-path: ${{ github.workspace }} # location for the settings.xml file 79 | 80 | - uses: browser-actions/setup-chrome@latest 81 | - name: test chrome 82 | run: chrome --version 83 | 84 | - name: Build with Maven 85 | run: mvn -B package --file pom.xml -Dfile.encoding=UTF-8 86 | 87 | - name: Publish to GitHub Packages Apache Maven 88 | run: mvn deploy -s $GITHUB_WORKSPACE/settings.xml -Dfile.encoding=UTF-8 89 | env: 90 | GITHUB_TOKEN: ${{ github.token }} 91 | USERNAME: ${{ github.actor }} 92 | 93 | -------------------------------------------------------------------------------- /src/test/java/com/blibli/oss/qa/util/UnicodeTest.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util; 2 | 3 | import com.blibli.oss.qa.util.services.NetworkListener; 4 | import io.github.bonigarcia.wdm.WebDriverManager; 5 | import org.junit.jupiter.api.BeforeEach; 6 | import org.junit.jupiter.api.Test; 7 | import org.openqa.selenium.chrome.ChromeDriver; 8 | import org.openqa.selenium.chrome.ChromeOptions; 9 | import org.openqa.selenium.remote.DesiredCapabilities; 10 | 11 | import java.io.IOException; 12 | import java.util.Optional; 13 | 14 | import static org.junit.jupiter.api.Assertions.assertTrue; 15 | 16 | public class UnicodeTest extends BaseTest { 17 | private NetworkListener networkListener; 18 | private static String HAR_UNICODE_NAME = "har-unicode.har"; 19 | private ChromeDriver driver; 20 | private ChromeOptions options; 21 | private DesiredCapabilities capabilities; 22 | 23 | @BeforeEach 24 | public void setup() { 25 | this.setupLocalDriver(); 26 | } 27 | public void setupLocalDriver(){ 28 | options = new ChromeOptions(); 29 | options.addArguments("--no-sandbox"); 30 | options.addArguments("--remote-allow-origins=*"); 31 | options.addArguments("--disable-dev-shm-usage"); 32 | if(Optional.ofNullable(System.getenv("CHROME_MODE")).orElse("").equalsIgnoreCase("headless")){ 33 | options.addArguments("--headless"); 34 | System.out.println("Running With headless mode"); 35 | }else{ 36 | System.out.println("Running Without headless mode"); 37 | } 38 | WebDriverManager.chromedriver().setup(); 39 | } 40 | 41 | 42 | @Test 43 | public void testUnicode() throws InterruptedException, IOException { 44 | driver = new ChromeDriver(options); 45 | driver.manage().window().maximize(); 46 | networkListener = new NetworkListener(driver, driver.getDevTools(), HAR_UNICODE_NAME); 47 | networkListener.setCharset("UTF-8"); 48 | networkListener.start(); 49 | driver.get("https://gosoft.web.id/selenium/unicode.html"); 50 | Thread.sleep(5000); // make sure the web are loaded 51 | driver.quit(); 52 | try { 53 | Thread.sleep(2000); 54 | } catch (InterruptedException e) { 55 | throw new RuntimeException(e); 56 | } 57 | // in the github actions we need add some wait , because chrome exited too slow , 58 | // so when we create new session previous chrome is not closed completly 59 | try { 60 | Thread.sleep(2000); 61 | } catch (InterruptedException e) { 62 | throw new RuntimeException(e); 63 | } 64 | networkListener.createHarFile(); 65 | String harFile = this.readHarData(HAR_UNICODE_NAME); 66 | System.out.println(harFile); 67 | assertTrue(harFile.contains("接口路径不存在 请前往")); 68 | } 69 | 70 | @Test 71 | public void testUnicodeDemo() throws InterruptedException, IOException { 72 | driver = new ChromeDriver(options); 73 | driver.manage().window().maximize(); 74 | networkListener = new NetworkListener(driver, driver.getDevTools(), HAR_UNICODE_NAME); 75 | networkListener.setCharset("UTF-8"); 76 | networkListener.start(); 77 | driver.get("https://devtools.glitch.me/network/getstarted.html"); 78 | Thread.sleep(5000); // make sure the web are loaded 79 | driver.quit(); 80 | try { 81 | Thread.sleep(2000); 82 | } catch (InterruptedException e) { 83 | throw new RuntimeException(e); 84 | } 85 | // in the github actions we need add some wait , because chrome exited too slow , 86 | // so when we create new session previous chrome is not closed completly 87 | try { 88 | Thread.sleep(2000); 89 | } catch (InterruptedException e) { 90 | throw new RuntimeException(e); 91 | } 92 | networkListener.createHarFile(); 93 | String harFile = this.readHarData(HAR_UNICODE_NAME); 94 | System.out.println(harFile); 95 | assertTrue(!harFile.isEmpty()); 96 | } 97 | 98 | 99 | } 100 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 4.0.0 6 | 7 | com.blibli.oss.qa.util 8 | har-util 9 | 1.5.2 10 | 11 | har-util 12 | http://www.blibli.com 13 | 14 | 15 | UTF-8 16 | 11 17 | 11 18 | 4.38.0 19 | 20 | 21 | 22 | 23 | github 24 | blibli-badak 25 | https://maven.pkg.github.com/blibli-badak/selenium-har-util/ 26 | 27 | 28 | 29 | 30 | 31 | io.github.bonigarcia 32 | webdrivermanager 33 | 6.3.2 34 | 35 | 36 | 37 | org.seleniumhq.selenium 38 | selenium-java 39 | ${selenium.version} 40 | 41 | 42 | org.junit.jupiter 43 | junit-jupiter 44 | 5.10.2 45 | test 46 | 47 | 48 | 49 | de.sstoehr 50 | har-reader 51 | 2.3.0 52 | 53 | 54 | 55 | com.fasterxml.jackson.core 56 | jackson-databind 57 | 2.20.0 58 | 59 | 60 | org.projectlombok 61 | lombok 62 | 1.18.42 63 | provided 64 | 65 | 66 | org.seleniumhq.selenium 67 | selenium-devtools-v142 68 | ${selenium.version} 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | maven-clean-plugin 78 | 3.1.0 79 | 80 | 81 | 82 | maven-resources-plugin 83 | 3.0.2 84 | 85 | 86 | maven-compiler-plugin 87 | 3.8.0 88 | 89 | 90 | maven-surefire-plugin 91 | 2.22.1 92 | 93 | 94 | maven-jar-plugin 95 | 3.0.2 96 | 97 | 98 | maven-install-plugin 99 | 2.5.2 100 | 101 | 102 | maven-deploy-plugin 103 | 2.8.2 104 | 105 | 106 | 107 | maven-site-plugin 108 | 3.7.1 109 | 110 | 111 | maven-project-info-reports-plugin 112 | 3.0.0 113 | 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /src/test/java/com/blibli/oss/qa/util/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util; 2 | 3 | 4 | import com.blibli.oss.qa.util.services.NetworkListener; 5 | import io.github.bonigarcia.wdm.WebDriverManager; 6 | import org.junit.jupiter.api.AfterEach; 7 | import org.junit.jupiter.api.BeforeAll; 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | import org.openqa.selenium.By; 11 | import org.openqa.selenium.Keys; 12 | import org.openqa.selenium.WebDriver; 13 | import org.openqa.selenium.WebElement; 14 | import org.openqa.selenium.chrome.ChromeDriver; 15 | import org.openqa.selenium.chrome.ChromeOptions; 16 | import org.openqa.selenium.remote.DesiredCapabilities; 17 | import org.openqa.selenium.remote.RemoteWebDriver; 18 | import org.openqa.selenium.support.ui.WebDriverWait; 19 | 20 | import java.net.MalformedURLException; 21 | import java.net.URL; 22 | import java.nio.file.Files; 23 | import java.nio.file.Path; 24 | import java.nio.file.Paths; 25 | import java.time.Duration; 26 | import java.util.ArrayList; 27 | import java.util.HashMap; 28 | import java.util.Map; 29 | import java.util.Optional; 30 | 31 | import static org.junit.jupiter.api.Assertions.assertFalse; 32 | 33 | 34 | /** 35 | * Unit test for simple App. 36 | */ 37 | public class AppTest { 38 | /** 39 | * Rigorous Test :-) 40 | */ 41 | 42 | private WebDriver driver; 43 | private NetworkListener networkListener; 44 | ChromeOptions options; 45 | DesiredCapabilities capabilities; 46 | @BeforeAll 47 | static void setupClass() { 48 | 49 | } 50 | 51 | @BeforeEach 52 | public void setup() { 53 | options = new ChromeOptions(); 54 | } 55 | 56 | @Test 57 | public void testWithLocalDriver() { 58 | setupLocalDriver(); 59 | driver = new ChromeDriver(options); 60 | driver.manage().window().maximize(); 61 | networkListener = new NetworkListener(driver, "har-local-driver.har"); 62 | networkListener.start(); 63 | driver.get("https://en.wiktionary.org/wiki/Wiktionary:Main_Page"); 64 | // WebDriverWait webDriverWait = new WebDriverWait(driver, Duration.ofSeconds(30)); 65 | // WebElement element =driver.findElement(By.id("searchInput")); 66 | // webDriverWait.until(webDriver -> element.isDisplayed()); 67 | // element.sendKeys("Kiwi/n"); 68 | Path path = Paths.get("har-local-driver.har"); 69 | assertFalse(Files.exists(path)); 70 | /** 71 | * Todo : https://www.browserstack.com/docs/automate/selenium/event-driven-testing#intercept-network ubah ChromeDriver jadi webdriver biasa , trus ganti devtoolsnya 72 | * Todo : Tambahin test jika buka 2 tab 73 | * 1. paksa listen di port yg sama 74 | * 2. beda port cdp - beda har 75 | */ 76 | } 77 | 78 | private void setupLocalDriver(){ 79 | options.addArguments("--no-sandbox"); 80 | options.addArguments("--remote-allow-origins=*"); 81 | options.addArguments("--disable-dev-shm-usage"); 82 | if(Optional.ofNullable(System.getenv("CHROME_MODE")).orElse("").equalsIgnoreCase("headless")){ 83 | options.addArguments("--headless"); 84 | System.out.println("Running With headless mode"); 85 | }else{ 86 | System.out.println("Running Without headless mode"); 87 | } 88 | WebDriverManager.chromedriver().setup(); 89 | } 90 | 91 | @Test 92 | public void testWithOpenNewTab(){ 93 | setupLocalDriver(); 94 | driver = new ChromeDriver(options); 95 | networkListener = new NetworkListener(driver,"har-new-tab.har"); 96 | networkListener.start(); 97 | driver.manage().window().maximize(); 98 | driver.get("http://gosoft.web.id/selenium/"); 99 | WebElement linkNewTab = driver.findElement(By.id("new-tab")); 100 | linkNewTab.click(); 101 | ArrayList tabs2 = new ArrayList<>(driver.getWindowHandles()); 102 | driver.switchTo().window(tabs2.get(1)); 103 | networkListener.start(); 104 | driver.navigate().refresh(); 105 | System.out.println("Completed test new Tab"); 106 | } 107 | 108 | @Test 109 | public void seleniumTest() { 110 | setupLocalDriver(); 111 | driver = new ChromeDriver(options); 112 | networkListener = new NetworkListener(driver, "har-new-tab.har"); 113 | networkListener.start(driver.getWindowHandle()); 114 | driver.get("http://gosoft.web.id/selenium/"); 115 | WebElement linkNewTab = driver.findElement(By.id("new-tab")); 116 | linkNewTab.click(); 117 | ArrayList tabs2 = new ArrayList<>(driver.getWindowHandles()); 118 | driver.switchTo().window(tabs2.get(1)); 119 | networkListener.switchWindow(driver.getWindowHandle()); 120 | 121 | driver.quit(); 122 | } 123 | 124 | 125 | @Test 126 | public void testNotFound(){ 127 | setupLocalDriver(); 128 | driver = new ChromeDriver(options); 129 | networkListener = new NetworkListener(driver,"har-not-found.har"); 130 | networkListener.start(); 131 | driver.manage().window().maximize(); 132 | driver.get("http://gosoft.web.id/error-not-found/"); 133 | System.out.println("Completed test not found"); 134 | } 135 | 136 | // @Test 137 | public void tryUsingRemoteAccess() throws MalformedURLException { 138 | Map prefs = new HashMap(); 139 | prefs.put("enableVNC", true); 140 | prefs.put("enableVideo", false); 141 | prefs.put("sessionTimeout", "120s"); 142 | capabilities = new DesiredCapabilities(); 143 | capabilities.setCapability("browserName", "chrome"); 144 | capabilities.setCapability("browserVersion", "96.0"); 145 | capabilities.setCapability("selenoid:options", prefs); 146 | options.merge(capabilities); 147 | // Todo : Check Selenoid implementation https://github.com/SeleniumHQ/selenium/issues/9803#issuecomment-1015300383 148 | String seleniumUrl = Optional.ofNullable(System.getenv("SE_REMOTE_URL")).orElse("http://localhost:4444/wd/hub/"); 149 | System.out.println("Running on Remote " + seleniumUrl); 150 | driver = new RemoteWebDriver(new URL(seleniumUrl), capabilities); 151 | // Todo : change to this one https://github.com/aerokube/chrome-developer-tools-protocol-java-example/blob/master/src/test/java/com/aerokube/selenoid/ChromeDevtoolsTest.java 152 | networkListener = new NetworkListener(driver, "har.har",Optional.ofNullable(System.getenv("SE_BASE_URL")).orElse("localhost:4444")); 153 | driver.get("https://en.wiktionary.org/wiki/Wiktionary:Main_Page"); 154 | networkListener.start(); 155 | driver.get("https://en.wiktionary.org/wiki/Wiktionary:Main_Page"); 156 | WebElement element = driver.findElement(By.id("searchInput")); 157 | element.sendKeys("Kiwi"); 158 | element.sendKeys(Keys.RETURN); 159 | try { 160 | Thread.sleep(10000); 161 | } catch (InterruptedException e) { 162 | e.printStackTrace(); 163 | } 164 | } 165 | 166 | @AfterEach 167 | public void tearDown() { 168 | driver.quit(); 169 | try { 170 | Thread.sleep(2000); 171 | } catch (InterruptedException e) { 172 | throw new RuntimeException(e); 173 | } 174 | networkListener.createHarFile(); 175 | // in the github actions we need add some wait , because chrome exited too slow , 176 | // so when we create new session previous chrome is not closed completly 177 | try { 178 | Thread.sleep(2000); 179 | } catch (InterruptedException e) { 180 | throw new RuntimeException(e); 181 | } 182 | } 183 | 184 | } 185 | -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/services/HarEntryConverter.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.services; 2 | 3 | import de.sstoehr.harreader.model.HarContent; 4 | import de.sstoehr.harreader.model.HarEntry; 5 | import de.sstoehr.harreader.model.HarHeader; 6 | import de.sstoehr.harreader.model.HarPostData; 7 | import de.sstoehr.harreader.model.HarRequest; 8 | import de.sstoehr.harreader.model.HarResponse; 9 | import de.sstoehr.harreader.model.HarTiming; 10 | import de.sstoehr.harreader.model.HttpMethod; 11 | import org.openqa.selenium.devtools.v142.network.model.*; 12 | 13 | import java.io.IOException; 14 | import java.io.InputStream; 15 | import java.util.ArrayList; 16 | import java.util.Date; 17 | import java.util.List; 18 | import java.util.Optional; 19 | 20 | public class HarEntryConverter { 21 | private final Request request; 22 | private final Response response; 23 | private final LoadingFailed loadingFailed; 24 | private final ResponseReceivedExtraInfo responseReceivedExtraInfo; 25 | 26 | private final HarEntry harEntry; 27 | private final long startTime; 28 | private final String pageRef; 29 | private final String responseBody; 30 | private long endTime = 0; 31 | private Optional timing; 32 | 33 | public HarEntryConverter(Request request, 34 | Response response, 35 | LoadingFailed loadingFailed, 36 | ResponseReceivedExtraInfo responseReceivedExtraInfo, 37 | List time, 38 | String pageRef, 39 | String responseBody) { 40 | this.loadingFailed = loadingFailed; 41 | this.responseReceivedExtraInfo = responseReceivedExtraInfo; 42 | if (time.size() == 0) { 43 | time.add(0L); 44 | } 45 | harEntry = new HarEntry(); 46 | this.request = request; 47 | this.startTime = time.get(0); 48 | this.response = response; 49 | this.responseBody = responseBody; 50 | if (response != null) { 51 | this.timing = response.getTiming(); 52 | if (timing.isPresent()) { 53 | ResourceTiming resourceTiming = timing.get(); 54 | Number receiveHeadersEnd = resourceTiming.getReceiveHeadersEnd(); 55 | this.endTime = time.get(0) + receiveHeadersEnd.longValue(); 56 | } 57 | } 58 | this.pageRef = pageRef; 59 | } 60 | 61 | public void setup() { 62 | harEntry.setRequest(convertHarRequest()); 63 | harEntry.setStartedDateTime(new Date(startTime)); 64 | harEntry.setTime((int) (endTime - startTime)); 65 | harEntry.setRequest(convertHarRequest()); 66 | if (response != null) { 67 | harEntry.setResponse(convertHarResponse()); 68 | } else { 69 | harEntry.setResponse(emptyHarResponse()); 70 | harEntry.setAdditionalField("_resourceType", "other"); 71 | } 72 | harEntry.setTimings(convertHarTiming()); 73 | harEntry.setPageref(pageRef); 74 | } 75 | 76 | public HarTiming convertHarTiming() { 77 | HarTiming harTiming = new HarTiming(); 78 | if (timing == null || timing.isEmpty()) { 79 | harTiming.setDns(-1); 80 | harTiming.setConnect(-1); 81 | harTiming.setSend(-1); 82 | harTiming.setWait(-1); 83 | harTiming.setReceive(-1); 84 | return harTiming; 85 | } 86 | harTiming.setDns(timing.get().getDnsStart().intValue()); 87 | harTiming.setConnect(timing.get().getConnectStart().intValue()); 88 | harTiming.setSend(timing.get().getSendStart().intValue()); 89 | harTiming.setWait( 90 | timing.get().getSendEnd().intValue() - timing.get().getSendStart().intValue()); 91 | harTiming.setReceive( 92 | timing.get().getReceiveHeadersEnd().intValue() - timing.get().getSendEnd().intValue()); 93 | return harTiming; 94 | } 95 | 96 | private HarResponse convertHarResponse() { 97 | HarResponse harResponse = new HarResponse(); 98 | harResponse.setStatus(response.getStatus()); 99 | harResponse.setStatusText(response.getStatusText()); 100 | harResponse.setHttpVersion("HTTP/1.1"); 101 | harResponse.setRedirectURL(""); 102 | harResponse.setHeaders(convertHarHeadersResponse(response.getHeaders())); 103 | harResponse.setContent(setHarContentResponse()); 104 | return harResponse; 105 | } 106 | 107 | private HarResponse emptyHarResponse() { 108 | HarResponse harResponse = new HarResponse(); 109 | if (responseReceivedExtraInfo != null) { 110 | harResponse.setStatus(responseReceivedExtraInfo.getStatusCode()); 111 | harResponse.setStatusText(""); 112 | harResponse.setHttpVersion(""); 113 | harResponse.setRedirectURL(""); 114 | harResponse.setHeaders(convertHarHeadersResponse(responseReceivedExtraInfo.getHeaders())); 115 | harResponse.setContent(setEmptyHarContentResponse()); 116 | if (loadingFailed != null) { 117 | harResponse.setAdditionalField("_error", loadingFailed.getErrorText()); 118 | harResponse.setAdditionalField("_fetchedViaServiceWorker", false); 119 | harResponse.setAdditionalField("_transferSize", 0); 120 | } 121 | }else { 122 | harResponse.setStatus(0); 123 | harResponse.setStatusText(""); 124 | harResponse.setHttpVersion("HTTP/1.1"); 125 | harResponse.setRedirectURL(""); 126 | harResponse.setHeaders(new ArrayList<>()); 127 | harResponse.setContent(setEmptyHarContentResponse()); 128 | } 129 | return harResponse; 130 | } 131 | 132 | private String convertInputStreamtoString(InputStream inputStream) { 133 | StringBuilder stringBuilder = new StringBuilder(); 134 | try { 135 | int read; 136 | while ((read = inputStream.read()) != -1) { 137 | stringBuilder.append((char) read); 138 | } 139 | } catch (IOException e) { 140 | e.printStackTrace(); 141 | } 142 | return stringBuilder.toString(); 143 | } 144 | 145 | private HarContent setHarContentResponse() { 146 | HarContent harContent = new HarContent(); 147 | harContent.setSize((long) response.getEncodedDataLength()); 148 | harContent.setText(responseBody); 149 | harContent.setMimeType(response.getMimeType()); 150 | return harContent; 151 | } 152 | 153 | private HarContent setEmptyHarContentResponse() { 154 | HarContent harContent = new HarContent(); 155 | harContent.setSize((long) 0); 156 | harContent.setMimeType("x-unknown"); 157 | return harContent; 158 | } 159 | 160 | private List convertHarHeadersResponse(Headers headers) { 161 | List harHeaders = new java.util.ArrayList<>(); 162 | headers.forEach((key, value) -> { 163 | HarHeader harHeader = new HarHeader(); 164 | harHeader.setName(key); 165 | harHeader.setValue(value.toString()); 166 | harHeaders.add(harHeader); 167 | }); 168 | return harHeaders; 169 | } 170 | 171 | public HarRequest convertHarRequest() { 172 | HarRequest harRequest = new HarRequest(); 173 | harRequest.setMethod(HttpMethod.valueOf(request.getMethod())); 174 | harRequest.setUrl(request.getUrl()); 175 | harRequest.setComment(""); 176 | harRequest.setHttpVersion("HTTP/1.1"); 177 | harRequest.setPostData(request.getHasPostData().orElse(false) ? setHarPostData() : null); 178 | List headers = new ArrayList<>(); 179 | request.getHeaders().entrySet().forEach(entry -> { 180 | HarHeader header = new HarHeader(); 181 | header.setName(entry.getKey()); 182 | header.setValue(entry.getValue().toString()); 183 | headers.add(header); 184 | }); 185 | harRequest.setHeaders(headers); 186 | return harRequest; 187 | } 188 | 189 | private HarPostData setHarPostData() { 190 | HarPostData harPostData = new HarPostData(); 191 | String postDataText = request.getPostDataEntries() 192 | .flatMap(entries -> entries.isEmpty() ? java.util.Optional.empty() : entries.get(0).getBytes()) 193 | .orElse(""); 194 | harPostData.setText(postDataText); 195 | return harPostData; 196 | } 197 | 198 | public HarEntry getHarEntry() { 199 | return harEntry; 200 | } 201 | 202 | } 203 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/main/java/com/blibli/oss/qa/util/services/NetworkListener.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util.services; 2 | 3 | import com.blibli.oss.qa.util.model.Constant; 4 | import com.blibli.oss.qa.util.model.HarModel; 5 | import com.blibli.oss.qa.util.model.RequestResponsePair; 6 | import com.blibli.oss.qa.util.model.RequestResponseStorage; 7 | import com.fasterxml.jackson.databind.ObjectMapper; 8 | import de.sstoehr.harreader.model.Har; 9 | import de.sstoehr.harreader.model.HarCreatorBrowser; 10 | import de.sstoehr.harreader.model.HarEntry; 11 | import de.sstoehr.harreader.model.HarLog; 12 | import de.sstoehr.harreader.model.HarPage; 13 | import de.sstoehr.harreader.model.HarPageTiming; 14 | import lombok.extern.slf4j.Slf4j; 15 | import org.openqa.selenium.MutableCapabilities; 16 | import org.openqa.selenium.WebDriver; 17 | import org.openqa.selenium.chromium.ChromiumDriver; 18 | import org.openqa.selenium.devtools.DevTools; 19 | import org.openqa.selenium.devtools.HasDevTools; 20 | import org.openqa.selenium.devtools.v142.network.Network; 21 | import org.openqa.selenium.devtools.v142.network.model.*; 22 | import org.openqa.selenium.remote.Augmenter; 23 | import org.openqa.selenium.remote.RemoteWebDriver; 24 | 25 | import java.io.BufferedReader; 26 | import java.io.IOException; 27 | import java.io.InputStream; 28 | import java.io.InputStreamReader; 29 | import java.lang.reflect.Field; 30 | import java.nio.file.Files; 31 | import java.util.ArrayList; 32 | import java.util.Date; 33 | import java.util.HashMap; 34 | import java.util.List; 35 | import java.util.Map; 36 | import java.util.Optional; 37 | import java.util.concurrent.ConcurrentHashMap; 38 | 39 | @Slf4j 40 | public class NetworkListener { 41 | static final String TARGET_PATH_FILE = System.getProperty("user.dir") + "/target/"; 42 | private final ConcurrentHashMap, HarModel> harModelHashMap = new ConcurrentHashMap<>(); 43 | private WebDriver driver; 44 | private String baseRemoteUrl; 45 | private DevTools devTools; 46 | private String harFile = ""; 47 | private String charset = Constant.DEFAULT_UNICODE; 48 | 49 | private HarCreatorBrowser harCreatorBrowser; 50 | 51 | private final Map windowHandleStorageMap = new ConcurrentHashMap<>(); 52 | private RequestResponseStorage requestResponseStorage; 53 | 54 | 55 | /** 56 | * Generate new network listener object 57 | * 58 | * @param driver chrome driver that you are using 59 | * @param harFileName file will be stored under target folder 60 | */ 61 | public NetworkListener(WebDriver driver, String harFileName) { 62 | this.driver = driver; 63 | this.harFile = harFileName; 64 | try { 65 | Files.delete(java.nio.file.Paths.get(harFile)); 66 | } catch (IOException e) { 67 | // Since it's expected to be failed , log level info is good 68 | log.info("Not able to find prevous har file " + e.getMessage()); 69 | } 70 | devTools = ((ChromiumDriver) driver).getDevTools(); 71 | createHarBrowser(); 72 | } 73 | public NetworkListener(WebDriver driver , DevTools devTools , String harFileName){ 74 | this.devTools = devTools; 75 | this.driver = driver; 76 | this.harFile = harFileName; 77 | try { 78 | Files.delete(java.nio.file.Paths.get(harFile)); 79 | } catch (IOException e) { 80 | // Since it's expected to be failed , log level info is good 81 | log.info("Not able to find prevous har file " + e.getMessage()); 82 | } 83 | createHarBrowser(); 84 | } 85 | 86 | 87 | /** 88 | * Generate new network listener object 89 | * 90 | * @param driver chrome driver that you are using 91 | * @param harFileName file will be stored under target folder 92 | * @param baseRemoteUrl Base Selenoid URl that you are using 93 | */ 94 | public NetworkListener(WebDriver driver, String harFileName, String baseRemoteUrl) { 95 | this.driver = driver; 96 | this.harFile = harFileName; 97 | this.baseRemoteUrl = baseRemoteUrl; 98 | try { 99 | Files.delete(java.nio.file.Paths.get(harFile)); 100 | } catch (IOException e) { 101 | // Since it's expected to be failed , log level info is good 102 | log.info("Not able to find prevous har file " + e.getMessage()); 103 | } 104 | createHarBrowser(); 105 | } 106 | 107 | 108 | /** 109 | * Generate new network listener object 110 | * 111 | * @param harFileName file will be stored under target folder 112 | */ 113 | 114 | public NetworkListener(String harFileName) { 115 | this.harFile = harFileName; 116 | } 117 | 118 | private static String convertInputStreamToString(InputStream inputStream) throws IOException { 119 | StringBuilder stringBuilder = new StringBuilder(); 120 | try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream))) { 121 | String line = bufferedReader.readLine(); 122 | while (line != null) { 123 | stringBuilder.append(line); 124 | line = bufferedReader.readLine(); 125 | } 126 | } 127 | return stringBuilder.toString(); 128 | } 129 | 130 | public void start() { 131 | // main listener to intercept response and continue 132 | // initializeCdp(); 133 | // Filter filterResponses = next -> req -> { 134 | // Long startTime = System.currentTimeMillis(); 135 | // HttpResponse res = next.execute(req); 136 | // Long endTime = System.currentTimeMillis(); 137 | // harModelHashMap.put(Lists.newArrayList(startTime, endTime), new HarModel(req, res)); 138 | // return res; 139 | // }; 140 | // NetworkInterceptor networkInterceptor = new NetworkInterceptor(driver, filterResponses); 141 | start(driver.getWindowHandle()); 142 | } 143 | 144 | public void start(String windowHandle) { 145 | initializeCdp(); 146 | devTools.createSession(windowHandle); 147 | devTools.send(Network.enable(Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty())); 148 | devTools.clearListeners(); 149 | 150 | requestResponseStorage = windowHandleStorageMap.get(windowHandle); 151 | if (requestResponseStorage == null) { 152 | requestResponseStorage = new RequestResponseStorage(); 153 | windowHandleStorageMap.put(windowHandle, requestResponseStorage); 154 | 155 | devTools.addListener(Network.requestWillBeSent(), requestConsumer -> { 156 | String requestId = String.valueOf(requestConsumer.getRequestId()); 157 | Request request = requestConsumer.getRequest(); 158 | requestResponseStorage.addRequest(requestId, request, new Date()); 159 | }); 160 | 161 | devTools.addListener(Network.responseReceived(), responseConsumer -> { 162 | Response response = responseConsumer.getResponse(); 163 | String responseBody = 164 | devTools.send(Network.getResponseBody(responseConsumer.getRequestId())) 165 | .getBody(); 166 | requestResponseStorage.addResponse(response, responseBody); 167 | }); 168 | 169 | devTools.addListener(Network.loadingFailed(), loadingFailedConsumer -> { 170 | requestResponseStorage.addLoadingFailed(loadingFailedConsumer); 171 | }); 172 | 173 | devTools.addListener(Network.responseReceivedExtraInfo(), responseReceivedExtraInfoConsumer -> { 174 | requestResponseStorage.addresponseReceivedExtraInfo(responseReceivedExtraInfoConsumer); 175 | }); 176 | } 177 | } 178 | 179 | private void initializeCdp(){ 180 | if(this.devTools != null ){ 181 | devTools.createSessionIfThereIsNotOne(); 182 | return; 183 | } 184 | try { 185 | if (driver instanceof RemoteWebDriver) { 186 | this.devTools = getCdpUsingCustomurl(); 187 | } else { 188 | this.devTools = ((HasDevTools) driver).getDevTools(); 189 | } 190 | devTools.createSessionIfThereIsNotOne(); 191 | } catch (Exception e) { 192 | log.error("CDP Can't be initialized " , e); 193 | } 194 | } 195 | 196 | public DevTools getCdpUsingCustomurl() { 197 | // Before proceeding, reach into the driver and manipulate the capabilities to 198 | // include the se:cdp and se:cdpVersion keys. 199 | try { 200 | Field capabilitiesField = RemoteWebDriver.class.getDeclaredField("capabilities"); 201 | capabilitiesField.setAccessible(true); 202 | String sessionId = ((RemoteWebDriver) driver).getSessionId().toString(); 203 | String devtoolsUrl = String.format("ws://%s/devtools/%s/page", baseRemoteUrl, sessionId); 204 | 205 | MutableCapabilities mutableCapabilities = (MutableCapabilities) capabilitiesField.get(driver); 206 | mutableCapabilities.setCapability("se:cdp", devtoolsUrl); 207 | mutableCapabilities.setCapability("se:cdpVersion", mutableCapabilities.getBrowserVersion()); 208 | } catch (Exception e) { 209 | log.info("Failed to spoof RemoteWebDriver capabilities :sadpanda:"); 210 | } 211 | 212 | // Proceed to "augment" the driver and get a dev tools client ... 213 | RemoteWebDriver augmenteDriver = (RemoteWebDriver) new Augmenter().augment(driver); 214 | DevTools devTools = ((HasDevTools) augmenteDriver).getDevTools(); 215 | this.driver = augmenteDriver; 216 | return devTools; 217 | } 218 | 219 | public WebDriver getDriver() { 220 | return driver; 221 | } 222 | 223 | public void setDriver(WebDriver driver) { 224 | this.driver = driver; 225 | } 226 | 227 | public void switchWindow(String windowHandle) { 228 | start(windowHandle); 229 | driver.navigate().refresh(); 230 | } 231 | 232 | public void createHarFile() { 233 | Har har = new Har(); 234 | HarLog harLog = new HarLog(); 235 | harLog.setCreator(harCreatorBrowser); 236 | harLog.setBrowser(harCreatorBrowser); 237 | List harPages = new ArrayList<>(); 238 | List harEntries = new ArrayList<>(); 239 | 240 | // Create a thread-safe snapshot of the map to prevent ConcurrentModificationException 241 | new HashMap<>(windowHandleStorageMap).forEach((windowHandle, reqResStorage) -> { 242 | harPages.add(createHarPage(windowHandle)); 243 | // Create a thread-safe snapshot of the request-response pairs list 244 | new ArrayList<>(reqResStorage.getRequestResponsePairs()).forEach(pair -> { 245 | harEntries.addAll(saveHarEntry(pair, windowHandle)); 246 | }); 247 | }); 248 | log.info("har entry size : {}", harEntries.size()); 249 | harLog.setPages(harPages); 250 | harLog.setEntries(harEntries); 251 | har.setLog(harLog); 252 | createFile(har); 253 | } 254 | 255 | public void createHarFile(String filter) { 256 | System.out.println("---- createHarFile - Filter ----"); 257 | Har har = new Har(); 258 | HarLog harLog = new HarLog(); 259 | harLog.setCreator(harCreatorBrowser); 260 | harLog.setBrowser(harCreatorBrowser); 261 | List harPages = new ArrayList<>(); 262 | List harEntries = new ArrayList<>(); 263 | 264 | // Create a thread-safe snapshot of the map to prevent ConcurrentModificationException 265 | new HashMap<>(windowHandleStorageMap).forEach((windowHandle, reqResStorage) -> { 266 | harPages.add(createHarPage(windowHandle)); 267 | // Create a thread-safe snapshot of the request-response pairs list 268 | new ArrayList<>(reqResStorage.getRequestResponsePairs()).forEach(pair -> { 269 | if (pair.getRequest().getUrl().contains(filter)) { 270 | harEntries.addAll(saveHarEntry(pair,windowHandle)); 271 | } 272 | }); 273 | }); 274 | log.info("har entry size : {}", harEntries.size()); 275 | harLog.setPages(harPages); 276 | 277 | harLog.setEntries(harEntries); 278 | har.setLog(harLog); 279 | createFile(har); 280 | } 281 | 282 | private List saveHarEntry(RequestResponsePair pair, String windowHandle){ 283 | List result = new ArrayList<>(); 284 | List time = new ArrayList<>(); 285 | if (pair.getResponse() != null) { 286 | pair.getResponse().getTiming().ifPresent(timing -> { 287 | time.add(pair.getRequestOn().getTime()); 288 | time.add(timing.getReceiveHeadersEnd().longValue()); 289 | }); 290 | result.add(createHarEntry(pair.getRequest(), 291 | pair.getResponse(), 292 | time, 293 | windowHandle, 294 | pair.getResponseBody(), 295 | pair.getLoadingFailed(), 296 | pair.getResponseReceivedExtraInfo())); 297 | } else { 298 | result.add(createHarEntry(pair.getRequest(), 299 | null, 300 | time, 301 | windowHandle, 302 | null, 303 | pair.getLoadingFailed(), 304 | pair.getResponseReceivedExtraInfo())); 305 | } 306 | return result; 307 | } 308 | 309 | private void createFile(Har har) { 310 | ObjectMapper om = new ObjectMapper(); 311 | try { 312 | String json = new String(om.writeValueAsString(har).getBytes(), charset); 313 | // write json to file 314 | Files.write(java.nio.file.Paths.get(harFile), json.getBytes()); 315 | } catch (IOException e) { 316 | e.printStackTrace(); 317 | } 318 | } 319 | 320 | /** 321 | * Setup Charset on the file generation 322 | * @param charset default will be UTF-8 , you can get from here https://docs.oracle.com/javase/8/docs/api/java/nio/charset/Charset.html 323 | */ 324 | public void setCharset(String charset){ 325 | this.charset = charset; 326 | } 327 | 328 | private void createHarBrowser() { 329 | harCreatorBrowser = new HarCreatorBrowser(); 330 | harCreatorBrowser.setName("gdn-qa-automation"); 331 | harCreatorBrowser.setVersion("0.0.1"); 332 | harCreatorBrowser.setComment("Created by HAR utils"); 333 | } 334 | 335 | public HarPage createHarPage(String title) { 336 | HarPage harPage = new HarPage(); 337 | harPage.setComment("Create by Har Utils"); 338 | HarPageTiming harPageTiming = new HarPageTiming(); 339 | harPageTiming.setOnContentLoad(0); 340 | harPage.setPageTimings(harPageTiming); 341 | harPage.setStartedDateTime(new Date()); 342 | harPage.setId(title); 343 | return harPage; 344 | } 345 | 346 | public HarEntry createHarEntry(Request request, 347 | Response response, 348 | List time, 349 | String pagref, 350 | String responseBody, 351 | LoadingFailed loadingFailed, 352 | ResponseReceivedExtraInfo responseReceivedExtraInfo) { 353 | HarEntryConverter harEntry = 354 | new HarEntryConverter(request, response,loadingFailed, responseReceivedExtraInfo, time, pagref, responseBody); 355 | harEntry.setup(); 356 | return harEntry.getHarEntry(); 357 | } 358 | 359 | 360 | /** 361 | * @param networkListener - NetworkListener 362 | * @param driver browser driver 363 | * @param tabIndex num tab of your destination, 0 is first tab index 364 | */ 365 | public static void switchTab(NetworkListener networkListener, WebDriver driver, Integer tabIndex) { 366 | driver.switchTo().window(new ArrayList<>(driver.getWindowHandles()).get(tabIndex)); 367 | networkListener.start(); 368 | driver.navigate().refresh(); 369 | } 370 | 371 | public DevTools getDevTools() { 372 | return devTools; 373 | } 374 | } 375 | -------------------------------------------------------------------------------- /src/test/java/com/blibli/oss/qa/util/ThreadSafeHarCreationTest.java: -------------------------------------------------------------------------------- 1 | package com.blibli.oss.qa.util; 2 | 3 | import com.blibli.oss.qa.util.model.RequestResponsePair; 4 | import com.blibli.oss.qa.util.model.RequestResponseStorage; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import de.sstoehr.harreader.HarReader; 7 | import de.sstoehr.harreader.model.Har; 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.Test; 10 | import org.openqa.selenium.devtools.v142.network.model.Request; 11 | import org.openqa.selenium.devtools.v142.network.model.Response; 12 | 13 | import java.io.File; 14 | import java.lang.reflect.Field; 15 | import java.nio.file.Files; 16 | import java.nio.file.Paths; 17 | import java.util.Date; 18 | import java.util.Map; 19 | import java.util.concurrent.ConcurrentHashMap; 20 | import java.util.concurrent.CopyOnWriteArrayList; 21 | import java.util.concurrent.CountDownLatch; 22 | import java.util.concurrent.ExecutorService; 23 | import java.util.concurrent.Executors; 24 | import java.util.concurrent.atomic.AtomicInteger; 25 | 26 | import static org.junit.jupiter.api.Assertions.*; 27 | 28 | /** 29 | * Thread-safety tests for HAR file creation. 30 | * Verifies that ConcurrentModificationException is not thrown when the browser 31 | * continues sending data while the HAR file is being written. 32 | */ 33 | public class ThreadSafeHarCreationTest { 34 | 35 | private RequestResponseStorage storage; 36 | private static final String TEST_HAR_FILE = "test-thread-safe.har"; 37 | 38 | @BeforeEach 39 | public void setup() { 40 | storage = new RequestResponseStorage(); 41 | // Clean up test file if it exists 42 | try { 43 | Files.deleteIfExists(Paths.get(TEST_HAR_FILE)); 44 | } catch (Exception e) { 45 | // Ignore 46 | } 47 | } 48 | 49 | /** 50 | * Test that RequestResponseStorage uses CopyOnWriteArrayList 51 | * which allows safe concurrent modifications. 52 | */ 53 | @Test 54 | public void testRequestResponseStorageUsesCopyOnWriteArrayList() { 55 | // Verify that the list is CopyOnWriteArrayList 56 | assertTrue(storage.getRequestResponsePairs() instanceof CopyOnWriteArrayList, 57 | "RequestResponseStorage should use CopyOnWriteArrayList for thread safety"); 58 | } 59 | 60 | /** 61 | * Test concurrent read and write operations on RequestResponseStorage. 62 | * Simulates the scenario where the browser sends new requests while 63 | * the HAR file is being serialized. 64 | */ 65 | @Test 66 | public void testConcurrentAddAndIterateRequests() throws InterruptedException { 67 | int numThreads = 5; 68 | int requestsPerThread = 20; 69 | ExecutorService executorService = Executors.newFixedThreadPool(numThreads); 70 | CountDownLatch startLatch = new CountDownLatch(1); 71 | CountDownLatch endLatch = new CountDownLatch(numThreads + 1); 72 | AtomicInteger iterationCount = new AtomicInteger(0); 73 | AtomicInteger exceptionCount = new AtomicInteger(0); 74 | 75 | // Start threads that add requests 76 | for (int i = 0; i < numThreads; i++) { 77 | final int threadId = i; 78 | executorService.submit(() -> { 79 | try { 80 | startLatch.await(); 81 | for (int j = 0; j < requestsPerThread; j++) { 82 | RequestResponsePair pair = new RequestResponsePair( 83 | "request-" + threadId + "-" + j, 84 | null, 85 | new Date(), 86 | null 87 | ); 88 | storage.getRequestResponsePairs().add(pair); 89 | Thread.yield(); 90 | } 91 | } catch (InterruptedException e) { 92 | Thread.currentThread().interrupt(); 93 | } finally { 94 | endLatch.countDown(); 95 | } 96 | }); 97 | } 98 | 99 | // Start a thread that iterates over the list (simulating HAR serialization) 100 | executorService.submit(() -> { 101 | try { 102 | startLatch.await(); 103 | Thread.sleep(50); // Let some requests accumulate 104 | for (int iteration = 0; iteration < 10; iteration++) { 105 | try { 106 | // This should not throw ConcurrentModificationException 107 | int size = 0; 108 | for (RequestResponsePair pair : storage.getRequestResponsePairs()) { 109 | size++; 110 | } 111 | iterationCount.incrementAndGet(); 112 | } catch (Exception e) { 113 | exceptionCount.incrementAndGet(); 114 | fail("ConcurrentModificationException should not occur: " + e.getMessage()); 115 | } 116 | Thread.sleep(10); 117 | } 118 | } catch (InterruptedException e) { 119 | Thread.currentThread().interrupt(); 120 | } finally { 121 | endLatch.countDown(); 122 | } 123 | }); 124 | 125 | startLatch.countDown(); 126 | boolean completed = endLatch.await(10, java.util.concurrent.TimeUnit.SECONDS); 127 | 128 | executorService.shutdown(); 129 | 130 | assertTrue(completed, "Test did not complete in time"); 131 | assertTrue(iterationCount.get() > 0, "Should have completed iterations"); 132 | assertEquals(0, exceptionCount.get(), "Should not have any exceptions"); 133 | assertEquals(numThreads * requestsPerThread, storage.getRequestResponsePairs().size(), 134 | "Should have collected all requests"); 135 | } 136 | 137 | /** 138 | * Test that snapshot creation prevents ConcurrentModificationException 139 | * during iteration (as done in createHarFile methods). 140 | */ 141 | @Test 142 | public void testSnapshotIterationDuringSerialization() throws InterruptedException { 143 | int numThreads = 3; 144 | ExecutorService executorService = Executors.newFixedThreadPool(numThreads + 1); 145 | CountDownLatch startLatch = new CountDownLatch(1); 146 | CountDownLatch endLatch = new CountDownLatch(numThreads + 1); 147 | AtomicInteger successfulSnapshots = new AtomicInteger(0); 148 | AtomicInteger exceptionCount = new AtomicInteger(0); 149 | 150 | // Add initial requests 151 | for (int i = 0; i < 10; i++) { 152 | storage.getRequestResponsePairs().add( 153 | new RequestResponsePair("initial-" + i, null, new Date(), null) 154 | ); 155 | } 156 | 157 | // Threads that continuously add requests 158 | for (int i = 0; i < numThreads; i++) { 159 | final int threadId = i; 160 | executorService.submit(() -> { 161 | try { 162 | startLatch.await(); 163 | for (int j = 0; j < 50; j++) { 164 | storage.getRequestResponsePairs().add( 165 | new RequestResponsePair("request-" + threadId + "-" + j, null, new Date(), null) 166 | ); 167 | Thread.sleep(1); 168 | } 169 | } catch (InterruptedException e) { 170 | Thread.currentThread().interrupt(); 171 | } finally { 172 | endLatch.countDown(); 173 | } 174 | }); 175 | } 176 | 177 | // Snapshot-based iteration (as used in createHarFile) 178 | executorService.submit(() -> { 179 | try { 180 | startLatch.await(); 181 | for (int iteration = 0; iteration < 20; iteration++) { 182 | try { 183 | // Create snapshot like the actual code does 184 | var snapshot = new java.util.ArrayList<>(storage.getRequestResponsePairs()); 185 | int snapshotSize = snapshot.size(); 186 | snapshot.forEach(pair -> assertNotNull(pair)); 187 | successfulSnapshots.incrementAndGet(); 188 | } catch (Exception e) { 189 | exceptionCount.incrementAndGet(); 190 | fail("Exception during snapshot iteration: " + e.getMessage()); 191 | } 192 | Thread.sleep(5); 193 | } 194 | } catch (InterruptedException e) { 195 | Thread.currentThread().interrupt(); 196 | } finally { 197 | endLatch.countDown(); 198 | } 199 | }); 200 | 201 | startLatch.countDown(); 202 | boolean completed = endLatch.await(15, java.util.concurrent.TimeUnit.SECONDS); 203 | 204 | executorService.shutdown(); 205 | 206 | assertTrue(completed, "Test did not complete in time"); 207 | assertTrue(successfulSnapshots.get() > 0, "Should have successful snapshots"); 208 | assertEquals(0, exceptionCount.get(), "Should not have any exceptions"); 209 | } 210 | 211 | /** 212 | * Test that NetworkListener uses ConcurrentHashMap for windowHandleStorageMap. 213 | * This verifies the thread-safety improvement at the NetworkListener level. 214 | */ 215 | @Test 216 | public void testNetworkListenerUsesThreadSafeMap() throws NoSuchFieldException, IllegalAccessException { 217 | // Create a NetworkListener instance (with minimal setup) 218 | com.blibli.oss.qa.util.services.NetworkListener listener = 219 | new com.blibli.oss.qa.util.services.NetworkListener("dummy.har"); 220 | 221 | // Use reflection to access the private windowHandleStorageMap field 222 | Field mapField = com.blibli.oss.qa.util.services.NetworkListener.class.getDeclaredField("windowHandleStorageMap"); 223 | mapField.setAccessible(true); 224 | Map map = (Map) mapField.get(listener); 225 | 226 | // Verify it's a ConcurrentHashMap 227 | assertTrue(map instanceof ConcurrentHashMap, 228 | "NetworkListener.windowHandleStorageMap should use ConcurrentHashMap for thread safety"); 229 | } 230 | 231 | /** 232 | * Test concurrent map and list operations together. 233 | * Simulates the complete scenario of multiple windows being monitored 234 | * while HAR file is being written. 235 | */ 236 | @Test 237 | public void testConcurrentWindowHandleAndRequestStorage() throws InterruptedException { 238 | Map windowMap = new ConcurrentHashMap<>(); 239 | int numWindows = 3; 240 | int threadsPerWindow = 2; 241 | int requestsPerThread = 10; 242 | 243 | // Initialize windows 244 | for (int w = 0; w < numWindows; w++) { 245 | windowMap.put("window-" + w, new RequestResponseStorage()); 246 | } 247 | 248 | ExecutorService executorService = Executors.newFixedThreadPool(numWindows * threadsPerWindow + 1); 249 | CountDownLatch startLatch = new CountDownLatch(1); 250 | CountDownLatch endLatch = new CountDownLatch(numWindows * threadsPerWindow + 1); 251 | AtomicInteger exceptions = new AtomicInteger(0); 252 | 253 | // Threads adding requests to different windows 254 | for (int w = 0; w < numWindows; w++) { 255 | for (int t = 0; t < threadsPerWindow; t++) { 256 | final int windowId = w; 257 | final int threadId = t; 258 | executorService.submit(() -> { 259 | try { 260 | startLatch.await(); 261 | RequestResponseStorage storage = windowMap.get("window-" + windowId); 262 | for (int i = 0; i < requestsPerThread; i++) { 263 | storage.getRequestResponsePairs().add( 264 | new RequestResponsePair( 265 | "win-" + windowId + "-thread-" + threadId + "-req-" + i, 266 | null, 267 | new Date(), 268 | null 269 | ) 270 | ); 271 | } 272 | } catch (InterruptedException e) { 273 | Thread.currentThread().interrupt(); 274 | } finally { 275 | endLatch.countDown(); 276 | } 277 | }); 278 | } 279 | } 280 | 281 | // Thread that iterates over all windows (simulating createHarFile) 282 | executorService.submit(() -> { 283 | try { 284 | startLatch.await(); 285 | Thread.sleep(50); 286 | for (int iteration = 0; iteration < 5; iteration++) { 287 | try { 288 | // Snapshot the map (as done in createHarFile) 289 | var snapshot = new java.util.HashMap<>(windowMap); 290 | snapshot.forEach((windowHandle, storage) -> { 291 | // Snapshot the list 292 | var listSnapshot = new java.util.ArrayList<>(storage.getRequestResponsePairs()); 293 | listSnapshot.forEach(pair -> assertNotNull(pair)); 294 | }); 295 | } catch (Exception e) { 296 | exceptions.incrementAndGet(); 297 | e.printStackTrace(); 298 | } 299 | Thread.sleep(10); 300 | } 301 | } catch (InterruptedException e) { 302 | Thread.currentThread().interrupt(); 303 | } finally { 304 | endLatch.countDown(); 305 | } 306 | }); 307 | 308 | startLatch.countDown(); 309 | boolean completed = endLatch.await(15, java.util.concurrent.TimeUnit.SECONDS); 310 | 311 | executorService.shutdown(); 312 | 313 | assertTrue(completed, "Test did not complete in time"); 314 | assertEquals(0, exceptions.get(), "Should not have any exceptions"); 315 | 316 | // Verify all data was collected 317 | int totalRequests = 0; 318 | for (RequestResponseStorage s : windowMap.values()) { 319 | totalRequests += s.getRequestResponsePairs().size(); 320 | } 321 | assertEquals(numWindows * threadsPerWindow * requestsPerThread, totalRequests, 322 | "Should have collected all requests"); 323 | } 324 | 325 | /** 326 | * Test that no exception is thrown when creating a snapshot 327 | * while the underlying list is being modified. 328 | */ 329 | @Test 330 | public void testSnapshotCreationWithConcurrentModification() throws InterruptedException { 331 | ExecutorService executorService = Executors.newFixedThreadPool(2); 332 | CountDownLatch startLatch = new CountDownLatch(1); 333 | CountDownLatch endLatch = new CountDownLatch(2); 334 | AtomicInteger snapshotCount = new AtomicInteger(0); 335 | AtomicInteger exceptions = new AtomicInteger(0); 336 | 337 | // Add initial data 338 | for (int i = 0; i < 100; i++) { 339 | storage.getRequestResponsePairs().add( 340 | new RequestResponsePair("initial-" + i, null, new Date(), null) 341 | ); 342 | } 343 | 344 | // Thread 1: Continuously create snapshots 345 | executorService.submit(() -> { 346 | try { 347 | startLatch.await(); 348 | for (int i = 0; i < 100; i++) { 349 | try { 350 | var snapshot = new java.util.ArrayList<>(storage.getRequestResponsePairs()); 351 | assertTrue(snapshot.size() >= 100); 352 | snapshotCount.incrementAndGet(); 353 | } catch (Exception e) { 354 | exceptions.incrementAndGet(); 355 | throw e; 356 | } 357 | } 358 | } catch (InterruptedException e) { 359 | Thread.currentThread().interrupt(); 360 | } finally { 361 | endLatch.countDown(); 362 | } 363 | }); 364 | 365 | // Thread 2: Continuously add more items 366 | executorService.submit(() -> { 367 | try { 368 | startLatch.await(); 369 | for (int i = 0; i < 500; i++) { 370 | storage.getRequestResponsePairs().add( 371 | new RequestResponsePair("concurrent-" + i, null, new Date(), null) 372 | ); 373 | } 374 | } catch (Exception e) { 375 | exceptions.incrementAndGet(); 376 | } finally { 377 | endLatch.countDown(); 378 | } 379 | }); 380 | 381 | startLatch.countDown(); 382 | boolean completed = endLatch.await(10, java.util.concurrent.TimeUnit.SECONDS); 383 | 384 | executorService.shutdown(); 385 | 386 | assertTrue(completed, "Test did not complete in time"); 387 | assertEquals(0, exceptions.get(), "Should not have any exceptions"); 388 | assertTrue(snapshotCount.get() > 0, "Should have created snapshots"); 389 | assertEquals(600, storage.getRequestResponsePairs().size(), 390 | "Should have 600 items (100 initial + 500 concurrent)"); 391 | } 392 | } 393 | --------------------------------------------------------------------------------