Develop Reference

How to replace a JSON String with Java?

In my JSON Array inside of its file, it has an array set that goes as follows:
{
"user_interface_fields" : {
"tip_identifier" : {
"uuid": "12345678-1234-1234-1234-123456789112"
}
"insert_time" : "2001-01-01T00:00:00Z"
}
And now I got this file written in java
#Test
public void testExampleMessageToJson() throws Throwable {
log.info("\n\nTesting persist service [message to json]?\n");
//Setup
String rawJson = TestUtils.getResourceMessage("examples/life.json");
String tip_identifier_uuid = "12345678-1234-1234-1234-123456789112";
log.info("Raw Json:\n():", rawJson);
assertTrue(StringUtils.isNotBlank(rawJson));
//Test
JSONObject foundObj = persistService.messageToJson(rawJson);
//Verify
String tip_identifier_time = foundObj.getString("insert_time");
String new_time = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").format(Calendar.getInstance().getTime());
foundObj.put(new_time, foundObj).getString("insert_time");\
assertNotNull(foundObj);
String new_uuid = foundObj.getJSONObject("user_interface_fields").getJSONObject("tip_identifier").getString("uuid");
assertEquals(tip_identifier_uuid , new_uuid);
assertEquals(tip_identifier_time, new_time);
log.info("\n\nTesting persist service [message to json] COMPLETE!\n");
}
My goal is to have life.json's time to be nearly identical to our current time (so like if it's 11/11/21 01:30:21.22, insert_time's data should be around 01:30:21.2234 or whatever and at any time i or any other person runs the code it should be a near match..)
But in my failure trace it tells me:
org.junit.ComparisonFailure: expected:<202[0-02-19'T'00:00:]00> but was:<202[1-11T17:46:46+00]00>
I'm thinking that the logging is messed up because expected should be nearly matched as the current time (as stated in my goal). Since messagetoJson gets the current time and inserts it, when my test pulls out the insert_time it should be a couple of milliseconds behind and so I need to validate that it's within a second or ~1000 milliseconds.
But I don't know what the heck I should do or where I should go from here in order to fix that.

Different results by different methods on Chrome Performance Metrics?

I am writing some code to automate calculating certain page performance metrics. The results I am getting for page size are different by different methods:
What I want to achieve is to read these values shown in this screenshot:
Methods I am using:
Method giving different page load time and different transferred sizes:
Totalbytes and NetData return very different numbers, both very far from what the screenshot would show
public void testing() throws HarReaderException {
JavascriptExecutor js1=((JavascriptExecutor)driver);
try {
Thread.sleep(5000);
}catch(Exception e) {e.printStackTrace();}
String url=driver.getCurrentUrl();
System.out.println("Current URL :"+url);
long pageLoadTime= (Long)js1.executeScript("return (window.performance.timing.loadEventEnd-window.performance.timing.responseStart)");
long TTFB= (Long)js1.executeScript("return (window.performance.timing.responseStart-window.performance.timing.navigationStart)");
long endtoendRespTime= (Long)js1.executeScript("return (window.performance.timing.loadEventEnd-window.performance.timing.navigationStart)");
Date date = new Date();
//Timestamp ts=new Timestamp(date.getTime());
System.out.println("PageLoadTime Time :"+pageLoadTime);
System.out.println("TTFB :"+TTFB);
System.out.println("Customer perceived Time :"+endtoendRespTime);
System.out.println("timeStamp");
String scriptToExecute = "var performance = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {}; var network = performance.getEntries() || {}; return network;";
String netData = ((JavascriptExecutor)driver).executeScript(scriptToExecute).toString();
System.out.println("Net data: " + netData);
String anotherScript = "return performance\n" +
" .getEntriesByType(\"resource\")\n" +
" .map((x) => x.transferSize)\n" +
" .reduce((a, b) => (a + b), 0);"; //I have tried encodedSize here as well, still gives different results
System.out.println("THIS IS HOPEFULLY THE TOTAL TRANSFER SIZE " + js1.executeScript((anotherScript)).toString());
int totalBytes = 0;
for (LogEntry entry : driver.manage().logs().get(LogType.PERFORMANCE)) {
if (entry.getMessage().contains("Network.dataReceived")) {
Matcher dataLengthMatcher = Pattern.compile("dataLength\":(.*?),").matcher(entry.getMessage()); //I tried encodedLength and other methods but always get different results from the actual page
dataLengthMatcher.find();
totalBytes = totalBytes + Integer.parseInt(dataLengthMatcher.group(1));
//Do whatever you want with the data here.
}
}
System.out.println(totalBytes);
}
Setting up selenium Chrome driver, enabling performance logging and mobbrowser proxy:
#BeforeTest
public void setUp() {
// start the proxy
proxy = new BrowserMobProxyServer();
proxy.start(0);
//get the Selenium proxy object - org.openqa.selenium.Proxy;
Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
// configure it as a desired capability
DesiredCapabilities capabilities = new DesiredCapabilities().chrome();
LoggingPreferences logPrefs = new LoggingPreferences();
logPrefs.enable(LogType.PERFORMANCE, Level.ALL);
capabilities.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);
capabilities.setCapability(CapabilityType.PROXY, seleniumProxy);
ChromeOptions options = new ChromeOptions();
options.addArguments("--incognito");
capabilities.setCapability(ChromeOptions.CAPABILITY, options);
//set chromedriver system property
System.setProperty("webdriver.chrome.driver", driverPath);
driver = new ChromeDriver(capabilities);
// enable more detailed HAR capture, if desired (see CaptureType for the complete list)
proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
}
Methods I am using to analyze the page:
This method was supposed to show the Load Time in chrome inspector, but it is always showing a lesser number (I think it is showing the time of the last response received instead of DOMContentLoaded or Load Time)
public double calculatePageLoadTime(String filename) throws HarReaderException {
HarReader harReader = new HarReader();
de.sstoehr.harreader.model.Har har = harReader.readFromFile(new File(filename));
HarLog log = har.getLog();
// Access all pages elements as an object
long startTime = log.getPages().get(0).getStartedDateTime().getTime();
// Access all entries elements as an object
List<HarEntry> hentry = log.getEntries();
long loadTime = 0;
int entryIndex = 0;
//Output "response" code of entries.
for (HarEntry entry : hentry)
{
long entryLoadTime = entry.getStartedDateTime().getTime() + entry.getTime();
if(entryLoadTime > loadTime){
loadTime = entryLoadTime;
}
entryIndex++;
}
long loadTimeSpan = loadTime - startTime;
Double webLoadTime = ((double)loadTimeSpan) / 1000;
double webLoadTimeInSeconds = Math.round(webLoadTime * 100.0) / 100.0;
return webLoadTimeInSeconds;
}
I am getting the total number of requests by reading the HAR file from the page, but for some reason it is always 10% less then the actual:
public int getNumberRequests(String filename) throws HarReaderException {
HarReader harReader = new HarReader();
de.sstoehr.harreader.model.Har har = harReader.readFromFile(new File(filename));
HarLog log = har.getLog();
return log.getEntries().size();
}
Testing this on google gives very different results by each method, which are usually 10-200% off from correct numbers.
Why does this happen? Is there a simple way to get those metrics properly from Chrome or any library that makes this easier? My task is automate doing performance analysis on thousands of pages.

I personally analyzed this on my system over and over again and came up with this -
The resource size which its showing currently is the amount of resource fetched till page load event is triggered.
So to overcome this you need to capture the the resource size variable after the page load event also until it stabilizes.Then it will match the actual console values.

Selenium How to wait multiple times for the same WebElement with different text

I'm attempting to wait for the WebElement to change from blank,to message 1, then message 2. The problem is I find the first message everytime, but i can't ever seem to wait for the second(it timesout looking for the text)
I've tried having the wait objects separate that didnt work. Ive tried a few of the expected condition methods(textToBePresent*), which after some reading (i found about the refreshed EC) to no avail.
#FindBy(xpath="//p[#class='statusText']")
WebElement statusMsg
public WebElement statusMsg(){
String msg1="Logging in, please wait."
String msg2="Login successful, please wait."
String msg3="Login attempt exception, error code: "
if(statusMsg.getText().contains(msg3)){
log.error(statusMsg.getText())
log.error("Something happened in the frontend")
Assert.fail(statusMsg.getText())
}else{
log.info(statusMsg.getText())
}
WebDriverWait wait = new WebDriverWait(driver,45)
wait.until(ExpectedConditions.textToBe(By.xpath("//p[#class='statusText']"), msg1))
if(statusMsg.getText().contains(msg3)){
log.error(statusMsg.getText())
log.error("Something happened in the backend")
Assert.fail(statusMsg.getText())
}else{
log.info(statusMsg.getText())
}
wait.until(ExpectedConditions.refreshed(ExpectedConditions.textToBe(By.xpath("//p[#class='statusText']"), msg2)))
log.info("Found: "+msg2)
return statusMsg
}
The result is testNG fails my test saying:
org.openqa.selenium.TimeoutException: Expected condition failed:
waiting for condition (element found by By.xpath:
//p[#class='statusText'] to have text "Login successful, please
wait.". Current text: "Logging in, please wait.")
Yet I can see the msg2 while the test is running. Does this have to do because I've already initialised the page objects via PageFactory.initElements(driver, this)?

Here is code you can use to check error and success messages sequencenly.
Logic:
Put expected messages to the expectedMessages list
Use Fluent Wait to check for custom conditions
Set polling every 50 milliseconds to able to catch messages
Check if status message element is visible, if so get text
Fail if actual status message is "error"
Check actual status message contains first text in the expectedMessages, if so remove first text from expectedMessages
Repeat step 6 until there's no items left if expectedMessages
WebDriverWait wait = new WebDriverWait(driver, 20);
String errorMessage = "Login attempt exception, error code:";
List<String> expectedMessages = Arrays.asList(
"Logging in, please wait.",
"Login successful, please wait.");
wait.pollingEvery(Duration.ofMillis(50))
.withMessage(String.format("Expecting %s login messages", expectedMessages))
.until(d -> {
if (!d.findElement(By.xpath("//p[#class='statusText']")).isDisplayed())
return false;
String actualMessage = d.findElement(By.xpath("//p[#class='statusText']")).getText();
if (actualMessage.contains(errorMessage))
Assert.fail(actualMessage);
String expectedMessage = expectedMessages.get(0);
if (expectedMessage.contains(actualMessage)) {
log.info(String.format("Message \"%s\" found", actualMessage));
expectedMessages.remove(expectedMessage);
}
return expectedMessages.size() == 0;
});
Second solution is to get all messages and after check.
Use code below to get all messages and check what you get from the website:
JavascriptExecutor js = (JavascriptExecutor) driver;
List<String> actualMessages = new ArrayList<>();
for (int i = 0; i < 30000; i++) {
actualMessages.addAll(
(ArrayList<String>) js.executeScript("return [...document.querySelectorAll('p.statusText')].map(e=>{return e.textContent})")
);
Thread.sleep(10);
}
// debug here to check message collected
System.out.println(actualMessages);

Jsoup fetches wrong results

Working with Jsoup. The URL works well on the browser. But it fetches wrong result on the server. I set the maxBodySize "0" as well. But it still only gets first few tags. Moreover the data is even different from the browser one. Can you guys give me a hand?
String queryUrl = "http://www.juso.go.kr/addrlink/addrLinkApi.do?confmKey=U01TX0FVVEgyMDE3MDYyODE0MTYyMzIyMTcw&currentPage=1&countPerPage=20&keyword=연남동";
Document document = Jsoup.connect(queryUrl).maxBodySize(0).get();

Are you aware that this endpoint returns paginated data? Your URL asks for 20 entries from the first page. I assume that the order of these entries is not specified so you can get different data each time you call this endpoint - check if there is a URL parameter that can determine specific sort order.
Anyway to read all 2037 entries you have to do it sequentially. Examine following code:
final String baseUrl = "http://www.juso.go.kr/addrlink/addrLinkApi.do";
final String key = "U01TX0FVVEgyMDE3MDYyODE0MTYyMzIyMTcw";
final String keyword = "연남동";
final int perPage = 100;
int currentPage = 1;
while (true) {
System.out.println("Downloading data from page " + currentPage);
final String url = String.format("%s?confmKey=%s&currentPage=%d&countPerPage=%d&keyword=%s", baseUrl, key, currentPage, perPage, keyword);
final Document document = Jsoup.connect(url).maxBodySize(0).get();
final Elements jusos = document.getElementsByTag("juso");
System.out.println("Found " + jusos.size() + " juso entries");
if (jusos.size() == 0) {
break;
}
currentPage += 1;
}
In this case we are asking for 100 entries per page (that's the maximum number this endpoint supports) and we call it 21 times, as long as calling for a specific page return any <juso> element. Hope it helps solving your problem.

HtmlUnit won't change TextArea's text

I have written a code to connect to this webpage: compilerjava.net
1) I found the text-area field within that page which accepts the code to compile.
2) I have found the button that compiles the code.
3) I have found the text-area which returns the result of the code.
The issue is, when I call textarea.setText( "something"), it (I think) doesn't actually change the code in the webpage. So when I click on the compile button, it compiles the default code within that page and returns the output of that default code.
I have tried to set focus to textarea, you can see all of those down below.
I called;
1) textArea.focus();
2) textArea.click();
3) I tried using textArea.setAttribute( "name", "code");
I have searched the internet and found various stackoverflow questions close to this problem, neither of them solved my issue and it just seems to work for everyone when they say textArea.setText().
Another interesting fact I should share with you is,
If I call textArea.setText( "...") and then I say;
HtmlTextArea textArea1 = form.getTextAreaByName( "code");
If I call textArea1.getText(), the value of this text will be "...". This should imply that I have actually managed to change the value of the text-area, but when I compile, it compiles the default text in the text-area and not the text that I have set it to.
Any help with this?
P.S: The reason why I put the result of the compilation on a while loop is related to network connection issues. If you try to run this code it might not work on your first try. Also note that the run-time is around 15 seconds, because it gives thousands of warnings which I blocked to print to console.
P.S2: I also looked at this page and none of these worked;
http://www.programcreek.com/java-api-examples/index.php?api=com.gargoylesoftware.htmlunit.html.HtmlTextArea
public class Test {
public static void main(String[] args) {
try {
// Prevents the program to print thousands of warning codes.
java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(java.util.logging.Level.OFF);
java.util.logging.Logger.getLogger("org.apache.http").setLevel(java.util.logging.Level.OFF);
// Initializes the web client and yet again stops some warning codes.
WebClient webClient = new WebClient( BrowserVersion.CHROME);
webClient.getOptions().setThrowExceptionOnFailingStatusCode( false);
webClient.getOptions().setThrowExceptionOnScriptError( false);
webClient.getOptions().setJavaScriptEnabled( true);
webClient.getOptions().setCssEnabled( true);
// Gets the html page, which is the online compiler I'm using.
HtmlPage page = webClient.getPage("https://www.compilejava.net/");
// Succesfully finds the form which has the required buttons etc.
List<HtmlForm> forms = page.getForms();
HtmlForm form = forms.get( 0);
// Finds the textarea which will hold the code.
HtmlTextArea textArea = form.getTextAreaByName( "code");
// Finds the textarea which will hold the result of the compilation.
HtmlTextArea resultArea = page.getHtmlElementById( "execsout");
// Finds the compile button.
HtmlButtonInput button = form.getInputByName( "compile");
textArea.click();
textArea.focus();
// Simple code to run.
textArea.setDefaultValue( "public class HelloWorld\n" +
"{\n" +
" // arguments are passed using the text field below this editor\n" +
" public static void main(String[] args)\n" +
" {\n" +
" System.out.print( \"Hello\");\n" +
" }\n" +
"}");
System.out.println( textArea.getText());
// Compiles.
button.click();
// Result of the compilation.
String str = resultArea.getText();
while ( resultArea.getText() == null || resultArea.getText().substring(0, 3).equals( "exe")) {
System.out.print( resultArea.getText());
}
System.out.println();
System.out.println( resultArea.getText());
} catch ( Exception e) {
e.printStackTrace();
}
}
}

a little patience helps here
// Result of the compilation.
while (resultArea.getText() == null || resultArea.getText().startsWith("exe")) {
System.out.println(resultArea.getText());
Thread.sleep(500);
}
System.out.println();
System.out.println(resultArea.getText());