Develop Reference

Spraping data from a table is slow but uncertain why

I am scraping data from a table using selenium with Java but is slow and I am not sure why. Is there a reason why and how can I speed it up? The other thing I noticed is that it seems to slow down more as it progresses. I noticed this by observing the print statements to the console.
Here is my code:
package mypackage;
import java.io.IOException;
import java.time.Duration;
import java.util.List;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import com.seleniumpractice.utilities.XLUtils;
import io.github.bonigarcia.wdm.WebDriverManager;
public class CovidWebTable {
static WebDriver driver;
static XLUtils xl;
static List<WebElement> header;
static List<WebElement> rows;
public static void main(String[] args) throws IOException {
WebDriverManager.chromedriver().setup();
driver = new ChromeDriver();
driver.get("https://www.worldometers.info/coronavirus");
driver.manage().window().maximize();
driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(10));
WebElement table = driver.findElement(By.xpath("//table[#id='main_table_countries_today']"));
rows = table.findElements(By.xpath(".//tr[#role='row']"));
System.out.println("Total rows: "+rows.size());
xl = new XLUtils(".\\datafiles\\covid.xls");
//xl.setCellData(null, rows, rows, null);
//Add header
header = table.findElements(By.xpath(".//thead//th"));
System.out.println("Header cols: "+ header.size());
for(int col=1; col<header.size()-1; col++) {
xl.setCellData("Covid Data", 0, col-1, header.get(col).getText());
}
int xlRow = 1;
for(int r=1; r<rows.size(); r++) {
String a = rows.get(r).getText();
if(rows.get(r).getText().equals("")) {
System.out.println("Skipped row: "+r);
continue;
}
System.out.println("Writing row "+r);
for(int c=1; c<header.size(); c++) {
//String data = rows.get(r).findElement(By.xpath(".//td["+(c+1)+"]")).getText();
xl.setCellData("Covid Data", xlRow, c-1, rows.get(r).findElement(By.xpath(".//td["+(c+1)+"]")).getText());
}
xlRow++;
}
System.out.println("Complete.");
driver.close();
}
}
The code that contains the code for writing to excel:
package com.internetBanking.utilities;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
public class XLUtils {
public static FileInputStream fi;
public static FileOutputStream fo;
public static HSSFWorkbook wb;
public static HSSFSheet ws;
public static HSSFRow row;
public static HSSFCell cell;
public static int getRowCount(String xlfile, String xlsheet) throws IOException {
fi = new FileInputStream(xlfile);
wb = new HSSFWorkbook(fi);
ws = wb.getSheet(xlsheet);
int rowcount = ws.getLastRowNum();
wb.close();
fi.close();
return rowcount;
}
public static int getCellCount(String xlFile, String xlSheet, int rowNum) throws IOException {
fi = new FileInputStream(xlFile);
wb = new HSSFWorkbook(fi);
ws = wb.getSheet(xlSheet);
row = ws.getRow(rowNum);
int cellCount = row.getLastCellNum();
wb.close();
fi.close();
return cellCount;
}
public static String getCellData(String xlFile, String xlSheet, int rowNum, int colNum) throws IOException {
fi = new FileInputStream(xlFile);
wb = new HSSFWorkbook(fi);
ws = wb.getSheet(xlSheet);
row = ws.getRow(rowNum);
cell = row.getCell(colNum);
String data;
try {
String cellData = new DataFormatter().formatCellValue(cell);
return cellData;
}
catch(Exception e) {
data = "";
}
wb.close();
fi.close();
return data;
}
public static void setCellData(String xlFile, String xlSheet, int rowNum, int colNum, String data) throws IOException{
fi = new FileInputStream(xlFile);
wb = new HSSFWorkbook(fi);
ws = wb.getSheet(xlSheet);
row = ws.getRow(rowNum);
Cell cell = row.createCell(colNum);
cell.setCellValue(data);
//cell = row.getCell(colNum);
//cell.setCellValue(data);
fo = new FileOutputStream(xlFile);
wb.write(fo);
wb.close();
fi.close();
fo.close();
}
}

Ok getting to understand the code will give you a hint of what you should do, the code is iterating 231 read data from the table and write it to an excel file (daaa!) Ok but when writing into the excel file you write row by row (so!) You then iterate cell by cell on each row on each cell you call setCellData(...) in that XLUtils and here is when it starts to slow down!
In that setCellData each time you call it, it read a file from the disk, opens it, appends data then close the file and since you call it by cell, you end up calling it ~231 (rows) x 15 (col) = 3465
Imagine the time consumed (opening file/writing data/ closing file) 3465 times
So what to do,
You need to create a list of list
List<ArrayList> rows = new ArrayList()
This is a list of list ^ where each record in the list is another list
and on each row you read from the table (even headers), you create a list of cells and then you add this list to the rows list!
and eventually, you add some utility function in that XLUtils that accept List<ArrayList> that will
Open the file once, Iterate on that rows list write their values, and close the file.
Imagine it as if you are trying to move from house to house, and you were using your personal small pickup truck car to move stuff. It won't take all the house stuff so you would go back and forth for a while. On the other hand, you could have used a furniture truck that picks up all furniture and loads it at once.

Unable to call the parameters from one class-method to another class-method

I have tried with the below code having two classes once is Class1-TestData and Method1-excel(),
And another call to access the parameters Class2-AdminLoginAction and Method2-Admin_Login().
Here is the problem with I need to call the string parameters like UID and PWD as I marked in the Screenshot attached. But the script showed some error and was unable to access it. So, How can I solve this problem, Am I going to the right approach or any other way? Please do need full as soon as possible.
Script image for two classes
import java.io.IOException;
import java.util.Date;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.testng.annotations.*;
import com.aventstack.extentreports.ExtentReports;
import com.aventstack.extentreports.ExtentTest;
import com.aventstack.extentreports.Status;
import com.aventstack.extentreports.reporter.ExtentSparkReporter;
public class AdminLoginAction extends TestData{
WebDriver d;
Date currentdate = new Date();
String Screenshotdate = currentdate.toString().replace(" ", "-").replace(":", "-");
ExtentSparkReporter spark = new ExtentSparkReporter("ExtentReport.html");
ExtentReports extent = new ExtentReports();
#Test()
public void Admin_Login() throws InterruptedException, IOException {
TestData excel = new TestData();
extent.attachReporter(spark);
ExtentTest test = extent.createTest("Launch browswer and access the WeClean Login page");
test.log(Status.PASS, "Launch browser success...!!!");
test.pass("Verified launching browser");
System.setProperty("webdriver.chrome.driver", "D:\\backup\\selenium\\chromedriver.exe");
d = new ChromeDriver();
d.manage().window().maximize();
d.get(URL);
Utils.CaptureScreenshot(d, Screenshotdate + "_Login.png");
d.findElement(By.id("loginUser")).sendKeys(UID);
d.findElement(By.id("password")).sendKeys(PWD);
d.findElement(By.id("loginButton")).click();
Thread.sleep(5000);
Utils.CaptureScreenshot(d, Screenshotdate + "_HomePage.png");
test.log(Status.PASS, "Admin Logged in Successful");
test.pass("Verified Admin logged in");
}
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Scanner;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.testng.annotations.Test;
public class TestData{
public void excel() throws IOException {
String filePath = System.getProperty("user.dir")+"\\Inputfiles";
File file = new File(filePath + "\\TestData.xlsx");
FileInputStream inputStream = new FileInputStream(file);
XSSFWorkbook wb=new XSSFWorkbook(inputStream);
XSSFSheet sheet=wb.getSheet("Admin_inputs");
XSSFRow row2=sheet.getRow(1);
XSSFCell cell=row2.getCell(0);
double UID= cell.getNumericCellValue();
XSSFCell cell2 = row2.getCell(1);
String PWD = cell2.getStringCellValue();
}

You can try like this:
public class TestData {
double UID;
String PWD;
public void excel() throws IOException {
String filePath = System.getProperty("user.dir") + "\\Inputfiles";
File file = new File(filePath + "\\TestData.xlsx");
FileInputStream inputStream = new FileInputStream(file);
XSSFWorkbook wb = new XSSFWorkbook(inputStream);
XSSFSheet sheet = wb.getSheet("Admin_inputs");
XSSFRow row2 = sheet.getRow(1);
XSSFCell cell = row2.getCell(0);
UID = cell.getNumericCellValue();
XSSFCell cell2 = row2.getCell(1);
PWD = cell2.getStringCellValue();
}
}
In Admin_Login() method, am just printing UID and PWD, change the code according to your requirement:
public class AdminLoginAction extends TestData {
#Test()
public void Admin_Login() throws InterruptedException, IOException {
// TestData excel = new TestData(); // you don't need to create this object because you are already inheriting `TestData` class.
excel(); // before using UID and PWD you have to call excel() method.
System.out.println(UID);
System.out.println(PWD);
}
}

For Loop NullPointerException with Apache POI

First of all, I apologize for my bad english, it's not my primary language.
I'm having a problem with my code when it's reading an excel file (xlsx). The first time I used my code it works perfectly, but now I can't use it.
The excel content is printed on the console, but next to it there is a NullPointerException that is caused by the for loop marked with the (-->>>).
If anyone can help me, I would be very thankful, this code has given me several headaches.
package modleerjava;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
public class leerExcel {
public void readExcel(String rutaFile,String nombreFile, String sheetName )
throws IOException {
File file = new File ("C:/Users/Pablo/Desktop/prueba.xlsx");
FileInputStream inputStream = new FileInputStream(file);
XSSFWorkbook excelWorkbook ;
excelWorkbook = new XSSFWorkbook(inputStream);
Sheet excelSheet = excelWorkbook.getSheet(sheetName);
int filasCount = excelSheet.getLastRowNum()-excelSheet.getFirstRowNum();
for (int i=0; i< filasCount+1 ; i++) {
Row filas;
filas = excelSheet.getRow(i);
-->>> for (int j=0 ; j < filas.getLastCellNum(); j++) {
System.out.print(filas.getCell(j).getStringCellValue()+"|| ");
}
System.out.println();
}
}
}

Your sheet probably has missing rows or cells. The best way to iterate over a spreadsheet in poi is using the for each syntax like this.
package modleerjava;
import java.io.File;
import java.io.IOException;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
public class leerExcel {
public void readExcel(String rutaFile,String nombreFile, String sheetName )
throws IOException {
Workbook wb = WorkbookFactory.create(new File("C:/Users/Pablo/Desktop/prueba.xlsx"));
Sheet excelSheet = wb.getSheet(sheetName);
for (Row filas: excelSheet) {
for (Cell cell: filas) {
System.out.print(cell.getStringCellValue()+"|| ");
}
System.out.println();
}
}
}
Many simple questions can be answered by looking at the quick guide on the poi website.

By seeing the code, I think that NullPointerException will occur if the first row in the excel sheet is blank. So, I have created an Iterator on rows.
Try the below code:
package modleerjava;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
public class leerExcel {
public void readExcel(String rutaFile,String nombreFile, String sheetName )
throws IOException {
File file = new File ("C:/Users/Pablo/Desktop/prueba.xlsx");
FileInputStream inputStream = new FileInputStream(file);
XSSFWorkbook excelWorkbook ;
excelWorkbook = new XSSFWorkbook(inputStream);
inputStream.close();
Sheet excelSheet = excelWorkbook.getSheet(sheetName);
//int filasCount = excelSheet.getLastRowNum()-excelSheet.getFirstRowNum();
Iterator<Row> row = excelSheet.rowIterator();
while(row.hasNext()) {
Row filas;
filas = row.next();
for (int j=0 ; j < filas.getLastCellNum(); j++) {
System.out.print(filas.getCell(j).getStringCellValue()+"|| ");
}
System.out.println();
}
}
}

Selenium/Java-How to make 'dataProvider' a common function inbetween multiple source folders

I have created a project with seperate source folder for each module, say mod1 & mod2 and the src source folder as common for each source folder.
And I created DataProvider_Repository.java file for reading data from excel sheet. Which now I need to put Data Provider in each source folder for accessing it in each testcase(Eg:Registration.java or Del.java).
How can I make it common & put in src folder.
DataProvider_Repository.java contains same code for reading data from excel sheet.
DataProvider_Repository.java
package online;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Method;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.testng.annotations.DataProvider;
public class DataProvider_Repository {
#DataProvider(name="dataProviderRepo")
public static Object[][] getDataFromDataprovider(Method m) throws IOException{
Object[][] object = null;
String filePath = System.getProperty("user.dir")+"\\";
String fileName = "sample.xlsx";
String sheetName= "sampleSheet";
File file = new File(filePath+"\\"+fileName);
FileInputStream inputStream = new FileInputStream(file);
Workbook wb = new XSSFWorkbook(inputStream);
Sheet s = wb.getSheet(sheetName);
//Find number of rows in excel file
int rowCount = s.getLastRowNum()-s.getFirstRowNum();
object = new Object[rowCount][6];
for (int i = 0; i < rowCount; i++) {
//Loop over all the rows
Row row = s.getRow(i+1);
int lst = row.getLastCellNum();
//Create a loop to print cell values in a row
for (int j = 0; j < lst; j++) {
//Print excel data in console
if(row.getCell(j) != null){
object[i][j] = row.getCell(j).toString();
}
}
object[i][lst-1] = Integer.toString(i);
}
return object;
}
}
Registration.java
package online;
import org.testng.annotations.Test;
public class Registration {
#Test(dataProviderClass=DataProvider_Repository.class,dataProvider="dataProviderRepo")
public void OnlineRegs(String testcaseName,String keyword,String objectName,String objectType,String value, String rowNum) throws Exception {
System.out.println("=======OnlineReg===========");
//Operations here...
}
}
The folder structure is

Reading excel to list of Hashtable in Java

I have a code to read an excel file and place it in a list Hash tables. The code that I have is:
package tests;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.List;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import jxl.Cell;
import jxl.Sheet;
import jxl.Workbook;
import jxl.read.biff.BiffException;
public class ReadExcelTest {
public static Hashtable htable=new Hashtable();
public static List<Hashtable<String,String>> exceldata = new ArrayList<Hashtable<String,String>>();
public static void main() throws IOException{
readexcel(System.getProperty("user.dir")+"//src//config//TestData.xls");
}
#Test
public static void readexcel() throws IOException
{
String inputFile=System.getProperty("user.dir")+"//src//config//TestData.xls";
File inputWorkbook = new File(inputFile);
Workbook w;
Cell firstrowelement = null;
try
{
w = Workbook.getWorkbook(inputWorkbook);
Sheet sheet = w.getSheet(0);
for (int j = 0; j <sheet.getRows(); j++)
{
for (int i = 0; i < sheet.getColumns(); i++)
{
firstrowelement = sheet.getCell(i, 0);
Cell cell = sheet.getCell(i, j);
htable.put(firstrowelement.getContents(),cell.getContents());
System.out.print(firstrowelement.getContents()+"->"+cell.getContents());
}
System.out.println(firstrowelement.getContents());
exceldata.add(j,htable);
}
//printing the list
for(Hashtable hash :exceldata)
{
Enumeration e = hash.keys();
while (e.hasMoreElements()) {
String key = (String) e.nextElement();
System.out.println(key + " : " + hash.get(key));
}
}
}
catch (BiffException e)
{
e.printStackTrace();
}
}
}
The output I am getting is all the hash tables in the list are displaying the same data instead of displaying each rows data.
Not exactly sure how to over come this issue. Any help is much appreciated.

You need to make a new htable for every row.
As it is, you keep re-using and updating a single HashTable.
(btw. prefer HashMap to HashTable).