I am developing some selenium code and was trying to get input from an Excel sheet using apache POI. So far I have managed to get the input but I am unable to transfer it from class to class. Please see the code below:
Functions To Be Called:
package Excel;
import java.io.File;
import java.io.FileInputStream;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
public class Read {
XSSFSheet Names;
public void read() throws Exception{
File src = new File("C:\\Users\\dindo\\Documents\\tests\\d2c-lv-int-01_DATA.xlsx");
FileInputStream fis = new FileInputStream(src);
XSSFWorkbook wb = new XSSFWorkbook(fis);
Names = wb.getSheetAt(0);
}
public void getcell(int row, int col){
String stringresult = Names.getRow(row).getCell(col).getStringCellValue();
String intresult = Names.getRow(row).getCell(col).getStringCellValue();
}
Trying To Call The Functions:
package Pages;
import Excel.Read;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.FindBy;
import org.openqa.selenium.support.How;
public class DetailsPage {
Read cel = new Read();
cel.getcell(2,4)
#FindBy(how = How.XPATH, using = "/*xpath*/")
public WebElement coveramountelement;
public String coveramount = cel.intresult;
public void EnterDetails() {
coveramountelement.sendKeys(coveramount);
}
}
All errors that are for line 10 are for cel.getcell(2,4);
I would suggest restructuring your code like so
Functions are meant to be reused - so hard-coding specific variables into them isn't a good idea. And you're not storing any state, so just return the objects you want
public class Read {
public static XSSFSheet getSheet(String file, int sheetIndex) throws Exception{
File src = new File(f);
FileInputStream fis = new FileInputStream(src);
XSSFWorkbook wb = new XSSFWorkbook(fis);
return wb.getSheetAt(sheetIndex);
}
public static Cell getCell(XSSFSheet s, int row, int col) {
return s.getRow(row).getCell(col);
}
Now, use those generic functions to do your specific tasks
public class DetailsPage {
private XSSFSheet names;
public DetailsPage() {
try {
names = Read.getSheet("C:\\Users\\dindo\\Documents\\tests\\d2c-lv-int-01_DATA.xlsx", 0);
} catch (Exception e) {
e.printStackTrace();
}
}
#FindBy(how = How.XPATH, using = "/*xpath*/")
public WebElement coverAmountElement;
public void enterDetails() {
if (names != null) {
XSSFCell c = Read.getCell(names, 2,4);
String coveramount = c.getStringCellValue();
coverAmountElement.sendKeys(coveramount);
}
}
}
You cannot call a method/function outside of another method/function unless it's to initialize a variable. Furthermore, the getCell method returns a void which means that you can't use it to assign to anything, Also, you're missing a semicolon. So change this
Read cel = new Read();
cel.getcell(2,4)
To something like this:
package Pages;
import Excel.Read;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.support.FindBy;
import org.openqa.selenium.support.How;
public class DetailsPage {
#FindBy(how = How.XPATH, using = "/*xpath*/")
public WebElement coveramountelement;
public void EnterDetails() {
Read cel = new Read();
String coveramount = cel.getcell(2,4);
coveramountelement.sendKeys(coveramount);
}
}
And for getCell have it be something like this:
public String getcell(int row, int col){
String stringresult = Names.getRow(row).getCell(col).getStringCellValue();
return stringresult;
}
Related
I am scraping data from a table using selenium with Java but is slow and I am not sure why. Is there a reason why and how can I speed it up? The other thing I noticed is that it seems to slow down more as it progresses. I noticed this by observing the print statements to the console.
Here is my code:
package mypackage;
import java.io.IOException;
import java.time.Duration;
import java.util.List;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import com.seleniumpractice.utilities.XLUtils;
import io.github.bonigarcia.wdm.WebDriverManager;
public class CovidWebTable {
static WebDriver driver;
static XLUtils xl;
static List<WebElement> header;
static List<WebElement> rows;
public static void main(String[] args) throws IOException {
WebDriverManager.chromedriver().setup();
driver = new ChromeDriver();
driver.get("https://www.worldometers.info/coronavirus");
driver.manage().window().maximize();
driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(10));
WebElement table = driver.findElement(By.xpath("//table[#id='main_table_countries_today']"));
rows = table.findElements(By.xpath(".//tr[#role='row']"));
System.out.println("Total rows: "+rows.size());
xl = new XLUtils(".\\datafiles\\covid.xls");
//xl.setCellData(null, rows, rows, null);
//Add header
header = table.findElements(By.xpath(".//thead//th"));
System.out.println("Header cols: "+ header.size());
for(int col=1; col<header.size()-1; col++) {
xl.setCellData("Covid Data", 0, col-1, header.get(col).getText());
}
int xlRow = 1;
for(int r=1; r<rows.size(); r++) {
String a = rows.get(r).getText();
if(rows.get(r).getText().equals("")) {
System.out.println("Skipped row: "+r);
continue;
}
System.out.println("Writing row "+r);
for(int c=1; c<header.size(); c++) {
//String data = rows.get(r).findElement(By.xpath(".//td["+(c+1)+"]")).getText();
xl.setCellData("Covid Data", xlRow, c-1, rows.get(r).findElement(By.xpath(".//td["+(c+1)+"]")).getText());
}
xlRow++;
}
System.out.println("Complete.");
driver.close();
}
}
The code that contains the code for writing to excel:
package com.internetBanking.utilities;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
public class XLUtils {
public static FileInputStream fi;
public static FileOutputStream fo;
public static HSSFWorkbook wb;
public static HSSFSheet ws;
public static HSSFRow row;
public static HSSFCell cell;
public static int getRowCount(String xlfile, String xlsheet) throws IOException {
fi = new FileInputStream(xlfile);
wb = new HSSFWorkbook(fi);
ws = wb.getSheet(xlsheet);
int rowcount = ws.getLastRowNum();
wb.close();
fi.close();
return rowcount;
}
public static int getCellCount(String xlFile, String xlSheet, int rowNum) throws IOException {
fi = new FileInputStream(xlFile);
wb = new HSSFWorkbook(fi);
ws = wb.getSheet(xlSheet);
row = ws.getRow(rowNum);
int cellCount = row.getLastCellNum();
wb.close();
fi.close();
return cellCount;
}
public static String getCellData(String xlFile, String xlSheet, int rowNum, int colNum) throws IOException {
fi = new FileInputStream(xlFile);
wb = new HSSFWorkbook(fi);
ws = wb.getSheet(xlSheet);
row = ws.getRow(rowNum);
cell = row.getCell(colNum);
String data;
try {
String cellData = new DataFormatter().formatCellValue(cell);
return cellData;
}
catch(Exception e) {
data = "";
}
wb.close();
fi.close();
return data;
}
public static void setCellData(String xlFile, String xlSheet, int rowNum, int colNum, String data) throws IOException{
fi = new FileInputStream(xlFile);
wb = new HSSFWorkbook(fi);
ws = wb.getSheet(xlSheet);
row = ws.getRow(rowNum);
Cell cell = row.createCell(colNum);
cell.setCellValue(data);
//cell = row.getCell(colNum);
//cell.setCellValue(data);
fo = new FileOutputStream(xlFile);
wb.write(fo);
wb.close();
fi.close();
fo.close();
}
}
Ok getting to understand the code will give you a hint of what you should do, the code is iterating 231 read data from the table and write it to an excel file (daaa!) Ok but when writing into the excel file you write row by row (so!) You then iterate cell by cell on each row on each cell you call setCellData(...) in that XLUtils and here is when it starts to slow down!
In that setCellData each time you call it, it read a file from the disk, opens it, appends data then close the file and since you call it by cell, you end up calling it ~231 (rows) x 15 (col) = 3465
Imagine the time consumed (opening file/writing data/ closing file) 3465 times
So what to do,
You need to create a list of list
List<ArrayList> rows = new ArrayList()
This is a list of list ^ where each record in the list is another list
and on each row you read from the table (even headers), you create a list of cells and then you add this list to the rows list!
and eventually, you add some utility function in that XLUtils that accept List<ArrayList> that will
Open the file once, Iterate on that rows list write their values, and close the file.
Imagine it as if you are trying to move from house to house, and you were using your personal small pickup truck car to move stuff. It won't take all the house stuff so you would go back and forth for a while. On the other hand, you could have used a furniture truck that picks up all furniture and loads it at once.
Here is the ExcelDataToDataTable format which converts exceldata to data table
package com.api.cucumber.transform;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;enter code here
import java.net.URI;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import com.automation.custom.utilities.ExcelLibrary;
import com.automation.custom.utilities.ExcelReader;
import cucumber.api.DataTable;
import cucumber.api.Transformer;
import cucumber.runtime.ParameterInfo;
import cucumber.runtime.table.TableConverter;
import cucumber.runtime.xstream.LocalizedXStreams;
import gherkin.formatter.model.Comment;
import gherkin.formatter.model.DataTableRow;
public class ExcelDataToDataTable extends Transformer<DataTable> {
#Override
public DataTable transform(String filePath) {
String file[] = filePath.split(";");
String path = getfilePath(file[0]);
ExcelReader reader = new ExcelReader.ExcelReaderBuilder()
.setFileLocation(path)
.setSheet(file[1])
.build();
List<List<String>> excelData = getExcelData(reader);
List<DataTableRow> dataTableRows = getDataTableRows(excelData);
DataTable table = getDataTable(dataTableRows);
return table;
}
public String getfilePath(String path) {
// File file = new File("TestData.xlsx");
//return file.toString();
}
private DataTable getDataTable(List<DataTableRow> dataTableRows) {
ParameterInfo parameterInfo = new ParameterInfo(null, null, null, null);
TableConverter tableConverter = new TableConverter(new LocalizedXStreams(Thread.currentThread().getContextClassLoader()).get(Locale.getDefault()), parameterInfo);
DataTable table = new DataTable(dataTableRows, tableConverter);
return table;
}
private List<DataTableRow> getDataTableRows(List<List<String>> excelData) {
List<DataTableRow> dataTableRows = new LinkedList<>();
int line = 1;
for(List<String> list : excelData){
Comment commnet = new Comment("", line);
DataTableRow tableRow = new DataTableRow(Arrays.asList(commnet), list, line++);
dataTableRows.add(tableRow);
}
return dataTableRows;
}
private List<List<String>> getExcelData(ExcelReader reader) {
List<List<String>> excelData = new LinkedList<>();
try {
excelData = reader.getSheetDataAt();
} catch (InvalidFormatException | IOException e) {
throw new RuntimeException(e.getMessage());
}
return excelData;
}
}
This is StepDefination file
//Testdata.xlsx;0 sheet
#Then("^I validate list of properties in page zero with data in excel at \"([^\"]*)\"$")
public void i_validate_list_of_properties_in_page_zero_with_data_in_excel_at(#Transform(ExcelDataToDataTable.class) DataTable table) throws Throwable {
System.out.println(table.toString());
List<String> dataList=table.asList(String.class);
for(String str : dataList)
{
System.out.println(str);
}
//Testdata.xlsx;1
#Then("^I validate list of properties in page one with data in excel at \"([^\"]*)\"$")
public void i_validate_list_of_properties_in_page_one_with_data_in_excel_at(#Transform(ExcelDataToDataTable.class) DataTable table) throws Throwable {
System.out.println(table.toString());
List<String> dataList=table.asList(String.class);
for(String str : dataList)
{
System.out.println(str);
}
This is the Feature file
Feature: Validate values in application pages
Description: Validate dynamic values
#Test
Scenario: User navigates to application site and validates tags
Given As User I need to tags in the application
When I navigate to zero page "application url"
Then I validate list of properties in page zero with data in excel at "TestData.xlsx;0”
#Test1
Scenario: User navigates to application site and validates tags
Then i select country
Then i validate list of properties in page one with data in exel at "TestData.xlsx;1”
This is the transformData class
package com.api.cucumber.transform;
import cucumber.api.Transformer;
public class TransformData extends Transformer<String>{
#Override
public String transform(String args) {
return args + " Transform";
}
}
This is the ExcelReader class to read the data from excel
package com.automation.custom.utilities;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.openqa.selenium.JavascriptExecutor;
import com.api.cucumber.transform.ExcelDataToDataTable;
public class ExcelReader {
public String fileName;
public String sheetName;
public int sheetIndex;
public XSSFWorkbook book;
public ExcelReader(ExcelReaderBuilder excelReaderBuilder) {
this.fileName = excelReaderBuilder.fileName;
this.sheetIndex = excelReaderBuilder.sheetIndex;
this.sheetName = excelReaderBuilder.sheetName;
}
public static class ExcelReaderBuilder{
public String fileName;
public String sheetName;
public int sheetIndex;
public ExcelReaderBuilder setFileLocation(String location) {
this.fileName = location;
return this;
}
public ExcelReaderBuilder setSheet(String sheetName) {
this.sheetName = sheetName;
return this;
}
public ExcelReaderBuilder setSheet(int sheetIndex) {
this.sheetIndex = sheetIndex;
return this;
}
public ExcelReader build() {
return new ExcelReader(this);
}
#Override
public String toString() {
return "fileName = "+this.fileName+" , sheetName= "+this.sheetName+", sheetIndex="+this.sheetIndex;
}
}
public XSSFWorkbook getWorkBook(String filePath) throws InvalidFormatException, IOException {
return new XSSFWorkbook(new File(filePath));
}
public XSSFSheet getWorkBookSheet(String fileName, String sheetName) throws InvalidFormatException, IOException {
this.book = getWorkBook(fileName);
return this.book.getSheet(sheetName);
}
public XSSFSheet getWorkBookSheet(String fileName, int sheetIndex) throws InvalidFormatException, IOException {
this.book = getWorkBook(fileName);
return this.book.getSheetAt(sheetIndex);
}
public List<List<String>> getSheetData() throws IOException{
XSSFSheet sheet;
List<List<String>> outerList = new LinkedList<>();
try {
sheet = getWorkBookSheet(fileName, sheetName);
outerList = getSheetData(sheet);
} catch (InvalidFormatException e) {
throw new RuntimeException(e.getMessage());
}
return outerList;
}
public List<List<String>> getSheetDataAt() throws InvalidFormatException, IOException {
XSSFSheet sheet;
List<List<String>> outerList = new LinkedList<>();
try {
sheet = getWorkBookSheet(fileName, sheetIndex);
outerList = getSheetData(sheet);
} catch (InvalidFormatException e) {
throw new RuntimeException(e.getMessage());
}
return outerList;
}
public List<List<String>> getSheetData(XSSFSheet sheet) {
List<List<String>> outerList = new LinkedList<>();
prepareOutterList(sheet, outerList);
return Collections.unmodifiableList(outerList);
}
public void prepareOutterList(XSSFSheet sheet, List<List<String>> outerList) {
for (int i = sheet.getFirstRowNum(); i <= sheet.getLastRowNum(); i++) {
List<String> innerList = new LinkedList<>();
XSSFRow xssfRow = sheet.getRow(i);
for (int j = xssfRow.getFirstCellNum(); j < xssfRow.getLastCellNum(); j++) {
prepareInnerList(innerList, xssfRow, j);
}
outerList.add(Collections.unmodifiableList(innerList));
}
}
public void prepareInnerList(List<String> innerList, XSSFRow xssfRow, int j) {
switch (xssfRow.getCell(j).getCellType()) {
case Cell.CELL_TYPE_BLANK:
innerList.add("");
break;
case Cell.CELL_TYPE_STRING:
innerList.add(xssfRow.getCell(j).getStringCellValue());
break;
case Cell.CELL_TYPE_NUMERIC:
innerList.add(xssfRow.getCell(j).getNumericCellValue() + "");
break;
case Cell.CELL_TYPE_BOOLEAN:
innerList.add(xssfRow.getCell(j).getBooleanCellValue() + "");
break;
default:
throw new IllegalArgumentException("Cannot read the column : " + j);
}
}
Note - Unable to print/fetch the data from second sheet in cucumber using java Every time the data is rendered from sheet one only
In the feature:
Then i validate list of properties in page one with data in exel at "TestData.xlsx;2" '''
But in the StepDefinition file:
#Then("^I validate list of properties in page zero with data in excel at \"([^\"]*)\"$")
Be careful: Are you sure you are writing "excel" with a "c" in both lines? Or any other way, but the same all the time?
i want to get the (valid and invalid) data from excel .xls file and apply that in my application login page.when the error message is display.i want to write failed in excel file.from my code, i can get the value from excel and apply it in my application but cant write the output in excel.i attached my code here.anyone please help me to fix this.
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import org.apache.poi.EncryptedDocumentException;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxDriver;
public class exceltowrite {
public FileInputStream fis = null;
public FileOutputStream fos = null;
public HSSFWorkbook workbook = null;
public HSSFSheet sheet = null;
public HSSFRow row = null;
public HSSFCell cell = null;
String xlFilePath;
public exceltowrite(String string, String xlFilePath) throws IOException {
this.xlFilePath = xlFilePath;
fis = new FileInputStream(xlFilePath);
workbook = new HSSFWorkbook(fis);
fis.close();
}
public static void main(String[]args) throws EncryptedDocumentException, InvalidFormatException, IOException, InterruptedException {
for(int i=1;i<=29;i++) {
String u1 =exceltowrite.getdata(i,0);
String p1=exceltowrite.getdata(i,1);
WebDriver d1=new FirefoxDriver();
d1.get("http:\\www.google.com");
d1.get("napplication site");
d1.findElement(By.name("AdminLoginForm[email]")).sendKeys(u1);
d1.findElement(By.name("AdminLoginForm[password]")).sendKeys(p1);
d1.findElement(By.name("login-button")).click();
Thread.sleep(2000);
WebElement e1= d1.findElement(By.xpath("email error message path"));
String s1=e1.getText();
WebElement e2=d1.findElement(By.xpath("password error message path"));
String s2=e2.getText();
if(s1.equals("Email cannot be blank.")) {
System.out.println("emailid failed");
d1.quit();
if(s2.equals("Password cannot be blank.")) {
System.out.println("password failed");
d1.quit();
} else {
System.out.println("password pass");
d1.quit();
}
} else {
d1.quit();
}
}
}
public static String getdata(int i, int j) throws EncryptedDocumentException, InvalidFormatException, IOException {
FileInputStream fis=new FileInputStream("file path");
Workbook wb=WorkbookFactory.create(fis);
String s=wb.getSheet("Sheet1").getRow(i).getCell(j).getStringCellValue();
return s;
}
public boolean setCellData(String sheetName, int colNumber, int rowNum, String value)
{
try
{
sheet = workbook.getSheet(sheetName);
row = sheet.getRow(rowNum);
System.out.println(row);
cell = row.getCell(colNumber);
System.out.println(cell);
cell.setCellValue(value);
fos = new FileOutputStream(xlFilePath);
workbook.write(fos);
fos.close();
System.out.println("Finished");
}
catch (Exception ex)
{
ex.printStackTrace();
return false;
}
return true;
}}
}
my another class is this
public class Demo {
public static void main(String args[]) throws Exception
{
// exceltowrite at=new exceltowrite("file path");
for(int j=0;j<=29;j++){
at.setCellData("Sheet1",2,j,"failed");
}}}
public static void write() throws IOException {
String home = System.getProperty("user.home");
String downlpad = home + "\\Downloads";
File file = new File(downlpad + "\\" + "Suhit-File-to-Download.xlsx");
FileOutputStream outputStream = new FileOutputStream(file);
XSSFWorkbook workbook = new XSSFWorkbook();
XSSFSheet sheet = workbook.createSheet("Java Books");
ArrayList<Object> bookData = new ArrayList<>();
bookData.add("Name");
bookData.add("Address");
bookData.add("Number");
XSSFRow row = sheet.createRow(0);
for (int i = 0; i < bookData.size(); i++) {
XSSFCell cell3 = row.createCell(i);
cell3.setCellValue((String) bookData.get(i));
}
workbook.write(outputStream);
workbook.close();
}
I am using this code from (by author: lchen) which reads contents from excel file based on number of rows I provide into method 'readRow()'.
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.openxml4j.opc.PackageAccess;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.usermodel.XSSFRichTextString;
import org.xml.sax.InputSource;
public class TestLargeFileRead {
private int rowNum = 0;
private OPCPackage opcPkg;
private ReadOnlySharedStringsTable stringsTable;
private XMLStreamReader xmlReader;
public void XExcelFileReader(String excelPath) throws Exception {
opcPkg = OPCPackage.open(excelPath, PackageAccess.READ);
this.stringsTable = new ReadOnlySharedStringsTable(opcPkg);
XSSFReader xssfReader = new XSSFReader(opcPkg);
XMLInputFactory factory = XMLInputFactory.newInstance();
InputStream inputStream = xssfReader.getSheetsData().next();
xmlReader = factory.createXMLStreamReader(inputStream);
while (xmlReader.hasNext()) {
xmlReader.next();
if (xmlReader.isStartElement()) {
if (xmlReader.getLocalName().equals("sheetData"))
break;
}
}
}
public int rowNum() {
return rowNum;
}
public List<String[]> readRows(int batchSize) throws XMLStreamException {
String elementName = "row";
List<String[]> dataRows = new ArrayList<String[]>();
if (batchSize > 0) {
while (xmlReader.hasNext()) {
xmlReader.next();
if (xmlReader.isStartElement()) {
if (xmlReader.getLocalName().equals(elementName)) {
rowNum++;
dataRows.add(getDataRow());
if (dataRows.size() == batchSize)
break;
}
}
}
}
return dataRows;
}
private String[] getDataRow() throws XMLStreamException {
List<String> rowValues = new ArrayList<String>();
while (xmlReader.hasNext()) {
xmlReader.next();
if (xmlReader.isStartElement()) {
if (xmlReader.getLocalName().equals("c")) {
CellReference cellReference = new CellReference(
xmlReader.getAttributeValue(null, "r"));
// Fill in the possible blank cells!
while (rowValues.size() < cellReference.getCol()) {
rowValues.add("");
}
String cellType = xmlReader.getAttributeValue(null, "t");
rowValues.add(getCellValue(cellType));
}
} else if (xmlReader.isEndElement()
&& xmlReader.getLocalName().equals("row")) {
break;
}
}
return rowValues.toArray(new String[rowValues.size()]);
}
private String getCellValue(String cellType) throws XMLStreamException {
String value = ""; // by default
while (xmlReader.hasNext()) {
xmlReader.next();
if (xmlReader.isStartElement()) {
if (xmlReader.getLocalName().equals("v")) {
if (cellType != null && cellType.equals("s")) {
int idx = Integer.parseInt(xmlReader.getElementText());
return new XSSFRichTextString(
stringsTable.getEntryAt(idx)).toString();
} else {
return xmlReader.getElementText();
}
}
} else if (xmlReader.isEndElement()
&& xmlReader.getLocalName().equals("c")) {
break;
}
}
return value;
}
#Override
protected void finalize() throws Throwable {
if (opcPkg != null)
opcPkg.close();
super.finalize();
}
public static void main(String[] args) {
try {
TestLargeFileRead howto = new TestLargeFileRead();
howto.XExcelFileReader("D:\\TEMP_CATALOG\\H1.xlsx");
} catch (Exception e) {
e.printStackTrace();
}
}
}
But it reads only First SHEET's contents and discards other subsequent SHEETS. My requirement is to read SHEET name; and based on name read that SHEET's contents. Can anyone help me to customize this above code fetch SHEET NAME and their contents ? please ?
The key class you need to work with, and tweak your use of, is XSSFReader. If you take a look at the Javadocs for it, you'll see it provides an Iterator of InputStreams of all the sheets, and a way to get at the root Workbook stream.
If you want to access all the sheets, you need to change these lines:
InputStream inputStream = xssfReader.getSheetsData().next();
xmlReader = factory.createXMLStreamReader(inputStream);
Into something more like:
Iterator<InputStream> sheetsData = xssfReader.getSheetsData();
while (sheetsData.hasNext()) {
InputStream inputStream = sheetsData.next();
xmlReader = factory.createXMLStreamReader(inputStream);
....
}
If you also want to get the sheet name as well, you'll want to do something like what is shown in the Apache POI XLSX event-based text extractor
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader.getSheetsData();
while (sheetsData.hasNext()) {
InputStream inputStream = sheetsData.next();
String sheetName = iter.getSheetName();
if (sheetName.equalsIgnoreCase("TheSheetIWant")) {
xmlReader = factory.createXMLStreamReader(inputStream);
....
}
}
If you want to know more about doing this stuff, then one of the best examples, that's easy to read and follow, is XSSFEventBasedExcelExtractor that comes with Apache POI - read the code for that and learn!
This is sort of inline w/ Writing a large ResultSet to a File but the file in question is an Excel file.
I'm using the Apache POI library to write an Excel file with a large data set retrieved from a ResultSet object. The data could range from a few thousand records to about 1 million; not sure how this translates into file system bytes in Excel format.
The following is a test code I wrote to check out the time taken to write such a large result set and also the performance implication w.r.t CPU & Memory.
protected void writeResultsetToExcelFile(ResultSet rs, int numSheets, String fileNameAndPath) throws Exception {
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(fileNameAndPath));
int numColumns = rs.getMetaData().getColumnCount();
Workbook wb = ExcelFileUtil.createExcelWorkBook(true, numSheets);
Row heading = wb.getSheetAt(0).createRow(1);
ResultSetMetaData rsmd = rs.getMetaData();
for(int x = 0; x < numColumns; x++) {
Cell cell = heading.createCell(x+1);
cell.setCellValue(rsmd.getColumnLabel(x+1));
}
int rowNumber = 2;
int sheetNumber = 0;
while(rs.next()) {
if(rowNumber == 65001) {
log("Sheet " + sheetNumber + "written; moving onto to sheet " + (sheetNumber + 1));
sheetNumber++;
rowNumber = 2;
}
Row row = wb.getSheetAt(sheetNumber).createRow(rowNumber);
for(int y = 0; y < numColumns; y++) {
row.createCell(y+1).setCellValue(rs.getString(y+1));
wb.write(bos);
}
rowNumber++;
}
//wb.write(bos);
bos.close();
}
Not much luck with the above code. The file which is created seems to grow rapidly (~70Mb per sec). So I stopped the execution after about 10 minutes (killed the JVM when the file reaches 7Gb) and tried to open the file in Excel 2007. The moment I open it, the file size becomes 8k(!) and only the header and the first row are created. Not sure what I'm missing here.
Any ideas?
Using SXSSF poi 3.8
package example;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.xssf.streaming.SXSSFSheet;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
public class SXSSFexample {
public static void main(String[] args) throws Throwable {
FileInputStream inputStream = new FileInputStream("mytemplate.xlsx");
XSSFWorkbook wb_template = new XSSFWorkbook(inputStream);
inputStream.close();
SXSSFWorkbook wb = new SXSSFWorkbook(wb_template);
wb.setCompressTempFiles(true);
SXSSFSheet sh = (SXSSFSheet) wb.getSheetAt(0);
sh.setRandomAccessWindowSize(100);// keep 100 rows in memory, exceeding rows will be flushed to disk
for(int rownum = 4; rownum < 100000; rownum++){
Row row = sh.createRow(rownum);
for(int cellnum = 0; cellnum < 10; cellnum++){
Cell cell = row.createCell(cellnum);
String address = new CellReference(cell).formatAsString();
cell.setCellValue(address);
}
}
FileOutputStream out = new FileOutputStream("tempsxssf.xlsx");
wb.write(out);
out.close();
}
}
It requires:
poi-ooxml-3.8.jar,
poi-3.8.jar,
poi-ooxml-schemas-3.8.jar,
stax-api-1.0.1.jar,
xml-apis-1.0.b2.jar,
xmlbeans-2.3.0.jar,
commons-codec-1.5.jar,
dom4j-1.6.1.jar
Useful link
Oh. I think you're writing the workbook out 944,000 times. Your wb.write(bos) call is in the inner loop. I'm not sure this is quite consistent with the semantics of the Workbook class? From what I can tell in the Javadocs of that class, that method writes out the entire workbook to the output stream specified. And it's gonna write out every row you've added so far once for every row as the thing grows.
This explains why you're seeing exactly 1 row, too. The first workbook (with one row) to be written out to the file is all that is being displayed - and then 7GB of junk thereafter.
Unless you have to write formulas or formatting you should consider writing out a .csv file. Infinitely simpler, infinitely faster, and Excel will do the conversion to .xls or .xlsx automatically and correctly by definition.
You can using SXSSFWorkbook implementation of Workbook, if you use style in your excel ,You can caching style by Flyweight Pattern to improve your performance.
You can increase the performance of excel export by following these steps:
1) When you fetch data from database, avoid casting the result set to the list of entity classes. Instead assign it directly to List
List<Object[]> resultList =session.createSQLQuery("SELECT t1.employee_name, t1.employee_id ... from t_employee t1 ").list();
instead of
List<Employee> employeeList =session.createSQLQuery("SELECT t1.employee_name, t1.employee_id ... from t_employee t1 ").list();
2) Create excel workbook object using SXSSFWorkbook instead of XSSFWorkbook and create new row using SXSSFRow when the data is not empty.
3) Use java.util.Iterator to iterate the data list.
Iterator itr = resultList.iterator();
4) Write data into excel using column++.
int rowCount = 0;
int column = 0;
while(itr.hasNext()){
SXSSFRow row = xssfSheet.createRow(rowCount++);
Object[] object = (Object[]) itr.next();
//column 1
row.setCellValue(object[column++]); // write logic to create cell with required style in setCellValue method
//column 2
row.setCellValue(object[column++]);
itr.remove();
}
5) While iterating the list, write the data into excel sheet and remove the row from list using remove method. This is to avoid holding unwanted data from the list and clear the java heap size.
itr.remove();
For now I took #Gian's advice & limited the number of records per Workbook to 500k and rolled over the rest to the next Workbook. Seems to be working decent. For the above configuration, it took me about 10 mins per workbook.
I updated BigGridDemo to support multiple sheets.
BigExcelWriterImpl.java
package com.gdais.common.apache.poi.bigexcelwriter;
import static com.google.common.base.Preconditions.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.io.FilenameUtils;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
public class BigExcelWriterImpl implements BigExcelWriter {
private static final String XML_ENCODING = "UTF-8";
#Nonnull
private final File outputFile;
#Nullable
private final File tempFileOutputDir;
#Nullable
private File templateFile = null;
#Nullable
private XSSFWorkbook workbook = null;
#Nonnull
private LinkedHashMap<String, XSSFSheet> addedSheets = new LinkedHashMap<String, XSSFSheet>();
#Nonnull
private Map<XSSFSheet, File> sheetTempFiles = new HashMap<XSSFSheet, File>();
BigExcelWriterImpl(#Nonnull File outputFile) {
this.outputFile = outputFile;
this.tempFileOutputDir = outputFile.getParentFile();
}
#Override
public BigExcelWriter createWorkbook() {
workbook = new XSSFWorkbook();
return this;
}
#Override
public BigExcelWriter addSheets(String... sheetNames) {
checkState(workbook != null, "workbook must be created before adding sheets");
for (String sheetName : sheetNames) {
XSSFSheet sheet = workbook.createSheet(sheetName);
addedSheets.put(sheetName, sheet);
}
return this;
}
#Override
public BigExcelWriter writeWorkbookTemplate() throws IOException {
checkState(workbook != null, "workbook must be created before writing template");
checkState(templateFile == null, "template file already written");
templateFile = File.createTempFile(FilenameUtils.removeExtension(outputFile.getName())
+ "-template", ".xlsx", tempFileOutputDir);
System.out.println(templateFile);
FileOutputStream os = new FileOutputStream(templateFile);
workbook.write(os);
os.close();
return this;
}
#Override
public SpreadsheetWriter createSpreadsheetWriter(String sheetName) throws IOException {
if (!addedSheets.containsKey(sheetName)) {
addSheets(sheetName);
}
return createSpreadsheetWriter(addedSheets.get(sheetName));
}
#Override
public SpreadsheetWriter createSpreadsheetWriter(XSSFSheet sheet) throws IOException {
checkState(!sheetTempFiles.containsKey(sheet), "writer already created for this sheet");
File tempSheetFile = File.createTempFile(
FilenameUtils.removeExtension(outputFile.getName())
+ "-sheet" + sheet.getSheetName(), ".xml", tempFileOutputDir);
Writer out = null;
try {
out = new OutputStreamWriter(new FileOutputStream(tempSheetFile), XML_ENCODING);
SpreadsheetWriter sw = new SpreadsheetWriterImpl(out);
sheetTempFiles.put(sheet, tempSheetFile);
return sw;
} catch (RuntimeException e) {
if (out != null) {
out.close();
}
throw e;
}
}
private static Function<XSSFSheet, String> getSheetName = new Function<XSSFSheet, String>() {
#Override
public String apply(XSSFSheet sheet) {
return sheet.getPackagePart().getPartName().getName().substring(1);
}
};
#Override
public File completeWorkbook() throws IOException {
FileOutputStream out = null;
try {
out = new FileOutputStream(outputFile);
ZipOutputStream zos = new ZipOutputStream(out);
Iterable<String> sheetEntries = Iterables.transform(sheetTempFiles.keySet(),
getSheetName);
System.out.println("Sheet Entries: " + sheetEntries);
copyTemplateMinusEntries(templateFile, zos, sheetEntries);
for (Map.Entry<XSSFSheet, File> entry : sheetTempFiles.entrySet()) {
XSSFSheet sheet = entry.getKey();
substituteSheet(entry.getValue(), getSheetName.apply(sheet), zos);
}
zos.close();
out.close();
return outputFile;
} finally {
if (out != null) {
out.close();
}
}
}
private static void copyTemplateMinusEntries(File templateFile,
ZipOutputStream zos, Iterable<String> entries) throws IOException {
ZipFile templateZip = new ZipFile(templateFile);
#SuppressWarnings("unchecked")
Enumeration<ZipEntry> en = (Enumeration<ZipEntry>) templateZip.entries();
while (en.hasMoreElements()) {
ZipEntry ze = en.nextElement();
if (!Iterables.contains(entries, ze.getName())) {
System.out.println("Adding template entry: " + ze.getName());
zos.putNextEntry(new ZipEntry(ze.getName()));
InputStream is = templateZip.getInputStream(ze);
copyStream(is, zos);
is.close();
}
}
}
private static void substituteSheet(File tmpfile, String entry,
ZipOutputStream zos)
throws IOException {
System.out.println("Adding sheet entry: " + entry);
zos.putNextEntry(new ZipEntry(entry));
InputStream is = new FileInputStream(tmpfile);
copyStream(is, zos);
is.close();
}
private static void copyStream(InputStream in, OutputStream out) throws IOException {
byte[] chunk = new byte[1024];
int count;
while ((count = in.read(chunk)) >= 0) {
out.write(chunk, 0, count);
}
}
#Override
public Workbook getWorkbook() {
return workbook;
}
#Override
public ImmutableList<XSSFSheet> getSheets() {
return ImmutableList.copyOf(addedSheets.values());
}
}
SpreadsheetWriterImpl.java
package com.gdais.common.apache.poi.bigexcelwriter;
import java.io.IOException;
import java.io.Writer;
import java.util.Calendar;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.ss.util.CellReference;
class SpreadsheetWriterImpl implements SpreadsheetWriter {
private static final String XML_ENCODING = "UTF-8";
private final Writer _out;
private int _rownum;
SpreadsheetWriterImpl(Writer out) {
_out = out;
}
#Override
public SpreadsheetWriter closeFile() throws IOException {
_out.close();
return this;
}
#Override
public SpreadsheetWriter beginSheet() throws IOException {
_out.write("<?xml version=\"1.0\" encoding=\""
+ XML_ENCODING
+ "\"?>"
+
"<worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\">");
_out.write("<sheetData>\n");
return this;
}
#Override
public SpreadsheetWriter endSheet() throws IOException {
_out.write("</sheetData>");
_out.write("</worksheet>");
closeFile();
return this;
}
/**
* Insert a new row
*
* #param rownum
* 0-based row number
*/
#Override
public SpreadsheetWriter insertRow(int rownum) throws IOException {
_out.write("<row r=\"" + (rownum + 1) + "\">\n");
this._rownum = rownum;
return this;
}
/**
* Insert row end marker
*/
#Override
public SpreadsheetWriter endRow() throws IOException {
_out.write("</row>\n");
return this;
}
#Override
public SpreadsheetWriter createCell(int columnIndex, String value, int styleIndex)
throws IOException {
String ref = new CellReference(_rownum, columnIndex).formatAsString();
_out.write("<c r=\"" + ref + "\" t=\"inlineStr\"");
if (styleIndex != -1) {
_out.write(" s=\"" + styleIndex + "\"");
}
_out.write(">");
_out.write("<is><t>" + value + "</t></is>");
_out.write("</c>");
return this;
}
#Override
public SpreadsheetWriter createCell(int columnIndex, String value) throws IOException {
createCell(columnIndex, value, -1);
return this;
}
#Override
public SpreadsheetWriter createCell(int columnIndex, double value, int styleIndex)
throws IOException {
String ref = new CellReference(_rownum, columnIndex).formatAsString();
_out.write("<c r=\"" + ref + "\" t=\"n\"");
if (styleIndex != -1) {
_out.write(" s=\"" + styleIndex + "\"");
}
_out.write(">");
_out.write("<v>" + value + "</v>");
_out.write("</c>");
return this;
}
#Override
public SpreadsheetWriter createCell(int columnIndex, double value) throws IOException {
createCell(columnIndex, value, -1);
return this;
}
#Override
public SpreadsheetWriter createCell(int columnIndex, Calendar value, int styleIndex)
throws IOException {
createCell(columnIndex, DateUtil.getExcelDate(value, false), styleIndex);
return this;
}
#Override
public SpreadsheetWriter createCell(int columnIndex, Calendar value)
throws IOException {
createCell(columnIndex, value, -1);
return this;
}
}