Develop Reference

how to retrieve value from specific columns of excel using spring rest

Below is my controller code and service code. The service code is returning only the last value of the map, though I am adding it in the for loop. Not sure what I missing. Also in the postman, I am not getting any response, just the success code as 200. At least I should be getting the last value. The expected output is the following:
"testCaseId": "TA_SCEN01",
"testCaseDescription" : some test
"executionStatus": "Yes",
"executionBrowser": "Chrome"
Please excuse me for the long desciption of the problem statement. I just tried to give as much info for better understanding.
the controller code is below
#RestController
public class ExcelController {
#Autowired
ExcelService service;
#GetMapping(value="/{excelSheetName}/{sheetName}", produces = "application/json")
public ResponseEntity<List<Map<String, String>>> getExcelDetails(#PathVariable("excelSheetName") String excelSheetName, #PathVariable("sheetName") String sheetName) {
service.getExcelDetails(excelSheetName, sheetName);
return new ResponseEntity<List<Map<String, String>>>(HttpStatus.OK);
}
}
The service code is below
public List<Map<String, String>> getExcelDetails(String excelSheetName, String sheetName) {
//Map<String, String> map = new LinkedHashMap<String, String>();
//List list = new LinkedList<Object>();
List list = new LinkedList<Object>();
String excelPath = "C:\\Users\\140117\\Desktop\\NSARF\\Successfactor_TalentAcquisition_TestCase.xls";
FileInputStream fis;
try {
fis = new FileInputStream(excelPath);
HSSFWorkbook workSheet = new HSSFWorkbook(fis);
HSSFSheet sheet = workSheet.getSheetAt(0);
ExcelDetail excelObj = new ExcelDetail();
String testId = "testCaseId";
String testDesc = "testCaseDescription";
String status = "executionStatus";
String browser = "executionBrowser";
for (int i = 0; i < sheet.getPhysicalNumberOfRows(); i++) {
Map map = new LinkedHashMap();
//Map<String, String> map = new LinkedHashMap<String, String>();
Cell tcId = sheet.getRow(i + 1).getCell(0);
Cell desc = sheet.getRow(i+1).getCell(1);
Cell execStatus = sheet.getRow(i + 1).getCell(2);
Cell execBrowser = sheet.getRow(i+1).getCell(5);
if(null != tcId || null != desc || execStatus != null || execBrowser != null) {
map.put(testId, tcId.toString());
map.put(testDesc, desc.toString());
map.put(status, execStatus.toString());
map.put(browser, execBrowser.toString());
list.add(map);
}else {
break;
}
//list.add(map);
System.out.println(list);
}
//list.add(map);
System.out.println(list);
return list;
}

Clear an arraylist after return statement

I have a method that returns an ArrayList. I want to clear or re-initialise the the array list after the return statement so that the old data is not repopulated. I know we can use arrayList.clear() statement but I am not sure where to place it in the code so that the re-initialization happens after returning the array list .
My code snippet :
public static List<HashMap<String, String>> GetDataIteration(String strDataSheetPath, String sTCID) {
HashMap<String, String> hm;
//List<Map<String, String>> hmList = new ArrayList();
//List<HashMap<String, String>> hmList = new ArrayList<HashMap<String, String>>();
try {
String sTestDataFile ="C:\\AutomationGoldCopy\\IEDSS_Automation\\src\\TestData\\Support\\Support_TC_001_AddUserBuildProfile1.xlsx";
File file = new File(sTestDataFile);
FileInputStream inputstream = new FileInputStream(file);
Workbook wb = null;
if (sTestDataFile.contains(".xlsx"))
{
wb = new XSSFWorkbook(inputstream);
} else if (sTestDataFile.contains(".xls"))
{
wb = new HSSFWorkbook(inputstream);
}
Sheet sheet = wb.getSheetAt(0);
int count = sheet.getPhysicalNumberOfRows() ;
System.out.println(count);
for (int i=1 ; i <count ; i++)
{
Row headerrow = sheet.getRow(0);
hm = new HashMap<String, String>();
System.out.println(i);
Row currentRow = sheet.getRow(i);
for (int j = 0; j < headerrow.getPhysicalNumberOfCells(); j++)
{
Cell currentCell = currentRow.getCell(j,Row.CREATE_NULL_AS_BLANK);
hm.put(headerrow.getCell(j).toString(), currentCell.toString());
}
//hmList.clear();
hmList.add(hm);
for (HashMap<String, String> h : hmList) {
for (String key : h.keySet()) {
System.out.println(key + "\t" + h.get(key));
}
System.out.println("\n");
}
// hmList.clear();
}
} catch (Exception e) {
System.out.println(e);
}
return hmList;
}
Kindly help me with the same.
main method :
public static void main(String[] args) {
String dtDataSheetPath = null;
String dtTestCaseID = null;
List<HashMap<String, String>> TestDataList = Excel.GetDataIteration(dtDataSheetPath,dtTestCaseID);
for(Map<String, String> TestData : TestDataList) {
String name = TestData.get("TestCase Name");
String name1 = TestData.get("SMU_LastName");
System.out.println( "Got it : " + name + " " + name1);
}
There are three rows in the excel . The first is the hearder row and the last two rows are the header values.
I am converting the excel rows into a hashmap with header row as keys and the second and third rows as values of the hashmap.
In the second run ( when i =2 ) , the first row is repopulated again. I just wanted the second row. When I clear the ArrayList hmList , it works fine but I wanted to return the list too.

In your function the initialization hmList = new ArrayList>(); creates a new empty List each time the function is called so there is no need to clear it after the return statement (which, anyway, is not possible in the same function).

As hmList seems to me a static variable you can return a copy of this list and clear the hmList before the return, if your specification says hmList should be empty by the return of your function.

Out of memory exception for more than 1 lac data

Hi I am facing issue with inserting 1lac data in a table with two columns(Id & Id). Can anybody optimise the code.
public class edgeConnection {
static ArrayList al3 = new ArrayList();
static HashSet set=null;
//static HashMap hm = null;
//static int val ;
//Database connection
public static DataSource getMySQLDataSource() throws Exception {
Properties props = new Properties();
FileInputStream fis = null;
MysqlDataSource mysqlDS = null;
try {
fis = new FileInputStream("D:/Assignments/Sequence/db.properties");
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
props.load(fis);
mysqlDS = new MysqlDataSource();
mysqlDS.setURL(props.getProperty("MYSQL_DB_URL"));
mysqlDS.setUser(props.getProperty("MYSQL_DB_USERNAME"));
mysqlDS.setPassword(props.getProperty("MYSQL_DB_PASSWORD"));
return mysqlDS;
}
//Adding values to Hashset
private static int addNode(){
set = new HashSet();
for(int i=1;i<=10000;i++){/*Change 10000 to 30000*/
set.add(i);
}
return 0;
}
private static int keyNode(int i){
int counter = 1;
Iterator it = set.iterator();
while(it.hasNext())
{
int value = (int) it.next();
if(i==counter)
{
//System.out.println("key value returned ::"+value);
return value;
}
counter++;
}
return 0;
}
private static String pairGenerator(){
ArrayList numbers1 = new ArrayList<Integer>();
Random randomGenerator1 = new Random();
while (numbers1.size() < 1)
{
int random = randomGenerator1 .nextInt(15);
if (!numbers1.contains(random)) {
numbers1.add(random);
}
}
Iterator it1 = numbers1.iterator();
while(it1.hasNext()){
return(String.valueOf(it1.next()));
}
return null;
}
private static List valueNodes(){
//Generate no randomly.
ArrayList<Integer> numbers = new ArrayList<Integer>();
Random randomGenerator = new Random();
String size = pairGenerator();
int size1= Integer.parseInt(size)+1;
//System.out.println("the size1 is ::"+size1);
while (numbers.size() < size1)
{
int random = randomGenerator .nextInt(10000);/*Change 10000 to 50000*/
if (!numbers.contains(random)) {
numbers.add(random);
}
}
Iterator it = numbers.iterator();
al3.clear();
while(it.hasNext()){
int listvalue = (int) it.next();
al3.add(listvalue);
//System.out.println(it.next());
}
//System.out.println(al3);
return al3;
}
public static void main(String[] args) throws Exception {
Connection con = null;
PreparedStatement pst = null;
ResultSet rs = null;
HashMap<Integer, List<String>> hm = new HashMap<Integer, List<String>>();
addNode();
//System.out.println("size of set is:"+set.size());
try {
con = getMySQLDataSource().getConnection();
List<Integer> valueList = new ArrayList<Integer>();
int nodeId;
for(int i=1;i<=set.size();i++)
{
hm.put(keyNode(i), valueNodes());
Iterator iter = hm.entrySet().iterator();
while(iter.hasNext())
{
Map.Entry entry = (Map.Entry) iter.next();
System.out.println(entry.getKey()+"<-->"+" "+entry.getValue());
nodeId = (int) entry.getKey();
valueList = (List<Integer>) entry.getValue();
//System.out.println("size of value list : "+valueList.size());
for(int j = 0;j<valueList.size();j++)
{
pst = con.prepareStatement("insert into nodes_connection values (?,?)");
pst.setInt(1, nodeId);
if(valueList.get(j)!=0)
{
pst.setInt(2,valueList.get(j));
}
else{
int updatedValue = valueList.get(j)+10000;/*Change 10000 to 30000*/
pst.setInt(2,updatedValue);
}
pst.executeUpdate();
//System.out.println(j+"record updated..");
}
iter.remove();
}
}
System.out.println("Record successfully added");
} catch (SQLException e) {
e.printStackTrace();
}finally{
try {
if(rs != null) rs.close();
if(pst != null) pst.close();
if(con != null) con.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}`
I need to remove the arraylist and hashmap. How could i optimise the code so that i wont get any java heap memory exception.

Since you are inserting so many rows, you should use batch update instead of inserting one row at a time.
PreparedStatement pst = con.prepareStatement("insert into nodes_connection values (?,?)");
for(int i=1;i<=set.size();i++)
{
hm.put(keyNode(i), valueNodes());
Iterator iter = hm.entrySet().iterator();
while(iter.hasNext())
{
Map.Entry entry = (Map.Entry) iter.next();
System.out.println(entry.getKey()+"<-->"+" "+entry.getValue());
nodeId = (int) entry.getKey();
valueList = (List<Integer>) entry.getValue();
//System.out.println("size of value list : "+valueList.size());
for(int j = 0;j<valueList.size();j++)
{
pst.setInt(1, nodeId);
if(valueList.get(j)!=0)
{
pst.setInt(2,valueList.get(j));
}
else{
int updatedValue = valueList.get(j)+10000;/*Change 10000 to 30000*/
pst.setInt(2,updatedValue);
}
pst.addBatch()
//System.out.println(j+"record updated..");
}
iter.remove();
}
}
pst.executeBatch()
You can go here for more info on batch inserts.

By far and away the easiest thing for you to do is to just increase the heap size.
If this is something you're only going to run once, or are playing around, basically anything that isn't production critical or will process HUGE amounts of data, then increasing the heap will give you what you want.
If you do need to keep the memory footprint down you will need to stream your data instead of reading it all into memory. Looking over your code it seems each line of data has no relation to the rest of the data in the file, so in a pseudo-code way, something like the following would work:
For each line in file
Calculate data to be inserted into database
Update database
You can increase efficiency more by instead of updating the database for every line, doing it in a batch, which would change your pseudo to this:
For each line in file
Calculate data to be inserted into database
Add update to a JDBC batch
If batch size > :somelimit
execute batch
Execute final batch

How to retrive specific rows from a matrix returned by reading a CSV file in java

I want to retrieve some rows of a 2d array.
example: I have file named as "data.csv", which contains
age sex zipcode classtype
21 m 23423 1
12 f 23133 2
23 m 32323 2
23 f 23211 1
The below mentioned code will give output like this:
{age=[21,12,23,23],sex=[m,f,m,f],zipcode=[23423,23133,32323,23211],classtype=[1,2,2,1]}
Now I want to retrieve rows which have classtype 1 and store this values in a new 2d array.
like partition1={{21,m,23423,1},{23,f,23211,1}}
public class CsvParser {
public static void main(String[] args) {
try {
FileReader fr = new FileReader((args.length > 0) ? args[0] : "data.csv");
Map<String, List<String>> values = parseCsv(fr, " ", true);
System.out.println(values);
List<List<String>> partition1 = new ArrayList<>(25);
List<String> classTypes = values.get("classtype");
for (int row = 0; row < classTypes.size(); row++) {
String classType = classTypes.get(row);
if ("1".equals(classType)) {
List<String> data = new ArrayList<>(25);
data.add(values.get("age").get(row));
data.add(values.get("sex").get(row));
data.add(values.get("zipcode").get(row));
data.add(values.get("classtype").get(row));
partition1.add(data);
}
}
System.out.println(partition1);
} catch (IOException e) {
e.printStackTrace();
}
}
public static Map<String, List<String>> parseCsv(Reader reader, String separator, boolean hasHeader) throws IOException {
Map<String, List<String>> values = new LinkedHashMap<String, List<String>>();
List<String> columnNames = new LinkedList<String>();
BufferedReader br = null;
br = new BufferedReader(reader);
String line;
int numLines = 0;
while ((line = br.readLine()) != null) {
if (StringUtils.isNotBlank(line)) {
if (!line.startsWith("#")) {
String[] tokens = line.split(separator);
if (tokens != null) {
for (int i = 0; i < tokens.length; ++i) {
if (numLines == 0) {
columnNames.add(hasHeader ? tokens[i] : ("row_"+i));
} else {
List<String> column = values.get(columnNames.get(i));
if (column == null) {
column = new LinkedList<String>();
}
column.add(tokens[i]);
values.put(columnNames.get(i), column);
}
}
}
++numLines;
}
}
}
return values;
}
}

FileReader file1 = new FileReader(file);
BufferedReader buffer = new BufferedReader(file1);
String line = "";
while ((line = buffer.readLine()) != null) {
StringBuilder sb = new StringBuilder();
String[] str = line.split(",");
if(str[0]!=null||str[1]!=null||str[2]!=null){
sb.append("'" + str[0] + "',");
sb.append("'" +str[1] + "',");
sb.append("'" +str[2] + "'");
}
CSV File Must to be split comma based it should be work

Once I changed Map<String, List<String>> values = parseCsv(fr, "\\s,", true); to Map<String, List<String>> values = parseCsv(fr, " ", true); I was able to get the data in the right format...
From there it was just a matter to read through each row of classtype, when I found a value that matched 1, I would pull out each property for the given row and add it to a List, forming a single row. This was then added to another List which would maintain all the matching rows, for example...
List<List<String>> partition1 = new ArrayList<>(25);
List<String> classTypes = values.get("classtype");
for (int row = 0; row < classTypes.size(); row++) {
String classType = classTypes.get(row);
if ("1".equals(classType)) {
List<String> data = new ArrayList<>(25);
data.add(values.get("age").get(row));
data.add(values.get("sex").get(row));
data.add(values.get("zipcode").get(row));
data.add(values.get("classtype").get(row));
partition1.add(data);
}
}
System.out.println(partition1);
Which outputs...
[[21, m, 23423, 1], [23, f, 23211, 1]]
If you're looking for a more automated method, then I'm afraid you're out of luck, as Map makes no guarantee about the order that the keys are stored, iterated.
Of course, instead of using a List<List>, you could use a List<Map> which would maintain the keys for each value, for example...
List<Map<String, String>> partition1 = new ArrayList<>(25);
List<String> classTypes = values.get("classtype");
for (int row = 0; row < classTypes.size(); row++) {
String classType = classTypes.get(row);
if ("1".equals(classType)) {
Map<String, String> data = new HashMap<>(25);
for (String key : values.keySet()) {
data.put(key, values.get(key).get(row));
}
partition1.add(data);
}
}
System.out.println(partition1);
Which outputs...
[{sex=m, classtype=1, zipcode=23423, age=21}, {sex=f, classtype=1, zipcode=23211, age=23}]

word count frequency in document

I have a directory in which I have 1000 txt.files in it. I want to know for every word how many times it occurs in the 1000 document. So say even the word "cow" occured 100 times in X it will still be counted as one. If it occured in a different document it is incremented by one. So the maximum is 1000 if "cow" appears in every single document. How do I do this the easy way without the use of any other external library. Here's what I have so far
private Hashtable<String, Integer> getAllWordCount()
private Hashtable<String, Integer> getAllWordCount()
{
Hashtable<String, Integer> result = new Hashtable<String, Integer>();
HashSet<String> words = new HashSet<String>();
try {
for (int j = 0; j < fileDirectory.length; j++){
File theDirectory = new File(fileDirectory[j]);
File[] children = theDirectory.listFiles();
for (int i = 0; i < children.length; i++){
Scanner scanner = new Scanner(new FileReader(children[i]));
while (scanner.hasNext()){
String text = scanner.next().replaceAll("[^A-Za-z0-9]", "");
if (words.contains(text) == false){
if (result.get(text) == null)
result.put(text, 1);
else
result.put(text, result.get(text) + 1);
words.add(text);
}
}
}
words.clear();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println(result.size());
return result;
}

You also need a HashSet<String> in which you store each unique word you've read from the current file.
Then after every word read, you should check if it's in the set, if it isn't, increment the corresponding value in the result map (or add a new entry if it was empty, like you already do) and add the word to the set.
Don't forget to reset the set when you start to read a new file though.

how about this?
private Hashtable<String, Integer> getAllWordCount()
{
Hashtable<String, Integer> result = new Hashtable<String, Integer>();
HashSet<String> words = new HashSet<String>();
try {
for (int j = 0; j < fileDirectory.length; j++){
File theDirectory = new File(fileDirectory[j]);
File[] children = theDirectory.listFiles();
for (int i = 0; i < children.length; i++){
Scanner scanner = new Scanner(new FileReader(children[i]));
while (scanner.hasNext()){
String text = scanner.next().replaceAll("[^A-Za-z0-9]", "");
words.add(text);
}
for (String word : words) {
Integer count = result.get(word)
if (result.get(word) == null) {
result.put(word, 1);
} else {
result.put(word, result.get(word) + 1);
}
}
words.clear();
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println(result.size());
return result;
}