Java CSV Writing - java

Im currently trying to write data into excel for a report. I can write data to the csv file however its not coming out in excel in the order I want. I need the data to print under best and worst fitness in each column instead of it all print under Average. Here is the relevant code, any help would be appreciated:
String [] Fitness = "Average fitness#Worst fitness #Best Fitness".split("#");
writer.writeNext(Fitness);
//takes data from average fitness and stores as an int
int aFit = myPop.individuals[25].getFitness();
//converts int to string
String aFit1 = Integer.toString(aFit);
//converts string to string array
String aFit2 [] = aFit1.split(" ");
//writes to csv
writer.writeNext(aFit2);
//String [] nextCol = "#".split("#");
int wFit = myPop.individuals[49].getFitness();
String wFit1 = Integer.toString(wFit);
String wFit2 [] = wFit1.split(" ");
writer.writeNext(wFit2);
int bFit = myPop.individuals[1].getFitness();
String bFit1 = Integer.toString(bFit);
String bFit2 [] = bFit1.split(" ");
writer.writeNext(bFit2);
enter image description here

I think you should call your "writeNext" method once per line of datas:
String [] Fitness = "Average fitness#Worst fitness #Best Fitness".split("#");
writer.writeNext(Fitness);
int aFit = myPop.individuals[25].getFitness();
String aFit1 = Integer.toString(aFit);
int wFit = myPop.individuals[49].getFitness();
String wFit1 = Integer.toString(wFit);
int bFit = myPop.individuals[1].getFitness();
String bFit1 = Integer.toString(bFit);
writer.writeNext(new String[]{aFit1, wFit1, bFit1});

From the docs at
CSVWriter.html#writeNext(java.lang.String[])
public void writeNext(String[] nextLine)
- Writes the next line to the file.
The String array to provide is
A string array with each comma-separated element as a separate entry.
You are writing 3 separate lines instead of 1 and each line you write contains an Array with a single entry.
writer.writeNext(aFit2);
writer.writeNext(wFit2);
writer.writeNext(bFit2);
Solution:
Create a single Array with all 3 entries (column values) and write that once on a single line.

I am assuming you are using CSVWriter to write to a CSV file. Please make sure to mention as much details as possible in a question, it makes it much more readable to others.
As you can see from the documentation of CSVWriter:
void writeNext(String[] nextLine)
Writes the next line to the file.
The writeNext method actually writes the array to the an individual line of the file. From your code:
writer.writeNext(aFit2);
writer.writeNext(wFit2);
writer.writeNext(bFit2);
So, instead of doing this `String aFit2 [] = aFit1.split(" ");
Create an array of the values and then pass that array to writeNext
As an example, you can consider you own example of passing the array of column names, which gets written in a single line:
writer.writeNext(Fitness);

Apache Commons CSV
Here is the same kind of solution, but using the Apache Commons CSV library. This library specifically supports the Microsoft Excel variant of CSV format, so you may find it particularly useful.
CSVFormat.Predefined.EXCEL
Your data, both read and written in this example.
The Commons CSV library can read the first row as header names.
Here is a complete example app in a single .java file. First the app reads from an existing WorstBest.csv data file:
Average,Worst,Best
10,5,15
11,5,16
10,6,16
11,6,15
10,5,16
10,5,16
10,4,16
Each row is represented as a List of three String objects, a List< String >. We add each row to a collection, a list of lists, a List< List< String > >.
Then we write out that imported data to another file. Each written file is name WorstBest_xxx.csv where xxx is the current moment in UTC.
package com.basilbourque.example;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class WorstBest {
public static void main ( String[] args ) {
WorstBest app = new WorstBest();
List < List < String > > data = app.read();
app.write( data );
}
private List < List < String > > read ( ) {
List < List < String > > listOfListsOfStrings = List.of();
try {
// Locate file to import and parse.
Path path = Paths.get( "/Users/basilbourque/WorstBest.csv" );
if ( Files.notExists( path ) ) {
System.out.println( "ERROR - no file found for path: " + path + ". Message # 3cf416de-c33b-4c39-8507-5fbb72e113f2." );
}
// Hold data read from file.
int initialCapacity = ( int ) Files.lines( path ).count();
listOfListsOfStrings = new ArrayList <>( initialCapacity );
// Read CSV file.
BufferedReader reader = Files.newBufferedReader( path );
Iterable < CSVRecord > records = CSVFormat.RFC4180.withFirstRecordAsHeader().parse( reader );
for ( CSVRecord record : records ) {
// Average,Worst,Best
// 10,5,15
// 11,5,16
String average = record.get( "Average" ); // Must use annoying zero-based index counting.
String worst = record.get( "Worst" );
String best = record.get( "Best" );
// Collect
listOfListsOfStrings.add( List.of( average , worst , best ) ); // For real work, you would define a class to hold these values.
}
} catch ( IOException e ) {
e.printStackTrace();
}
return listOfListsOfStrings;
}
private void write ( List < List < String > > listOfListsOfStrings ) {
Objects.requireNonNull( listOfListsOfStrings );
// Determine file in which to write data.
String when = Instant.now().truncatedTo( ChronoUnit.SECONDS ).toString().replace( ":" , "•" ); // Colons are forbidden in names by some file systems such as HFS+.
Path path = Paths.get( "/Users/basilbourque/WorstBest_" + when + ".csv" );
// Loop collection of data (a list of lists of strings).
try ( final CSVPrinter printer = CSVFormat.EXCEL.withHeader( "Average" , "Worst" , "Best" ).print( path , StandardCharsets.UTF_8 ) ; ) {
for ( List < String > list : listOfListsOfStrings ) {
printer.printRecord( list.get( 1 - 1 ) , list.get( 2 - 1 ) , list.get( 3 - 1 ) ); // Annoying zero-based index counting.
}
} catch ( IOException e ) {
e.printStackTrace();
}
}
}

Related

Store the data from text file to a hashmap

I am trying to read a text file and store with a hashmap. The file contains information like this:
1946-01-12;13:00:00;0.3;G
1946-01-12;18:00:00;-2.8;G
1946-01-13;07:00:00;-6.2;G
1946-01-13;13:00:00;-4.7;G
1946-01-13;18:00:00;-4.3;G
1946-01-14;07:00:00;-1.5;G
1946-01-14;13:00:00;-0.2;G
I want to store the dates as keys and then "13:00:00;0.3;G" as value, where 13:00 is time, 0.3 is temperature and G represent a quality code. I wonder if this is even possbile since many rows in the file has the same date? I already wrote a code for storing the data in a list, but now I want to store it in a map instead. My old code looks like this:
/**
* Provides methods to retrieve temperature data from a weather station file.
*/
public class WeatherDataHandler {
private List<Weather> weatherData = new ArrayList<>();
public void loadData(String filePath) throws IOException {
List<String> fileData = Files.readAllLines(Paths.get("filepath"));
for(String str : fileData) {
List<String> parsed = parseData(str);
LocalDate date = LocalDate.parse(parsed.get(0));
LocalTime time = LocalTime.parse(parsed.get(1));
double temperature = Double.parseDouble(parsed.get(2));
String quality = parsed.get(3);
//new Weather object
Weather weather = new Weather(date, time, temperature, quality);
weatherData.add(weather);
}
}
private List<String> parseData(String s) {
return Arrays.asList(s.split(";"));
}
I got stuck when implementing the hashmap. I started with some code below, but I do not know how to loop over a sequence of dates. What is the simplest way to store the data from the file in a map?
public class WeatherDataHandler {
public void loadData(String filePath) throws IOException {
Map<LocalDate, String> map =new HashMap<LocalDate, String>();
BufferedReader br = new BufferedReader(new FileReader("filepath"));
String line="";
int i=0;
while (line != null) {
line = br.readLine();
map.put(i,line);
i++;
}
String date="";
String time="";
String temperature="";
String quality="";
for(int j=0;j<map.size();j++){
if(!(map.get(j)== null)){
String[] getData=map.get(j).toString().split("\\,");
date = getData[0];
time = getData[1];
temperature = getData[2];
quality = getData[3];
}
}
}
Using the stream API you can create a map where the key is the date and the [map] value is a list.
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public class WeatherDataHandler {
public static void main(String[] args) {
Path path = Paths.get("filepath");
try {
Map<String, List<String>> map = Files.lines(path)
.collect(Collectors.groupingBy(line -> line.split(";", 2)[0]));
map.entrySet()
.stream()
.forEach(entry -> System.out.printf("%s = %s%n", entry.getKey(), entry.getValue()));
}
catch (IOException x) {
x.printStackTrace();
}
}
}
Method lines() in class java.nio.file.Files creates a stream where each stream element is a single line of the file being read.
Method split() splits the line into a two element array (because of the second argument which is the number 2).
The first array element, i.e. the date, becomes the [map] key and the rest of the line becomes the [map] value.
Whenever a duplicate key is encountered, the value is appended to the existing value creating a list. Hence the type parameters for the map are String for the [map] key and List<String> for the [map] value.
Running the above code on your sample data yields:
1946-01-14 = [1946-01-14;07:00:00;-1.5;G, 1946-01-14;13:00:00;-0.2;G]
1946-01-12 = [1946-01-12;13:00:00;0.3;G , 1946-01-12;18:00:00;-2.8;G]
1946-01-13 = [1946-01-13;07:00:00;-6.2;G , 1946-01-13;13:00:00;-4.7;G, 1946-01-13;18:00:00;-4.3;G ]

validate ArrayList contents against specific set of data

I want to check and verify that all of the contents in the ArrayList are similar to the value of a String variable. If any of the value is not similar, the index number to be printed with an error message like (value at index 2 didn't match the value of expectedName variable).
After I run the code below, it will print all the three indexes with the error message, it will not print only the index number 1.
Please note that here I'm getting the data from CSV file, putting it into arraylist and then validating it against the expected data in String variable.
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
public class ValidateVideoDuration {
private static final String CSV_FILE_PATH = "C:\\Users\\videologs.csv";
public static void main(String[] args) throws IOException {
String expectedVideo1Duration = "00:00:30";
String expectedVideo2Duration = "00:00:10";
String expectedVideo3Duration = "00:00:16";
String actualVideo1Duration = "";
String actualVideo2Duration = "";
String actualVideo3Duration = "";
ArrayList<String> actualVideo1DurationList = new ArrayList<String>();
ArrayList<String> actualVideo2DurationList = new ArrayList<String>();
ArrayList<String> actualVideo3DurationList = new ArrayList<String>();
try (Reader reader = Files.newBufferedReader(Paths.get(CSV_FILE_PATH));
CSVParser csvParser = new CSVParser(reader,
CSVFormat.DEFAULT.withFirstRecordAsHeader().withIgnoreHeaderCase().withTrim());) {
for (CSVRecord csvRecord : csvParser) {
// Accessing values by Header names
actualVideo1Duration = csvRecord.get("Video 1 Duration");
actualVideo1DurationList.add(actualVideo1Duration);
actualVideo2Duration = csvRecord.get("Video 2 Duration");
actualVideo2DurationList.add(actualVideo2Duration);
actualVideo3Duration = csvRecord.get("Video 3 Duration");
actualVideo3DurationList.add(actualVideo3Duration);
}
}
for (int i = 0; i < actualVideo2DurationList.size(); i++) {
if (actualVideo2DurationList.get(i) != expectedVideo2Duration) {
System.out.println("Duration of Video 1 at index number " + Integer.toString(i)
+ " didn't match the expected duration");
}
}
The data inside my CSV file look like the following:
video 1 duration, video 2 duration, video 3 duration
00:00:30, 00:00:10, 00:00:16
00:00:30, 00:00:15, 00:00:15
00:00:25, 00:00:10, 00:00:16
Don't use == or != for string compare. == checks the referential equality of two Strings and not the equality of the values. Use the .equals() method instead.
Change your if condition to if (!actualVideo2DurationList.get(i).equals(expectedVideo2Duration))

Java csv parser/writer

I'm trying to get a CSV from some data retrieved by Oracle. I have just to write the csv, using the result of the query as column of csv. This is my code:
// get data
final List<myDto> dataCsv = myDao.getdata();
StringWriter writer = new StringWriter();
CSVWriter csvWriter = new CSVWriter(writer,';');
List<String[]> result = toStringArray(dataCsv);
csvWriter.writeAll(result);
csvWriter.close();
return Response.ok(result).header("Content-Disposition", "attachment; filename=" + fileName).build();`
Obviously it can't find toStringArray(). But have I to build it? Do I really need it? How do I have to edit the edit to get it working?
If you just follow the example from the link that you've given, you'll see what they're doing...
private static List<String[]> toStringArray(List<Employee> emps) {
List<String[]> records = new ArrayList<String[]>();
//add header record
records.add(new String[]{"ID","Name","Role","Salary"});
Iterator<Employee> it = emps.iterator();
while(it.hasNext()){
Employee emp = it.next();
records.add(new String[]{emp.getId(),emp.getName(),emp.getRole(),emp.getSalary()});
}
return records;
}
Essentially, you need to build a List of String[]. Each String[] represents a single line of data for the CSV, with each element of the array being a value. So, yes, you need to build a List from your data model and pass it to the CSVWriter's writeAll() method.
The first String[] in the list is the column headers for the CSV. The subsequent String arrays are the data itself.
Apache Commons CSV
The Apache Commons CSV library can help with the chore of reading/writing CSV files. It handles several variants of the CSV format, including the one used by Oracle.
• CSVFormat.ORACLE
Employee.java
Let's make a class for Employee.
package work.basil.example;
import java.util.Objects;
public class Employee {
public Integer id;
public String name, role;
public Integer salary;
public Employee ( Integer id , String name , String role , Integer salary ) {
Objects.requireNonNull( id ); // etc.
this.id = id;
this.name = name;
this.role = role;
this.salary = salary;
}
#Override
public String toString ( ) {
return "Employee{ " +
"id=" + id +
" | name='" + name + '\'' +
" | role='" + role + '\'' +
" | salary=" + salary +
" }";
}
}
Example app
Make another class to mimic retrieving your DTOs. Then we write to a CSV file.
Obviously it can't find toStringArray(). But have I to build it? Do I really need it? How do I have to edit the edit to get it working?
To answer your specific Question, there is no toStringArray method to create field values for the CSV from your DTO object‘s member variables.
Binding
This idea of mapping input or output data with member variables of a Java object is generally known as binding.
There are sophisticated binding libraries for Java to bind your objects with XML and with JSON, JAXB and JSON-B respectively. Objects can be automatically written out to XML or JSON text, as well as “re-hydrated” back to objects when read from such XML or JSON text.
But for CSV with a simpler library such as Apache Commons CSV, we read and write each field of data individually for each object. You pass each DTO object member variable, and Commons CSV will write those values out to the CSV text with any needed encapsulating quote marks, commas, and escaping.
You can see this in the code below, in this line:
printer.printRecord( e.id , e.name , e.role , e.salary );
EmployeeIo.java
Here is the entire EmployeeIo.java file where Io means input-output.
package work.basil.example;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
public class EmployeeIo {
public static void main ( String[] args ) {
EmployeeIo app = new EmployeeIo();
app.doIt();
}
private void doIt ( ) {
// Mimic a collection of DTO objects coming from the database.
List < Employee > employees = new ArrayList <>( 3 );
employees.add( new Employee( 101 , "Alice" , "Boss" , 11_000 ) );
employees.add( new Employee( 102 , "Bob" , "Worker" , 12_000 ) );
employees.add( new Employee( 103 , "Carol" , "Worker" , 13_000 ) );
Path path = Paths.get( "/Users/basilbourque/Employees.csv" );
this.write( employees , path );
}
public void write ( final List < Employee > employees , final Path path ) {
try ( final CSVPrinter printer = CSVFormat.ORACLE.withHeader( "Id" , "Name" , "Role" , "Salary" ).print( path , StandardCharsets.UTF_8 ) ; ) {
for ( Employee e : employees ) {
printer.printRecord( e.id , e.name , e.role , e.salary );
}
} catch ( IOException e ) {
e.printStackTrace();
}
}
}
When run, we produce a file.

How to convert CSV string to integer and add it. store ing txtfile

can you help me to this, I wanted to add 2 integer coming from CSV and store it in txtfile, but the problem is it was string and if i convert it to an integer i've gots lots of error.. Thank you guys..
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.List;
public class CSVReader {
public static void main (String[]arg)throws Exception {
String readFile = "C:/Users/user/Desktop/student.csv";
String writeFile = "C:/Users/user/Desktop/data.txt";
// Read a comma-separated values (CSV) file.
BufferedReader CSVFile = new BufferedReader (new FileReader(readFile));
// Read line.
String dataRow = CSVFile.readLine();
// Create an array of student
List<Student> students = new ArrayList <Student> ();
// The while checks to see if the data is null. If
// it is, we’ve hit the end of the file. If not,
// process the data.
while (dataRow !=null){
String [] dataArray = dataRow.split(",");
System.out.println(dataRow);
Student student = new Student();
student.setStudentName(dataArray[0]);
student.setScore1(dataArray[1]);
student.setScore2(dataArray[2]);
students.add(student);
dataRow = CSVFile.readLine();
}
// Close the file once all data has been read.
CSVFile.close();
StringBuilder sb = new StringBuilder();
FileWriter fw = new FileWriter(writeFile);
for (Student s : students){
sb.append(s.studentName);
System.out.println(s.studentName + " - " + (s.score1 + s.score2));
// Writing to a text file.
fw.write(sb.toString());
}
// Close the file once all data has been written.
fw.close();
}
}
output:
che,cheche,chet
100,100,100
100,100,100
null - 0
null - 0
null - 0
it should br:
che,cheche,chet
100,100,100
100,100,100
che - 200
cheche -200
chet - 200
If the info you have provided is correct, then the main issue you have is the CSV data is in a columnar format, rather than a typical row format. By that, I mean the first row is name, with the next rows the scores. Each "column" of data matches up with the "header" at the same index.
Your example data:
che, cheche, chet -- row[0]
100, 100, 100 -- row[1]
100, 100, 100 -- row[2]
So row[0] is the name, but you are parsing te data as if the 1st item of a row is the name, and the 2nd and 3rd items are scores - which is not the case based on this sample data.
If you wanted scores you'd need to get the proper index for each row - so che would be row[1][0] and row[2][0].
If this is in fact the case, then you'll want to process the first row to get the names, then you'll want to process the remaining rows to get the scores.
You can try
int number = Integer.parseInt("your string here");
Example:
String one = "1";
String two = "2";
System.out.println(Integer.parseInt(one) + Integer.parseInt(two));
You have made few mistakes in the code.
The score variables in the student class should be integer.
To convert a string to Integer you need to use Integer.parseInt
method. Ideally your conversion should be at the stage when you are
setting the score values.
Why are you adding student object to an ArrayList. Can't you
directly write to the text file?

Fast CSV parsing

I have a java server app that download CSV file and parse it. The parsing can take from 5 to 45 minutes, and happens each hour.This method is a bottleneck of the app so it's not premature optimization. The code so far:
client.executeMethod(method);
InputStream in = method.getResponseBodyAsStream(); // this is http stream
String line;
String[] record;
reader = new BufferedReader(new InputStreamReader(in), 65536);
try {
// read the header line
line = reader.readLine();
// some code
while ((line = reader.readLine()) != null) {
// more code
line = line.replaceAll("\"\"", "\"NULL\"");
// Now remove all of the quotes
line = line.replaceAll("\"", "");
if (!line.startsWith("ERROR"){
//bla bla
continue;
}
record = line.split(",");
//more error handling
// build the object and put it in HashMap
}
//exceptions handling, closing connection and reader
Is there any existing library that would help me to speed up things? Can I improve existing code?
Apache Commons CSV
Have you seen Apache Commons CSV?
Caveat On Using split
Bear in mind is that split only returns a view of the data, meaning that the original line object is not eligible for garbage collection whilst there is a reference to any of its views. Perhaps making a defensive copy will help? (Java bug report)
It also is not reliable in grouping escaped CSV columns containing commas
opencsv
Take a look at opencsv.
This blog post, opencsv is an easy CSV parser, has example usage.
The problem of your code is that it's using replaceAll and split which are very costly operation. You should definitely consider using a csv parser/reader that would do a one pass parsing.
There is a benchmark on github
https://github.com/uniVocity/csv-parsers-comparison
that unfortunately is ran under java 6. The number are slightly different under java 7 and 8. I'm trying to get more detail data for different file size but it's work in progress
see https://github.com/arnaudroger/csv-parsers-comparison
Apart from the suggestions made above, I think you can try improving your code by using some threading and concurrency.
Following is the brief analysis and suggested solution
From the code it seems that you are reading the data over the network (most possibly apache-common-httpclient lib).
You need to make sure that bottleneck that you are saying is not in the data transfer over the network.
One way to see is just dump the data in some file (without parsing) and see how much does it take. This will give you an idea how much time is actually spent in parsing (when compared to current observation).
Now have a look at how java.util.concurrent package is used. Some of the link that you can use are (1,2)
What you ca do is the tasks that you are doing in for loop can be executed in a thread.
Using the threadpool and concurrency will greatly improve your performance.
Though the solution involves some effort, but at the end this will surly help you.
opencsv
You should have a look at OpenCSV. I would expect that they have performance optimizations.
A little late here, there is now a few benchmarking projects for CSV parsers. Your selection will depend on the exact use-case (i.e. raw data vs data binding etc).
SimpleFlatMapper
uniVocity
sesseltjonna-csv (disclaimer: I wrote this parser)
Quirk-CSV
The new kid on the block. It uses java annotations and is built on apache-csv which one of the faster libraries out there for csv parsing.
This library is also thread safe as well if you wanted to re-use the CSVProcessor you can and should.
Example:
Pojo
#CSVReadComponent(type = CSVType.NAMED)
#CSVWriteComponent(type = CSVType.ORDER)
public class Pojo {
#CSVWriteBinding(order = 0)
private String name;
#CSVWriteBinding(order = 1)
#CSVReadBinding(header = "age")
private Integer age;
#CSVWriteBinding(order = 2)
#CSVReadBinding(header = "money")
private Double money;
#CSVReadBinding(header = "name")
public void setA(String name) {
this.name = name;
}
#Override
public String toString() {
return "Name: " + name + System.lineSeparator() + "\tAge: " + age + System.lineSeparator() + "\tMoney: "
+ money;
}}
Main
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.*;
public class SimpleMain {
public static void main(String[] args) {
String csv = "name,age,money" + System.lineSeparator() + "Michael Williams,34,39332.15";
CSVProcessor processor = new CSVProcessor(Pojo.class);
List<Pojo> list = new ArrayList<>();
try {
list.addAll(processor.parse(new StringReader(csv)));
list.forEach(System.out::println);
System.out.println();
StringWriter sw = new StringWriter();
processor.write(list, sw);
System.out.println(sw.toString());
} catch (IOException e) {
}
}}
Since this is built on top of apache-csv you can use the powerful tool CSVFormat. Lets say the delimiter for the csv are pipes (|) instead of commas(,) you could for Example:
CSVFormat csvFormat = CSVFormat.DEFAULT.withDelimiter('|');
List<Pojo> list = processor.parse(new StringReader(csv), csvFormat);
Another benefit are inheritance is also consider.
For other examples on handling reading/writing non-primitive data
For speed you do not want to use replaceAll, and you don't want to use regex either. What you basically always want to do in critical cases like that is making a state-machine character by character parser. I've done that having rolled the whole thing into an Iterable function. It also takes in the stream and parses it without saving it out or caching it. So if you can abort early that's likely going to go fine as well. It should also be short enough and well coded enough to make it obvious how it works.
public static Iterable<String[]> parseCSV(final InputStream stream) throws IOException {
return new Iterable<String[]>() {
#Override
public Iterator<String[]> iterator() {
return new Iterator<String[]>() {
static final int UNCALCULATED = 0;
static final int READY = 1;
static final int FINISHED = 2;
int state = UNCALCULATED;
ArrayList<String> value_list = new ArrayList<>();
StringBuilder sb = new StringBuilder();
String[] return_value;
public void end() {
end_part();
return_value = new String[value_list.size()];
value_list.toArray(return_value);
value_list.clear();
}
public void end_part() {
value_list.add(sb.toString());
sb.setLength(0);
}
public void append(int ch) {
sb.append((char) ch);
}
public void calculate() throws IOException {
boolean inquote = false;
while (true) {
int ch = stream.read();
switch (ch) {
default: //regular character.
append(ch);
break;
case -1: //read has reached the end.
if ((sb.length() == 0) && (value_list.isEmpty())) {
state = FINISHED;
} else {
end();
state = READY;
}
return;
case '\r':
case '\n': //end of line.
if (inquote) {
append(ch);
} else {
end();
state = READY;
return;
}
break;
case ',': //comma
if (inquote) {
append(ch);
} else {
end_part();
break;
}
break;
case '"': //quote.
inquote = !inquote;
break;
}
}
}
#Override
public boolean hasNext() {
if (state == UNCALCULATED) {
try {
calculate();
} catch (IOException ex) {
}
}
return state == READY;
}
#Override
public String[] next() {
if (state == UNCALCULATED) {
try {
calculate();
} catch (IOException ex) {
}
}
state = UNCALCULATED;
return return_value;
}
};
}
};
}
You would typically process this quite helpfully like:
for (String[] csv : parseCSV(stream)) {
//<deal with parsed csv data>
}
The beauty of that API there is worth the rather cryptic looking function.
Apache Commons CSV ➙ 12 seconds for million rows
Is there any existing library that would help me to speed up things?
Yes, the Apache Commons CSV project works very well in my experience.
Here is an example app that uses Apache Commons CSV library to write and read rows of 24 columns: An integer sequential number, an Instant, and the rest are random UUID objects.
For 10,000 rows, the writing and the read each take about half a second. The reading includes reconstituting the Integer, Instant, and UUID objects.
My example code lets you toggle on or off the reconstituting of objects. I ran both with a million rows. This creates a file of 850 megs. I am using Java 12 on a MacBook Pro (Retina, 15-inch, Late 2013), 2.3 GHz Intel Core i7, 16 GB 1600 MHz DDR3, Apple built-in SSD.
For a million rows, ten seconds for reading plus two seconds for parsing:
Writing: PT25.994816S
Reading only: PT10.353912S
Reading & parsing: PT12.219364S
Source code is a single .java file. Has a write method, and a read method. Both methods called from a main method.
I opened a BufferedReader by calling Files.newBufferedReader.
package work.basil.example;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.UUID;
public class CsvReadingWritingDemo
{
public static void main ( String[] args )
{
CsvReadingWritingDemo app = new CsvReadingWritingDemo();
app.write();
app.read();
}
private void write ()
{
Instant start = Instant.now();
int limit = 1_000_000; // 10_000 100_000 1_000_000
Path path = Paths.get( "/Users/basilbourque/IdeaProjects/Demo/csv.txt" );
try (
Writer writer = Files.newBufferedWriter( path, StandardCharsets.UTF_8 );
CSVPrinter printer = new CSVPrinter( writer , CSVFormat.RFC4180 );
)
{
printer.printRecord( "id" , "instant" , "uuid_01" , "uuid_02" , "uuid_03" , "uuid_04" , "uuid_05" , "uuid_06" , "uuid_07" , "uuid_08" , "uuid_09" , "uuid_10" , "uuid_11" , "uuid_12" , "uuid_13" , "uuid_14" , "uuid_15" , "uuid_16" , "uuid_17" , "uuid_18" , "uuid_19" , "uuid_20" , "uuid_21" , "uuid_22" );
for ( int i = 1 ; i <= limit ; i++ )
{
printer.printRecord( i , Instant.now() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() , UUID.randomUUID() );
}
} catch ( IOException ex )
{
ex.printStackTrace();
}
Instant stop = Instant.now();
Duration d = Duration.between( start , stop );
System.out.println( "Wrote CSV for limit: " + limit );
System.out.println( "Elapsed: " + d );
}
private void read ()
{
Instant start = Instant.now();
int count = 0;
Path path = Paths.get( "/Users/basilbourque/IdeaProjects/Demo/csv.txt" );
try (
Reader reader = Files.newBufferedReader( path , StandardCharsets.UTF_8) ;
)
{
CSVFormat format = CSVFormat.RFC4180.withFirstRecordAsHeader();
CSVParser parser = CSVParser.parse( reader , format );
for ( CSVRecord csvRecord : parser )
{
if ( true ) // Toggle parsing of the string data into objects. Turn off (`false`) to see strictly the time taken by Apache Commons CSV to read & parse the lines. Turn on (`true`) to get a feel for real-world load.
{
Integer id = Integer.valueOf( csvRecord.get( 0 ) ); // Annoying zero-based index counting.
Instant instant = Instant.parse( csvRecord.get( 1 ) );
for ( int i = 3 - 1 ; i <= 22 - 1 ; i++ ) // Subtract one for annoying zero-based index counting.
{
UUID uuid = UUID.fromString( csvRecord.get( i ) );
}
}
count++;
if ( count % 1_000 == 0 ) // Every so often, report progress.
{
//System.out.println( "# " + count );
}
}
} catch ( IOException e )
{
e.printStackTrace();
}
Instant stop = Instant.now();
Duration d = Duration.between( start , stop );
System.out.println( "Read CSV for count: " + count );
System.out.println( "Elapsed: " + d );
}
}

Categories

Resources