I have dataset(input.csv) containing 35 columns(0-34 positions). If I run my MRv2 program on this then I am getting "ArrayIndexOutOfBoundException".
But, if I run the program on snapshot of dataset containing the same columns, then it runs successfully.
Error
15/07/20 11:05:55 INFO mapreduce.Job: Task Id : attempt_1437379028043_0018_m_000000_2, Status : FAILED
Error: java.lang.ArrayIndexOutOfBoundsException: 34
at lotus.staging.StageMapper.map(StageMapper.java:88)
at lotus.staging.StageMapper.map(StageMapper.java:1)
at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:145)
at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:784)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:163)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
StageMapper
package lotus.staging;
import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class StageMapper extends Mapper<LongWritable, Text, Text, Text> {
#Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String[] record = value.toString().split(",");
// Key
String stg_table = null;
String report_code = record[0].trim();
String product_type_description = null;
String principal_amount = record[1];
String funded = record[2].trim();
String facility_id = record[3];
String loan_id = record[4];
// Start Date
String start_date = record[5];
// Maturity Date
String end_date = record[6];
DateFormat df = new SimpleDateFormat("MM/dd/yyyy");
Date startDate;
Date endDate;
long diff;
long diffDays = 0l;
try {
startDate = df.parse(start_date);
endDate = df.parse(end_date);
df.format(startDate);
df.format(endDate);
diff = endDate.getTime() - startDate.getTime();
diffDays = diff / (24 * 60 * 60 * 1000);
} catch (ParseException e) {
e.printStackTrace();
}
// Date Diff
String date_diff = String.valueOf(diffDays);
String next_reset_date = record[7];
String interest_rate = record[8];
String base_interest_rate = record[9];
String counterparty_industry_id = record[10];
String industry_name = record[11];
String counterparty_id = record[12];
String counterparty_name = record[13];
// Bank Number
String vehicle_code = record[14];
String vehicle_description = record[15];
// Branch Number
String cost_center_code = record[16];
String branch_borrower_name = record[17];
String igl_code = record[20];
// Participation Bal Begin Month
String participated_amt = record[21];
String sys_id = record[23];
// Loan To Value
String ltv = record[26];
String accrual_status = record[27];
String country_code = record[30];
String fiscal_year = record[31];
String accounting_period = record[32];
String accounting_day = record[33];
String control_category = record[34];
// CONTROL_CATEGORY_DESC, Secred_BY_Re
if (report_code.equalsIgnoreCase("1")) {
product_type_description = "Commercial_Loan";
stg_table = "stg_lon";
} else if (report_code.equalsIgnoreCase("2")) {
product_type_description = "Mortgage_Loan";
stg_table = "stg_mgt";
} else if (report_code.equalsIgnoreCase("3")) {
product_type_description = "Installment_Loan";
stg_table = "stg_lon";
} else if (report_code.equalsIgnoreCase("4")) {
product_type_description = "Revolving Credit";
stg_table = "stg_lon";
}
// Value
String data = report_code + "," + product_type_description + ","
+ principal_amount + "," + funded + "," + facility_id + ","
+ loan_id + "," + start_date + "," + end_date + "," + date_diff
+ "," + next_reset_date + "," + interest_rate + ","
+ base_interest_rate + "," + counterparty_industry_id + ","
+ industry_name + "," + counterparty_id + ","
+ counterparty_name + "," + vehicle_code + ","
+ vehicle_description + "," + cost_center_code + ","
+ branch_borrower_name + "," + igl_code + ","
+ participated_amt + "," + sys_id + "," + ltv + ","
+ accrual_status + "," + country_code + "," + fiscal_year + ","
+ accounting_period + "," + accounting_day + ","
+ control_category;
context.write(new Text(stg_table), new Text(data));
} // map() ends
} // Mapper ends
StageReducer
package lotus.staging;
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
public class StageReducer extends Reducer<Text, Text, Text, Text> {
private MultipleOutputs mos;
#Override
protected void setup(Context context) throws IOException,
InterruptedException {
mos = new MultipleOutputs(context);
}
#Override
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
for (Text value : values) {
mos.write(key, value, key.toString());
}
}
#Override
protected void cleanup(Context context) throws IOException,
InterruptedException {
mos.close();
}
}
StageDriver
package lotus.staging;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
public class StageDriver {
// Main
public static void main(String[] args) throws IOException,
ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "StageDriver");
// conf.set("mapreduce.textoutputformat.separator", ",");
// conf.set("mapreduce.output.textoutputformat.separator", ",");
//conf.set("mapreduce.output.key.field.separator", ",");
job.setJarByClass(StageDriver.class);
LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);
// Mapper and Mapper-Output Key
job.setMapperClass(StageMapper.class);
job.setMapOutputKeyClass(Text.class);
conf.set("mapred.max.split.size", "1020");
// Reducer and Output Key and Value
job.setReducerClass(StageReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
// Input parameters to execute
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
// deleting the output path automatically from hdfs so that we don't
// have delete it explicitly
// outputPath.getFileSystem(conf).delete(outputPath);
// exiting the job only if the flag value becomes false
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
Below are the datasets
Snapshot-Dataset Complete-Dataset
Please assist
One of the rows in the input.csv is incomplete or has a formatting error (improperly escaped). Try to figure out which row it is. You can catch an exception where this error occurs and print out the row number and fix your data.
try {
CODE WHERE THE OUTOFBOUNDS HAPPENS
}
catch (Exception e) {
LOG.warn(String.format("Invalid data in row: %d", row));
System.out.println(String.format("Invalid data in row: %d", row));
}
So in your case that means:
#Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String[] record = value.toString().split(",");
// Key
String stg_table = null;
try{
String report_code = record[0].trim();
String product_type_description = null;
String principal_amount = record[1];
String funded = record[2].trim();
String facility_id = record[3];
String loan_id = record[4];
// Start Date
String start_date = record[5];
// Maturity Date
String end_date = record[6];
DateFormat df = new SimpleDateFormat("MM/dd/yyyy");
Date startDate;
Date endDate;
long diff;
long diffDays = 0l;
try {
startDate = df.parse(start_date);
endDate = df.parse(end_date);
df.format(startDate);
df.format(endDate);
diff = endDate.getTime() - startDate.getTime();
diffDays = diff / (24 * 60 * 60 * 1000);
} catch (ParseException e) {
e.printStackTrace();
}
// Date Diff
String date_diff = String.valueOf(diffDays);
String next_reset_date = record[7];
String interest_rate = record[8];
String base_interest_rate = record[9];
String counterparty_industry_id = record[10];
String industry_name = record[11];
String counterparty_id = record[12];
String counterparty_name = record[13];
// Bank Number
String vehicle_code = record[14];
String vehicle_description = record[15];
// Branch Number
String cost_center_code = record[16];
String branch_borrower_name = record[17];
String igl_code = record[20];
// Participation Bal Begin Month
String participated_amt = record[21];
String sys_id = record[23];
// Loan To Value
String ltv = record[26];
String accrual_status = record[27];
String country_code = record[30];
String fiscal_year = record[31];
String accounting_period = record[32];
String accounting_day = record[33];
String control_category = record[34];
}
catch (Exception e) {
if {record.size() > 0} {
// LOG.warn(String.format("Invalid data in row: %s", record[0].trim()));
System.out.println(String.format("Invalid data in record id: %s", record[0].trim()));}
else{
System.out.println("Empty Record Found");
}
return void;
}
...
I'm using the record id, because you don't have a row number, but then you can search your that for that record id. And presumably there is at least a first entry in your record. Otherwise you can also check to see if the record is empty.
Related
Hey I am trying to figure out how to get my save / load methods to put the .txt file they generate into a zip and how can I set the location of the file. this is my code so far :
save method - pulls data from the current array list and writes them to the file
public void save(String path) {
ArrayList<String> lines = new ArrayList<String>();
for (int i = 0; i < this.listaKorisnika.size(); i++) {
User korisnik = this.listaKorisnika.get(i);
String ime = korisnik.getIme();
String prezime = korisnik.getPrezime();
LocalDate datum = korisnik.getDatumRodjenja();
DateTimeFormatter dtf = DateTimeFormatter.ofPattern("dd.MM.yyyy.");
String formiraniDatum = dtf.format(datum);
String jmbg = korisnik.getJmbg();
String zanimanjePoSk = korisnik.getZanimanjePoSkoli();
String gdeRadi = korisnik.getGdeRadi();
String bolesti = korisnik.getBolesti();
String alergije = korisnik.getAlergije();
String line = ime + ";" + prezime + ";" + formiraniDatum + ";" + jmbg + ";" + zanimanjePoSk + ";" + gdeRadi
+ ";" + bolesti + ";" + alergije;
lines.add(line);
}
try {
Files.write(Paths.get(path), lines, Charset.defaultCharset(), StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE);
} catch (java.io.IOException e) {
System.out.println("Datoteka " + path + " nije pronađena.");
}
}
// this load method pulls data from the generated .txt file and puts them in memory.
public void load(String path) {
this.listaKorisnika = new ArrayList<User>();
List<String> lines;
try {
lines = Files.readAllLines(Paths.get(path), Charset.defaultCharset());
for (String line : lines) {
String[] attributes = line.split(";");
/*
* String line = ime +";"+ prezime +";"+ formiraniDatum +";"+ jmbg +";"+
* zanimanjePoSk +";"+ gdeRadi +";"+ bolesti +";"+ alergije;
*/
String ime = attributes[0];
String prezime = attributes[1];
String datum = attributes[2];
DateTimeFormatter dtf = DateTimeFormatter.ofPattern("dd.MM.yyyy.");
LocalDate datumZaCuvanje = LocalDate.parse(datum, dtf);
String jmbg = attributes[3];
String zanimanjePoSkoli = attributes[4];
String gdeRadi = attributes[5];
String bolesti = attributes[6];
String Alergije = attributes[7];
User user = new User(ime, prezime, datumZaCuvanje, jmbg, zanimanjePoSkoli, gdeRadi, bolesti, Alergije);
this.listaKorisnika.add(user);
}
} catch (java.io.IOException e) {
System.out.println("Datoteka " + path + " nije pronađena.");
} catch (Exception e) {
System.out.println("Desila se greška pri parsiranju datuma.");
}
}
Helo, I've got this code, which query the instances in calendarcontract:
Uri uri = Uri.parse(CalendarContract.Instances.CONTENT_URI + "/" +
Long.toString(from) + "/" +
Long.toString(to));
String[] mProjection =
{
CalendarContract.Instances._ID,
CalendarContract.Instances.EVENT_ID,
CalendarContract.Instances.CALENDAR_ID,
CalendarContract.Instances.TITLE,
CalendarContract.Instances.EVENT_LOCATION,
CalendarContract.Instances.DESCRIPTION,
CalendarContract.Instances.EVENT_COLOR,
CalendarContract.Instances.DTSTART,
CalendarContract.Instances.DTEND,
CalendarContract.Instances.EVENT_TIMEZONE,
CalendarContract.Instances.EVENT_END_TIMEZONE,
CalendarContract.Instances.DURATION,
CalendarContract.Instances.ALL_DAY,
CalendarContract.Instances.RRULE,
CalendarContract.Instances.RDATE,
CalendarContract.Instances.EXDATE
};
ContentResolver cr2 = this.c.getContentResolver();
String selection2 = "( " + selection_allcalendars + " (" + CalendarContract.Instances.RRULE + " IS NOT NULL) AND (deleted != 1) AND (" + CalendarContract.Instances.ALL_DAY + " = 0))";
Cursor cur2 = cr2.query(uri, mProjection, selection2, null, CalendarContract.Instances.DTSTART + " ASC");
How can I remove double instances from query? I want to use a "GROUP BY" clause, but I don't know how to do that.
Please help me :-)
Thanks!
Edit:
I want to group by EVENT_ID, because I get multiple entries. This ist the whole code I use for recurring events:
ContentResolver cr2 = this.c.getContentResolver();
String selection2 = "( " + selection_allcalendars + " (" + CalendarContract.Instances.RRULE + " IS NOT NULL) AND (deleted != 1) AND (" + CalendarContract.Instances.ALL_DAY + " = 0))";
Cursor cur2 = cr2.query(uri, mProjection, selection2, null, CalendarContract.Instances.DTSTART + " ASC");
ArrayList<String> checkexists = new ArrayList<>();
while (cur2.moveToNext()) {
String eventid = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.EVENT_ID));
if(checkexists.contains(eventid)) {
}
else {
checkexists.add(eventid);
String rrule = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.RRULE));
TimeZone tz = TimeZone.getTimeZone("Europe/Berlin");
Long dtstart;
try {
dtstart = Long.parseLong(cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.DTSTART)));
dtstart = tz.getOffset(dtstart) + dtstart;
} catch (NumberFormatException e) {
dtstart = 0l;
}
Long dtend;
try {
dtend = dtstart + RFC2445ToMilliseconds(cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.DURATION)));
/* dtend = Long.parseLong(cur.getString(cur.getColumnIndex(CalendarContract.Events.DTEND)));
dtend = tz.getOffset(dtend) + dtend; */
} catch (NumberFormatException e) {
dtend = 0l;
}
Calendar cal4 = Calendar.getInstance();
cal4.setTimeInMillis(dtstart);
LocalDate localdate = new LocalDate(cal4.get(Calendar.YEAR), cal4.get(Calendar.MONTH) + 1, cal4.get(Calendar.DAY_OF_MONTH));
long calendar_id = Long.valueOf(cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.CALENDAR_ID)));
String id = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances._ID));
String title = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.TITLE));
String eventlocation = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.EVENT_LOCATION));
String description = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.DESCRIPTION));
String eventcolor = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.EVENT_COLOR));
String eventtimezone = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.EVENT_TIMEZONE));
String eventtimezone_end = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.EVENT_END_TIMEZONE));
String duration = cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.DURATION));
int allday = Integer.valueOf(cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.ALL_DAY)));
rrule = "RRULE:" + rrule;
long lastrecurrencetime;
if(rrule.contains("UNTIL")) {
lastrecurrencetime = RruleHelper.getUntil(rrule).getTimeInMillis();
}
else {
lastrecurrencetime = -1;
}
//System.out.println("EEE: " + title + " " + rrule);
try {
ArrayList<ReadEvent> recurrenceevents = new ArrayList<>();
for (LocalDate date : LocalDateIteratorFactory.createLocalDateIterable(rrule, localdate, true)) {
// System.out.println("GGG:" + title + " " + i);
Calendar newcal_from = Calendar.getInstance();
newcal_from.setTimeInMillis(dtstart);
newcal_from.set(Calendar.DAY_OF_MONTH, date.getDayOfMonth());
newcal_from.set(Calendar.MONTH, date.getMonthOfYear() - 1);
newcal_from.set(Calendar.YEAR, date.getYear());
long dtstart_new = newcal_from.getTimeInMillis();
long dtend_new = dtstart_new + RFC2445ToMilliseconds(cur2.getString(cur2.getColumnIndex(CalendarContract.Instances.DURATION)));
if (dtstart_new >= from && dtstart_new <= to) {
ReadEvent newreadevent = new ReadEvent(calendar_id, id, eventid, title, eventlocation, description, eventcolor, dtstart_new, dtend_new, eventtimezone, eventtimezone_end, duration, allday, rrule);
newreadevent.setFirstReccurenceTime(dtstart);
newreadevent.setLastReccurenceTime(lastrecurrencetime);
if(!checkIfAlreadyExits(readevent, newreadevent)) {
//newreadevent.setReccurenceCount(i);
readevent.add(newreadevent);
}
}
if (dtstart_new > to) {
break;
}
}
} catch (Exception e) {
}
}
}
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Formatter;
public class JtoCModified {
private static final String COMMA_DELIMITER = ",";
private static final String DB_DRIVER = "com.mysql.jdbc.Driver";
private static final String DB_CONNECTION="jdbc:mysql://localhost:3306/test2";
private static final String DB_USER = "root";
private static final String DB_PASSWORD = "root";
private static int RECORD_COUNT =24;
static final String DATEFORMAT = "yyyy-MM-dd HH:mm:ss";
static final String DATE_FORMAT = "yyyy-MM-dd";
static final String TIME_FORMAT = "HH:mm:ss";
private static final int ADD_MINUTES = 2;
static final String FromDate = "2016-01-01 00:00:00";
#SuppressWarnings("unused")
public static void main(String[] args) throws Exception {
List<String> records = new ArrayList<String>();
StringBuffer record = new StringBuffer();
DateFormat d_f = new SimpleDateFormat(DATE_FORMAT);
DateFormat tf = new SimpleDateFormat(TIME_FORMAT);
Calendar cal = Calendar.getInstance();
cal.setTime(d_f.parse(FromDate));
record.append("\t");
for (int i = 1; i <= RECORD_COUNT; i++) {
if (i % 100000 == 0) {
records = new ArrayList<String>(RECORD_COUNT);
}
for (int j = 0; j < 2730; j++) {
int OPV = 230 + j % 15; // 230 - 244 by 1
String String = Integer.toString(OPV);
String str = Integer.toString(OPV);
record.append("OPV");
double IKW = 1.3 + j % 17 * 0.1; // 1.3 - 2.9 by 0.1
String aString = Double.toString(IKW);
String IKW2 = String.valueOf(IKW);
record.append("IKW");
double OKW = 0.01 + j % 49 * 0.01; // 0.01 - 0.49 by 0.01
String bString = Double.toString(IKW);
String OKW2 = String.valueOf(OKW);
record.append("OKW");
double OPI = 0.05 + j % 105 * 0.01; // 0.05 - 1.09 by 0.01
String cString = Double.toString(OPI);
String OPI2 = String.valueOf(OPI);
record.append("OPI");
double IPI = 0.00 + j % 8 * 0.01;
String dString = Double.toString(IPI);
String IPI2 = String.valueOf(IPI);
record.append("IPI");
int NA1 = 000;
String eString = Integer.toString(NA1);
String NA12 = Integer.toString(NA1);
record.append("NA1");
int BVV = 104 + j % 13;
String fString = Integer.toString(BVV);
String BVV2 = Integer.toString(BVV);
record.append("BVV");
double BVI = 1.3 + j % 15 * 0.8;
String gString = Double.toString(BVI);
String BVI2 = String.valueOf(BVI);
record.append("BVI");
int NA2 = 000;
String hString = Integer.toString(NA2);
String NA22 = Integer.toString(NA2);
record.append("NA2");
int NA3 = 000;
String iString = Integer.toString(NA3);
String NA32 = Integer.toString(NA3);
record.append("NA3");
int IPV = 241 + j % 1;
String jString = Integer.toString(IPV);
String IPV2 = Integer.toString(IPV);
record.append("IPV");
int _IF = 50;
String kString = Integer.toString(_IF);
String _IF2 = Integer.toString(_IF);
record.append("_IF");
int _OF = 50;
String lString = Integer.toString(_OF);
String _OF2 = Integer.toString(_OF);
record.append("_OF");
int NA4 = 000;
String mString = Integer.toString(NA4);
String NA42 = Integer.toString(NA4);
record.append("NA4");
int SERIAL = 12345;
String oString = Integer.toString(SERIAL);
String SERIAL2 = Integer.toString(SERIAL);
record.append("SERIAL");
int NA5 = 000;
String nString = Integer.toString(NA4);
String NA52 = Integer.toString(NA5);
record.append("NA5");
int NA6 = 000;
String qString = Integer.toString(NA6);
String NA62 = Integer.toString(NA6);
record.append("NA6");
int STATUS = 000 + j % 001;
String rString = Integer.toString(STATUS);
String STATUS2 = Integer.toString(STATUS);
record.append("STATUS");
int PVV=000;
String sString = Integer.toString(PVV);
String PVV2 = Integer.toString(NA2);
record.append("PVV");
double PVI=00.0;
String tString = Double.toString(PVI);
String PVI2 = String.valueOf(OPI);
record.append("PVI");
double PKW=00.0;
String uString = Double.toString(PKW);
String PKW2 = String.valueOf(PKW);
record.append("PKW");
double PKWH=00.0;
String vString = Double.toString(PKWH);
String PKWH2 = String.valueOf(PKWH);
record.append("PKWH");
record.append((d_f.format(cal.getTime()))+", "+tf.format(cal.getTime()));
int Device_id=101;
addToBuffer(record, Device_id);
record.append("\n");
cal.add(Calendar.MINUTE, ADD_MINUTES);
records.add(record.toString());
record.delete(0, record.length());
addToBuffer(record,OPV);
addToBuffer(record,IKW);
addToBuffer(record,OKW);
addToBuffer(record,OPI);
addToBuffer(record,IPI);
addToBuffer(record,NA1);
addToBuffer(record,BVV);
addToBuffer(record,BVI);
addToBuffer(record,NA2);
addToBuffer(record,NA3);
addToBuffer(record,IPV);
addToBuffer(record,_IF);
addToBuffer(record,_OF);
addToBuffer(record,NA4);
addToBuffer(record,SERIAL);
addToBuffer(record,NA5);
addToBuffer(record,NA6);
addToBuffer(record,NA6);
addToBuffer(record,PVV);
addToBuffer(record,PVI);
addToBuffer(record,PKW);
addToBuffer(record,PKWH);
record.delete(0, record.length());
record.append(OKW);
record.append(OPI);
record.append(IPI);
record.append(NA1);
record.append(BVV);
record.append(BVI);
record.append(NA2);
record.append(NA3);
record.append(IPV);
record.append(_IF);
record.append(_OF);
record.append(NA4);
record.append(SERIAL);
record.append(NA5);
record.append(NA6);
record.append(STATUS);
record.append(PVV);
record.append(PVI);
record.append(PKW);
record.append(PKWH);
record.append(STATUS);
record.append((d_f.format(cal.getTime()))+", "+tf.format(cal.getTime()));
record.append("\t\t");
record.append(COMMA_DELIMITER);
// int Device_id=101;
addToBuffer(record, Device_id);
record.append("\n");
cal.add(Calendar.MINUTE, ADD_MINUTES);
records.add(record.toString());
String insertTableSQL = "INSERT INTO DBUSER"
+ "(OPV, IKW, OKW ,OPI, IPI,NA1, BVV, BVI,NA2, NA3,IPV, _IF, _OF,NA4 , SERIAL ,NA5, NA6, STATUS ,PVI ,PKW , PKWH) "
+ "VALUES" + "("+ OPV + IKW + OKW + OPI + IPI + NA1 + BVV + BVI + NA2 + NA3 + IPV + _IF + _OF + NA4 + SERIAL + NA5 + NA6 + STATUS+ PVI + PKW + PKWH +")";
// record.append("\t\t");
record.append(COMMA_DELIMITER);
try {
insertRecordIntoDbUserTable();
Connection dbConnection = null;
Statement statement = null;
dbConnection = getDBConnection();
statement = dbConnection.createStatement();
System.out.println(insertTableSQL);
// execute insert SQL stetement
statement.executeUpdate(insertTableSQL);
System.out.println(insertTableSQL);
System.out.println("Record is inserted into DbUser table!");
} catch (SQLException e) {
System.out.println(e.getMessage());
} finally {
}
}
}
}
private static void addToBuffer(StringBuffer buffer, Object data) {
buffer.append(data);
buffer.append(", ");
}
private static void insertRecordIntoDbUserTable()
{
}
private static Connection getDBConnection() {
Connection dbConnection = null;
try {
Class.forName(DB_DRIVER);
} catch (ClassNotFoundException e) {
System.out.println(e.getMessage());
}
try {
dbConnection = DriverManager.getConnection(
DB_CONNECTION, DB_USER,DB_PASSWORD);
return dbConnection;
} catch (SQLException e) {
System.out.println(e.getMessage());
}
return dbConnection;
}
}
I want to export data from java to mysql db but i am not getting any data, initially i want to print the data in console after that i want to add into a database table .I have done jdbc connectivity.But still its not fetching any data please suggest me something.I am getting data in console but not in format what should i do to make the console data in format and to export data into db.
There is a problem with the SQL statement you create here:
String insertTableSQL = "INSERT INTO DBUSER"
+ "(OPV, IKW, OKW ,OPI, IPI,NA1, BVV, BVI,NA2, NA3,IPV, _IF, _OF,NA4 , SERIAL ,NA5, NA6, STATUS ,PVI ,PKW , PKWH) "
+ "VALUES" + "("+ OPV + IKW + OKW + OPI + IPI + NA1 + BVV + BVI + NA2 + NA3 + IPV + _IF + _OF + NA4 + SERIAL + NA5 + NA6 + STATUS+ PVI + PKW + PKWH +")";
Print out the insertTableSQL string to see what it actually contains.
Also, I think there should be an error message printed by this:
System.out.println(e.getMessage());
Show it to us.
UPDATE
INSERT INTO DBUSER(OPV, IKW, OKW ,OPI, IPI,NA1, BVV, BVI,NA2,
NA3,IPV, _IF, _OF,NA4 , SERIAL ,NA5, NA6, STATUS ,PVI ,PKW , PKWH)
VALUES(2341.70000000000000020.050.090.0401084.50024150500123450000.00.00.0)
As I suspected, the problem is that your string concatenation has resulted in an SQL statement with one long meaningless "value". It is invalid SQL.
The solutions:
Put commas between the values in your concatenation.
Use a PreparedStatement instead of a Statement with ? placeholders, and then use setXxxx to set the query arguments.
I am using POIFSFileSystem and HSSFWorkbook to read my excel and upload it to my db.i have declared all the var in my pgm as string and also i have formatted my excel cells to text..Even then I am getting "java.lang.IllegalStateException: Cannot get a text value from a numeric formula cell". The cell type is showing as numeric but I formatted the cells to text.Any solution guys??
here is my code:
/*
* FarmerQueryMigration.java
* Copyright (c) 2014-2015, SourceTrace Systems, All Rights Reserved.
*
* This software is the confidential and proprietary information of SourceTrace Systems
* ("Confidential Information"). You shall not disclose such Confidential Information and shall use
* it only in accordance with the terms of the license agreement you entered into with
* SourceTrace Systems.
*/
package com.ese.view.profile;
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import com.ese.util.DateUtil;
import com.ese.util.ObjectUtil;
import com.ese.util.StringUtil;
public class FarmerQueryMigration {
private static final SimpleDateFormat fileNameDateFormat = new SimpleDateFormat("dd-MM-yyyy");
/**
* The main method.
* #param args the arguments
* #throws IOException Signals that an I/O exception has occurred.
*/
#SuppressWarnings({ "deprecation", "deprecation" })
public static void main(String args[]) throws IOException {
String fileName = "E:\\viji\\proj docs\\aditi upload files\\Bajolga Provisional Approved Farmers List.XLS";
FileInputStream myInput = new FileInputStream(fileName);
FileOutputStream myOutput = new FileOutputStream(
"E:\\viji\\proj docs\\aditi upload files\\FarmerInsertQuery_"
+ fileNameDateFormat.format(new Date()) + ".sql");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
POIFSFileSystem myFileSystem = new POIFSFileSystem(myInput);
HSSFWorkbook myWorkBook = new HSSFWorkbook(myFileSystem);
HSSFSheet mySheet = myWorkBook.getSheetAt(1);
StringBuilder sb = new StringBuilder();
String initialQuery = "INSERT INTO FARMER VALUES(null,\"";
String accountInsertQuery = "INSERT INTO ESE_ACCOUNT VALUES (NULL,\"22";
String contractQuery = "INSERT INTO CONTRACT VALUES (NULL,\"";
String cardInsertQuery = "INSERT INTO ESE_CARD VALUES (NULL,\"12";
String contarctPricePatternQuery = "INSERT INTO CONTRACT_PRICEPATTERN_MAP VALUES ('";
String nullString = null;
int farmerSeq = 100;
int accountSeq = 103;
int contractSeq = 103;
int contractPriceSeq = 103;
int cardSeq = 103;
int i = 3;
int rowCount = mySheet.getLastRowNum();
/* String test = null;
String test1 = null;
while (i <=0) {
HSSFRow myRow = mySheet.getRow(i);
// if(i==rowCount){
System.out.println("----------ROW_NO:" + i + "-----------");
String firstName = !StringUtil.isEmpty(myRow.getCell(0).getStringCellValue()) ? String
.valueOf((int) myRow.getCell(0).getNumericCellValue()) : nullString;
test = !ObjectUtil.isEmpty(myRow.getCell(1)) ? String
.valueOf((int) myRow.getCell(1).getNumericCellValue()) : nullString;
test1 = !ObjectUtil.isEmpty(myRow.getCell(2)) ? String
.valueOf((int) myRow.getCell(2).getNumericCellValue()) : nullString;
//}
System.out.println("---------" + String
.valueOf(myRow.getCell(0).getStringCellValue()) + "-----------");
if(!ObjectUtil.isEmpty(myRow.getCell(1))&&!ObjectUtil.isEmpty(myRow.getCell(2))){
System.out.println(myRow.getCell(1).getStringCellValue());
System.out.println(myRow.getCell(2).getStringCellValue());
}
else{
System.out.println("----------Else" + i + "-----------");
}
//System.out.println("----------ROW_NO:" + i + "-----------");
i++;
}
*/
String customerProjectQuery="INSERT INTO `customer_project` VALUES ('22', '00320002', 'Chitradurga', '1', '1', '1', '1', 'Chitradurga', '0', '0', null, '1', '1', '32', '5', '0', '20150210184539');";
sb.append(customerProjectQuery+"\n");
try {
while (i <= rowCount) {
HSSFRow myRow = mySheet.getRow(i);
String farmerId = getExact(String.valueOf(farmerSeq++), 6);
String farmerCode = null;//myRow.getCell(0).getStringCellValue().trim();
String firstName = myRow.getCell(1).getStringCellValue().trim();
String lastName= null;
String fatherName =myRow.getCell(2).getStringCellValue().trim();
String gender = String.valueOf(myRow.getCell(3).getStringCellValue().trim()).toUpperCase();
Date dob =null;
// Date doj = !StringUtil.isEmpty(myRow.getCell(4).getStringCellValue().trim()) ? myRow.getCell(4) .getDateCellValue() : null;
String dojString = "2015-05-21";
/* if (!StringUtil.isEmpty(myRow.getCell(5).getStringCellValue().trim())) {
// dojString = DateUtil.convertDateToString(doj, DateUtil.TXN_DATE_TIME);
}*/
/* Date dob = !StringUtil.isEmpty(myRow.getCell(3)) ? myRow.getCell(3)
.getDateCellValue() : null;
String dobString = null;
if (!StringUtil.isEmpty(dob)) {
dobString = DateUtil.convertDateToString(dob, DateUtil.TXN_DATE_TIME);
}*/
String noOfFamilyMembers = null;
if(!ObjectUtil.isEmpty(myRow.getCell(29))){
noOfFamilyMembers= myRow.getCell(29).getStringCellValue();
}
String imgInfo=null;
String address = !StringUtil.isEmpty(myRow.getCell(7)) ? myRow.getCell(7)
.getStringCellValue().trim() : null;
String cityName = myRow.getCell(11).getStringCellValue().trim();
String villageName = myRow.getCell(13).getStringCellValue().trim();
String gramPanchayatName = myRow.getCell(14).getStringCellValue().trim();
String pincode = null;
if(!ObjectUtil.isEmpty(myRow.getCell(27))){
pincode= myRow.getCell(27).getStringCellValue();
}
String postOffice = null;//myRow.getCell(18).getStringCellValue().trim();
String phoneNumber = null;
String mobileNumber= "";
if(!ObjectUtil.isEmpty(myRow.getCell(28))){
mobileNumber= myRow.getCell(28).getStringCellValue();
}
String eMail = myRow.getCell(17).getStringCellValue();
long revisionNumber=DateUtil.getRevisionNumber();
String latitude=null;
String longtitude=null;
String photoCaptTime=null;
String status="1";
String samithiName=myRow.getCell(26).getStringCellValue().trim();
String projectSubGroupName=myRow.getCell(26).getStringCellValue().trim();
String cetificationStandard="5";
String cetificationLevel="0";
String cetificationType="2";
String martialStatus = "0";//myRow.getCell(19).getStringCellValue().trim();
String education = myRow.getCell(20).getStringCellValue().trim();
if(education.equalsIgnoreCase("No")){
education="0";
}else if(education.equalsIgnoreCase("2th") || education.equalsIgnoreCase("3rd")|| education.equalsIgnoreCase("3th")||education.equalsIgnoreCase("4th")||education.equalsIgnoreCase("5th")){
education="1";
}else if(education.equalsIgnoreCase("6th") || education.equalsIgnoreCase("7th")||education.equalsIgnoreCase("8th")||education.equalsIgnoreCase("5th")){
education="2";
}
else if(education.equalsIgnoreCase("SSLC") || education.equalsIgnoreCase("PUC")||education.equalsIgnoreCase("P U C")){
education="3";
}
String childCount =myRow.getCell(30).getStringCellValue();
if(childCount.equalsIgnoreCase("-")){
childCount="0";
}
String childCount1 = "0";
String childCount2 = "0";
String inspectionType="0";
String icsStatus="3";
String CustomerProjectName= myRow.getCell(15).getStringCellValue().trim();
String farmerEconomyId = null;
String statusCode="0";
String statusMsg="SUCCCESS";
sb.append(initialQuery);
sb.append(farmerId);
sb.append("\",");
sb.append(farmerCode+",");
sb.append("'"+firstName+"',");
sb.append(lastName+",");
sb.append("\""+fatherName+"\",");
sb.append("\""+gender+"\",");
sb.append("NULL,"+"'");
sb.append(dojString+"','");
sb.append(noOfFamilyMembers+"',");
sb.append("NULL,"+"\"");
sb.append(address+"\",");
sb.append("(SELECT ID FROM CITY WHERE NAME=\"" + cityName + "\"),");
sb.append("(SELECT V.ID FROM VILLAGE V INNER JOIN CITY C ON V.CITY_ID=C.ID WHERE V.NAME=\""
+ villageName + "\" AND C.NAME=\"" + cityName + "\"),");
sb.append("(SELECT GP.ID FROM GRAM_PANCHAYAT GP INNER JOIN VILLAGE V ON GP.VILLAGE_ID=V.ID WHERE GP.NAME=\""
+ gramPanchayatName + "\" AND V.NAME=\"" + villageName + "\"),");
sb.append("\""+pincode+"\",NULL,NULL,'");
sb.append(mobileNumber+"',\"");
sb.append(eMail+"\",'"+revisionNumber+"',NULL,NULL,NULL,'1'");
String samithiIdQuery ="(SELECT WVM.WAREHOUSE_ID FROM WAREHOUSE_VILLAGE_MAP WVM INNER JOIN WAREHOUSE W ON W.ID=WVM.WAREHOUSE_ID" +
" WHERE WVM.VILLAGE_ID=(SELECT V.ID FROM VILLAGE V INNER JOIN CITY C ON V.CITY_ID=C.ID WHERE V.NAME=\""
+ villageName + "\" AND C.NAME=\"" + cityName + "\") AND W.REF_WAREHOUSE_ID IS NOT NULL)";
sb.append(","+samithiIdQuery+",");
sb.append(""+samithiIdQuery+",");
sb.append("'"+cetificationStandard+"',");
sb.append("'"+cetificationLevel+"',");
sb.append("'"+cetificationType+"',");
sb.append("'"+martialStatus+"',");
sb.append("'"+education+"',");
sb.append("'"+childCount+"',");
sb.append("'"+childCount1+"',");
sb.append("'"+childCount2+"',");
sb.append("'"+inspectionType+"',");
sb.append("'"+icsStatus+"',");
sb.append("(SELECT ID FROM CUSTOMER_PROJECT WHERE NAME_OF_PROJECT=\"Chitradurga\"),");
sb.append(farmerEconomyId+",");
sb.append("'"+statusCode+"',");
sb.append("'"+statusMsg+"');\n");
sb.append(accountInsertQuery);
String accountNo = getExact(String.valueOf(accountSeq++), 10);
sb.append(accountNo);
sb.append("\",'SB','3',CURDATE(),'1',NOW(),NOW(),\"");
sb.append(farmerId);
sb.append("\",0.00,0.00);\n");
sb.append(contractQuery);
String contractNo = getExact(String.valueOf(contractSeq++), 6);
sb.append(contractNo);
sb.append("\",(SELECT ID FROM FARMER WHERE FARMER_ID=\"");
sb.append(farmerId);
sb.append("\"),null,1,null,(SELECT ID FROM ESE_ACCOUNT WHERE PROFILE_ID=\"");
sb.append(farmerId);
sb.append("\"),0.000,0.00,1);\n");
sb.append(contarctPricePatternQuery);
sb.append(contractPriceSeq+"',"+"1);\n");
sb.append(contarctPricePatternQuery);
sb.append(contractPriceSeq+"',"+"2);\n");
sb.append(contarctPricePatternQuery);
sb.append(contractPriceSeq+"',"+"3);\n");
sb.append(contarctPricePatternQuery);
sb.append(contractPriceSeq+"',"+"4);\n");
contractPriceSeq++;
sb.append(cardInsertQuery);
String cardNo = getExact(String.valueOf(cardSeq++), 10);
sb.append(cardNo);
sb.append("\",NULL,'2',CURDATE(),'0',NOW(),NOW(),\"");
sb.append(farmerId);
sb.append("\",'0');\n\n");
System.out.println("----------ROW_NO:" + i + "-----------");
i++;
}
sb.append("UPDATE ESE_SEQ SET SEQ_VAL=");
sb.append(contractSeq - 1);
sb.append(" WHERE SEQ_KEY='CONTRACT_NO_SEQ';\n");
sb.append("UPDATE ESE_SEQ SET SEQ_VAL=");
sb.append(accountSeq - 1);
sb.append(" WHERE SEQ_KEY='FARMER_ACCOUNT_NO_SEQ';\n");
sb.append("UPDATE ESE_SEQ SET SEQ_VAL=");
sb.append(cardSeq - 1);
sb.append(" WHERE SEQ_KEY='FARMER_CARD_ID_SEQ';\n");
sb.append("UPDATE FARMER_ID_SEQ SET WEB_SEQ=");
sb.append(farmerSeq - 1);
sb.append(";\n");
baos.write(sb.toString().getBytes());
myOutput.write(baos.toByteArray());
System.out.println("------Query Generated Successfully-------");
} catch (Exception e) {
System.out.println("Exception:" + e);
System.out.println("----------ROW_NO:" + i + "-----------" );
e.printStackTrace();
}
}
/**
* Gets the exact.
* #param value the value
* #param count the count
* #return the exact
*/
public static String getExact(String value, int count) {
return ((value.length() > count) ? (value.substring(0, count)) : (getEmptyLength(count
- value.length()))
+ value);
}
/**
* Gets the empty length.
* #param count the count
* #return the empty length
*/
public static String getEmptyLength(int count) {
StringBuffer data = new StringBuffer();
for (int i = 0; i < count; i++)
data.append("0");
return data.toString();
}
}
All numeric values/data in the excel sheet must have a leading single quote: '
For example :
'07543234337
'0051698435638
'10005
switch (cell.getCellType()) {
case Cell.CELL_TYPE_BOOLEAN:
cell.getBooleanCellValue()
break;
case Cell.CELL_TYPE_NUMERIC:
cell.getNumericCellValue()
break;
case Cell.CELL_TYPE_STRING:
cell.getStringCellValue()
break;
case Cell.CELL_TYPE_BLANK:
break;
default:
cell.toString()
}
try above one
private static String REGEX_ANY_MONTH = "January|Jan|February|Feb|March|Mar|April|Apr|May|June|Jun|"
+ "July|Jul|August|Aug|September|Sep|October|Oct|November|Nov|December|Dec";
private static String REGEX_ANY_YEAR = "[0-9]{4}";
private static String REGEX_ANY_DATE = "[0-9]{1,2}";
private String getFormat(String date) {
if (date.matches(REGEX_ANY_MONTH + " " + REGEX_ANY_DATE + "," + " " + REGEX_ANY_YEAR)) {
return "{MONTH} {DAY}, {YEAR}";
} else if (date.matches(REGEX_ANY_MONTH + " " + REGEX_ANY_YEAR)){
return "{MONTH} {YEAR}";
}
return null;
}
#Test
public void testGetFormatDateString() throws Exception{
String format = null;
String test = null;
test = "March 2012";
format = Whitebox.<String> invokeMethod(obj, "getFormat", test);
Assert.assertEquals("{MONTH} {YEAR}", format);
test = "March 10, 2012";
format = Whitebox.<String> invokeMethod(obj, "getFormat", test);
Assert.assertEquals("{MONTH} {DATE}, {YEAR}", format);
}
Both of the asserts are failing for me. What am I missing?
You need to wrap your piped list of month names in parentheses in order for it to match.
private static String REGEX_ANY_MONTH = "(January|Jan|February|Feb|March|Mar|April|Apr|May|June|Jun|"
+ "July|Jul|August|Aug|September|Sep|October|Oct|November|Nov|December|Dec)";
Otherwise the 'or' condition will be or-ing more than just the month.