Develop Reference

Jackson ObjectMapper - writing value to stream

I have a requirement to fetch and write thousands of records from DB, convert to json and write to zip file. I am able to write with below implementation as well.
#Override
public StreamingResponseBody fetchAndWriteOrderAllocationsToFile(LocalDate date) {
int orderCount = orderDao.getOrderCount(date);
// Pagination
return outputStream -> {
try (ZipOutputStream zipOut = new ZipOutputStream(new BufferedOutputStream(outputStream))) {
zipOut.putNextEntry(new ZipEntry("report.txt"));
int startIndex = 0, count = orderCount;
do {
List<String> orderSerialNos = orderDao.getOrderSerialNos(date, startIndex, PAGESIZE);
orderSerialNos.parallelStream().forEach(orderSerialNo -> {
try {
writeToStream(zipOut, allocationsService.getAllocationsFromOrderItems(orderSerialNo), objectMapper);
} catch (Exception e) {
writeToStream(zipOut, Allocations.builder()
.orderSerialNo(orderSerialNo)
.build(), objectMapper);
}
});
count -= PAGESIZE;
startIndex += PAGESIZE;
} while (count > 0);
}
};
}
private static void writeToStream(OutputStream outputStream,
Object result,
ObjectMapper objectMapper) {
try {
objectMapper.writeValue(outputStream, result);
} catch (IOException e) {
log.error("Error writing results to stream", e);
}
}
However I would like to introduce a new line character(or comma) after every json being written to file.
The closest I got was overriding PrettyPrinter.writeEndObject method to something like below and use the overridden PrettyPrinter class. This obviously adds new line char to all the sub objects of json as well as every new json. The expectation is to have the new line character only after each json.
Is there any way to accomplish this?
#Override
public void writeEndObject(JsonGenerator g, int nrOfEntries) throws IOException {
g.writeRaw("}\n");
}
Above code gives:
{"orderSerialNo":"1234-ABCD","orderId":1,"shippingAllocations":[{"recipientId":25,"itemId":3893814,"itemSku":"ABC","quantity":1,"shippingItemId":3893815,"shippingSku":"DEF","shipperId":66,"allocation":0}
],"sdAllocations":[],"idAllocations":[]}
{"orderSerialNo":"6789-EFGH","orderId":2,"shippingAllocations":[{"recipientId":45,"itemId":88,"itemSku":"BLAH","quantity":1,"shippingItemId":78,"shippingSku":"HELP","shipperId":99,"allocation":7.95}
],"sdAllocations":[],"idAllocations":[]}
The expectation is:
{"orderSerialNo":"1234-ABCD","orderId":1,"shippingAllocations"[{"recipientId":25,"itemId":3893814,"itemSku":"ABC","quantity":1,"shippingItemId":3893815,"shippingSku":"DEF","shipperId":66,"allocation":0}],"sdAllocations":[],"idAllocations":[]}
{"orderSerialNo":"6789-EFGH","orderId":2,"shippingAllocations":[{"recipientId":45,"itemId":88,"itemSku":"BLAH","quantity":1,"shippingItemId":78,"shippingSku":"HELP","shipperId":99,"allocation":7.95}],"sdAllocations":[],"idAllocations":[]}

Read from splunk source and write to topic - writing same record. not pulling latest records

same record is being written to topic. not pulling latest records from splunk. time parameters are set in start method to pull last one min data. Any inputs.
currently i dont set offset from source. when poll is run every time, does it look for source offset and then poll? in logs can we have time as offset.
#Override
public List<SourceRecord> poll() throws InterruptedException {
List<SourceRecord> results = new ArrayList<>();
Map<String, String> recordProperties = new HashMap<String, String>();
while (true) {
try {
String line = null;
InputStream stream = job.getResults(previewArgs);
String earlierKey = null;
String value = null;
ResultsReaderCsv csv = new ResultsReaderCsv(stream);
HashMap<String, String> event;
while ((event = csv.getNextEvent()) != null) {
for (String key: event.keySet()) {
if(key.equals("rawlogs")){
recordProperties.put("rawlogs", event.get(key)); results.add(extractRecord(Splunklog.SplunkLogSchema(), line, recordProperties));
return results;}}}
csv.close();
stream.close();
Thread.sleep(500);
} catch(Exception ex) {
System.out.println("Exception occurred : " + ex);
}
}
}
private SourceRecord extractRecord(Schema schema, String line, Map<String, String> recordProperties) {
Map<String, String> sourcePartition = Collections.singletonMap(FILENAME_FIELD, FILENAME);
Map<String, String> sourceOffset = Collections.singletonMap(POSITION_FIELD, recordProperties.get(OFFSET_KEY));
return new SourceRecord(sourcePartition, sourceOffset, TOPIC_NAME, schema, recordProperties);
}
#Override
public void start(Map<String, String> properties) {
try {
config = new SplunkSourceTaskConfig(properties);
} catch (ConfigException e) {
throw new ConnectException("Couldn't start SplunkSourceTask due to configuration error", e);
}
HttpService.setSslSecurityProtocol(SSLSecurityProtocol.TLSv1_2);
Service service = new Service("splnkip", port);
String credentials = "user:pwd";
String basicAuthHeader = Base64.encode(credentials.getBytes());
service.setToken("Basic " + basicAuthHeader);
String startOffset = readOffset();
JobArgs jobArgs = new JobArgs();
if (startOffset != null) {
log.info("-------------------------------task OFFSET!NULL ");
jobArgs.setExecutionMode(JobArgs.ExecutionMode.BLOCKING);
jobArgs.setSearchMode(JobArgs.SearchMode.NORMAL);
jobArgs.setEarliestTime(startOffset);
jobArgs.setLatestTime("now");
jobArgs.setStatusBuckets(300);
} else {
log.info("-------------------------------task OFFSET=NULL ");
jobArgs.setExecutionMode(JobArgs.ExecutionMode.BLOCKING);
jobArgs.setSearchMode(JobArgs.SearchMode.NORMAL);
jobArgs.setEarliestTime("+419m");
jobArgs.setLatestTime("+420m");
jobArgs.setStatusBuckets(300);
}
String mySearch = "search host=search query";
job = service.search(mySearch, jobArgs);
while (!job.isReady()) {
try {
Thread.sleep(500);
} catch (InterruptedException ex) {
log.error("Exception occurred while waiting for job to start: " + ex);
}
}
previewArgs = new JobResultsPreviewArgs();
previewArgs.put("output_mode", "csv");
stop = new AtomicBoolean(false);
}

how to fetch and validate csv header in open csv?

I want to fetch header from csv file . If I am not use this skipLines then I will get header at 0 index array . But I want to fetch header directly using HeaderColumnNameMappingStrategy but it will not work with my code.
I also want to validate header column list ( like csv had not allowed to contain extra column)
I had also check this How to validate the csv headers using opencsv but it was not helpful to me.
#SuppressWarnings({ "unchecked", "rawtypes" })
public Map<String, Object> handleStockFileUpload(MultipartFile file, Long customerId) {
Map<String, Object> responseMap = new HashMap<>();
responseMap.put("datamap", "");
responseMap.put("errormap", "");
responseMap.put("errorkeys", "");
List<Map<String, Integer>> list = new ArrayList<>();
List<StockCsvDTO> csvStockList = new ArrayList<>();
try {
String fileName = new SimpleDateFormat("yyyy_MM_dd_HHmmss").format(new Date()) + "_" + file.getOriginalFilename();
responseMap.put("filename", fileName);
File stockFile = new File(productsUploadFilePath + fileName);
stockFile.getParentFile().mkdirs();
FileOutputStream fos = new FileOutputStream(stockFile);
fos.write(file.getBytes());
fos.close();
CsvTransfer csvTransfer = new CsvTransfer();
ColumnPositionMappingStrategy ms = new ColumnPositionMappingStrategy();
ms.setType(StockCsv.class);
Reader reader = Files.newBufferedReader(Paths.get(productsUploadFilePath + fileName));
CSVReader csvReader = new CSVReader(reader);
CsvToBean cb = new CsvToBeanBuilder(reader)
.withType(StockCsv.class)
.withMappingStrategy(ms)
.withSkipLines(1)
.build();
csvTransfer.setCsvList(cb.parse());
reader.close();
csvStockList = csvTransfer.getCsvList();
} catch (Exception e) {
e.printStackTrace();
responseMap.put("status", "servererror");
}
responseMap.put("datamap", csvStockList);
return responseMap;
}

I found the following solution:
Use #CsvBindByName with HeaderColumnNameMappingStrategy,e.g. annotate your bean properties with #CsvBindByName:
public static class HollywoodActor {
private int id;
#CsvBindByName(column = "First Name")
private String firstName;
#CsvBindByName(column = "Last Name")
private String lastName;
// getter / setter
}
Add a method like this:
public class CsvParser {
public <T> ParseResult<T> parseByPropertyNames(Reader csvReader, Class<T> beanClass) throws IOException {
CSVReader reader = new CSVReaderBuilder(csvReader).withCSVParser(new
CSVParserBuilder().build()).build();
CsvToBean<T> bean = new CsvToBean();
HeaderColumnNameMappingStrategy<T> mappingStrategy = new HeaderColumnNameMappingStrategy();
mappingStrategy.setType(beanClass);
bean.setMappingStrategy(mappingStrategy);
bean.setCsvReader(reader);
List<T> beans = bean.parse();
return new CsvParseResult<>(mappingStrategy.generateHeader(), beans);
}
and also don't forget to add public class ParseResult
public class ParseResult <T> {
private final String[] headers;
private final List<T> lines;
// all-args constructor & getters
}
Use then use them in your code:
String csv = "Id,First Name,Last Name\n" + "1, \"Johnny\", \"Depp\"\n" + "2, \"Al\", \"Pacino\"";
CsvParseResult<HollywoodActor> parseResult = parser
.parseByPropertyNames(new InputStreamReader(new ByteArrayInputStream(csv.getBytes(StandardCharsets.UTF_8), HollywoodActor.class)));
From ParseResult.headers you can get actual headers from which were in your .csv file. Just compare them with what's expected.
Hope that helps!

Here I was comparing my csvHeader with originalHeader:
List<String> originalHeader = fileUploadUtility.getHeader(new StockCsv());
List<String> invalidHeader = csvHeader.stream().filter(o -> (originalHeader.stream().filter(f -> f.equalsIgnoreCase(o)).count()) < 1).collect(Collectors.toList());
if(null != invalidHeader && invalidHeader.size() > 0 && invalidHeader.toString().replaceAll("\\[\\]", "").length() > 0) {
msg = "Invalid column(s) : " + invalidHeader.toString().replace(", ]", "]") + ". Please remove invalid column(s) from file.";
resultMap.put(1, msg);
}
public List<String> getHeader(T pojo) {
// TODO Auto-generated method stub
final CustomMappingStrategy<T> mappingStrategy = new CustomMappingStrategy<>();
mappingStrategy.setType((Class<? extends T>) pojo.getClass());
String header[] = mappingStrategy.generateHeader();
List<String> strHeader = Arrays.asList(header);
return strHeader;
}

Here is an alternative to your present problem.First, define what you expect your headers to look like. For example:
public static final ArrayList<String> fileFormat = new ArrayList<> (Arrays.asList("Values1", "Values2", "Values3", "Values4"));
Now, write a method to return custom errors if any exist:
public String validateCsvFileDetails(MultipartFile file, Set<String> requiredHeadersArray) {
Set<String> errors = new HashSet<>();
try {
InputStream stream = file.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
String headerLine = reader.readLine();
if (Objects.isNull(headerLine))
return "The file has no headers, please ensure it has the correct upload format";
List<String> headersInFileList;
String[] headersInFileArray;
if (headerLine.contains(",")) {
headersInFileArray = StringUtils.split(headerLine, ",");
headersInFileList = Arrays.asList(headersInFileArray);
} else//the headerline has only one headerfield
{
headersInFileList = Collections.singletonList(headerLine);
}
for (String header : requiredHeadersArray) {
if (!headersInFileList.contains(header))
errors.add("The file has the wrong header format, please ensure " + header + " header is present");
}
//if there are errors, return it
if (!errors.isEmpty())
return sysUtils.getStringFromSet(errors);
//Ensure the csv file actually has values after the header, but don't read beyond the first line
String line;
int counter = 0;
while ((line = reader.readLine()) != null) {
counter++;
if (counter > 0)
break;
}
//if line is null return validation error
if (Objects.isNull(line))
return "Cannot upload empty file";
} catch (Exception e) {
logger.error(new Object() {
}.getClass().getEnclosingMethod().getName(), e);
return "System Error";
}
return null;
}
Now you can validate you file headers as follows:
String errors = validateCsvFileDetails(file, new HashSet<>(fileFormat));
if (errors != null)
return error
//proceed

Give this a try using captureHeader as a pre-filter:
...
private class CustomHeaderColumnNameMappingStrategy<T> extends HeaderColumnNameMappingStrategy {
private String[] expectedHeadersOrdered = {"Column1", "Column2", "Column3", "Column4", "Column5"};
#Override
public void captureHeader(CSVReader reader) throws IOException, CsvRequiredFieldEmptyException {
String[] actualCsvHeaders = reader.peek();
String actualHeader, expectedHeader;
if (expectedHeadersOrdered.length > actualCsvHeaders.length) {
throw new CsvRequiredFieldEmptyException("Missing header column.");
} else if (expectedHeadersOrdered.length < actualCsvHeaders.length) {
throw new IOException("Unexpected extra header column.");
}
// Enforce strict column ordering with index
// TODO: you might want to employ simple hashMap, List, set, etc. as needed
for (int i=0; i<actualCsvHeaders.length; i++) {
actualHeader = actualCsvHeaders[i];
expectedHeader = expectedHeadersOrdered[i];
if ( ! expectedHeader.equals(actualHeader) ) {
throw new IOException("Header columns mismatch in ordering.");
}
}
super.captureHeader(reader); // Back to default processing if the headers include ordering are as expected
}
}
CustomHeaderColumnNameMappingStrategy yourMappingStrategy = new CustomHeaderColumnNameMappingStrategy<YourPOJO>();
ourMappingStrategy.setType(YourPOJO.class);
try {
pojosFromCsv = new CsvToBeanBuilder<YourPOJO>(new FileReader(csvFile))
.withType(YourPOJO.class)
.withMappingStrategy(yourMappingStrategy)
.build();
pojosFromCsv.stream();
}
Inspired by Using captureHeader in OpenCSV

Apache POI event API Update existing Excel sheet

I have large excel file with several worksheets.
I want to process just one sheet in file...Read value from two columns and update two columns.
Using this code, I am able to read data from sheet.But unable to figure out, how to save output back.
public class ExcelFunctions {
private class ExcelData implements SheetContentsHandler {
private Record rec ;
public void startRow(int rowNum) {
rec = new Record();
output.put("R"+rowNum, rec);
}
public void endRow(int rowNum) {
}
public void cell(String cellReference, String formattedValue,
XSSFComment comment) {
int thisCol = (new CellReference(cellReference)).getCol();
if(thisCol==7){
try {
rec.setK1(formattedValue);
} catch (Exception e) {
}
}
if(thisCol==8){
try {
rec.setK2(formattedValue);
} catch (Exception e) {
}
}
if(thisCol == 27){
String key = rec.full_key();
System.out.println(key);
///////Process Matched Key...get Data
//////Set value to column 27
}
if(thisCol == 28){
String key = rec.full_key();
System.out.println(key);
///////Process Matched Key...get Data
//////Set value to column 28
}
}
public void headerFooter(String text, boolean isHeader, String tagName) {
}
}
///////////////////////////////////////
private final OPCPackage xlsxPackage;
private final Map<String, Record> output;
public ExcelFunctions(OPCPackage pkg, Map<String, Record> output) {
this.xlsxPackage = pkg;
this.output = output;
}
public void processSheet(
StylesTable styles,
ReadOnlySharedStringsTable strings,
SheetContentsHandler sheetHandler,
InputStream sheetInputStream)
throws IOException, ParserConfigurationException, SAXException {
DataFormatter formatter = new DataFormatter();
InputSource sheetSource = new InputSource(sheetInputStream);
try {
XMLReader sheetParser = SAXHelper.newXMLReader();
ContentHandler handler = new XSSFSheetXMLHandler(
styles, null, strings, sheetHandler, formatter, false);
sheetParser.setContentHandler(handler);
sheetParser.parse(sheetSource);
} catch(ParserConfigurationException e) {
throw new RuntimeException("SAX parser appears to be broken - " + e.getMessage());
}
}
public void process()
throws IOException, OpenXML4JException, ParserConfigurationException, SAXException {
ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(this.xlsxPackage);
XSSFReader xssfReader = new XSSFReader(this.xlsxPackage);
StylesTable styles = xssfReader.getStylesTable();
XSSFReader.SheetIterator iter = (XSSFReader.SheetIterator) xssfReader.getSheetsData();
boolean found = false;
while (iter.hasNext() && !found) {
InputStream stream = iter.next();
String sheetName = iter.getSheetName();
if(sheetName.equals("All Notes") ){
processSheet(styles, strings, new ExcelData(), stream);
found = true;
}
stream.close();
}
}
#SuppressWarnings("unused")
public static void main(String[] args) throws Exception {
File xlsxFile = new File("C:\\Users\\admin\\Downloads\\Unique Name Macro\\big.xlsm");
if (!xlsxFile.exists()) {
System.err.println("Not found or not a file: " + xlsxFile.getPath());
return;
}
// The package open is instantaneous, as it should be.
OPCPackage p = OPCPackage.open(xlsxFile.getPath(), PackageAccess.READ_WRITE);
Map<String, Record> output = new HashMap<String, Record>();
ExcelFunctions xlFunctions = new ExcelFunctions(p, output);
xlFunctions.process();
p.close();
if (output != null){
for(Record rec : output.values()){
System.out.println(rec.full_key());
}
}
}
}
File is very large and I only want to use Event API.
I have successfully tested Using this code.
But this loads Whole file in memory(causing application to crash)...While I only need to edit One sheet.
public static void saveToExcel(String ofn, Map<String, Record> data) {
FileInputStream infile;
try {
infile = new FileInputStream(new File("C:\\Users\\admin\\Downloads\\Unique Name Macro\\big.xlsm"));
XSSFWorkbook workbook = new XSSFWorkbook (infile);
XSSFSheet sheet = workbook.getSheet("All Notes");
for(Record rec : output.values()){
Row dataRow = rec.getRow(rev.getRownum-1);
setCellValue(dataRow, 26, "SomeValue");
setCellValue(dataRow, 27, "SomeValue");
}
FileOutputStream out = new FileOutputStream(new File(ofn));
workbook.write(out);
infile.close();
out.close();
workbook.close();
}
catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private static void setCellValue(Row row,int col, String value){
Cell c0 = row.getCell(col);
if (c0 == null){
c0 = row.createCell(col);
}
c0.setCellValue(value);
}

I don't think there is anything provided in POI out of the box which allows to do that.
Therefore you might be better off doing this by unzipping the XLSX/XLSM file (they are actually a bunch of xml-files inside a zip) and reading the xml-files as text-files or with a normal XML Parser so that you can easily write out the changed file again to produce the XLSX/XLSM file again.

Matching Keys in a HashMap

I am attempting to do the following (in psuedocode):
Generate HashMapOne that will be populated by results
found in a DICOM file (the Key was manipulated for matching
purposes).
Generate a second HashMapTwo that will be read from a
text document.
Compare the Keys of both HashMaps, if a match add the results of
the value of HashMapOne in a new HashMapThree.
I am getting stuck with adding the matched key's value to the HashMapThree. It always populates a null value despite me declaring this a public static variable. Can anyone please tell me why this may be? Here is the code snippets below:
public class viewDICOMTags {
HashMap<String,String> dicomFile = new HashMap<String,String>();
HashMap<String,String> dicomTagList = new HashMap<String,String>();
HashMap<String,String> Result = new HashMap<String, String>();
Iterator<org.dcm4che2.data.DicomElement> iter = null;
DicomObject working;
public static DicomElement element;
DicomElement elementTwo;
public static String result;
File dicomList = new File("C:\\Users\\Ryan\\dicomTagList.txt");
public void readDICOMObject(String path) throws IOException
{
DicomInputStream din = null;
din = new DicomInputStream(new File(path));
try {
working = din.readDicomObject();
iter = working.iterator();
while (iter.hasNext())
{
element = iter.next();
result = element.toString();
String s = element.toString().substring(0, Math.min(element.toString().length(), 11));
dicomFile.put(String.valueOf(s.toString()), element.vr().toString());
}
System.out.println("Collected tags, VR Code, and Description from DICOM file....");
}
catch (IOException e)
{
e.printStackTrace();
return;
}
finally {
try {
din.close();
}
catch (IOException ignore){
}
}
readFromTextFile();
}
public void readFromTextFile() throws IOException
{
try
{
String dicomData = "DICOM";
String line = null;
BufferedReader bReader = new BufferedReader(new FileReader(dicomList));
while((line = bReader.readLine()) != null)
{
dicomTagList.put(line.toString(), dicomData);
}
System.out.println("Reading Tags from Text File....");
bReader.close();
}
catch(FileNotFoundException e)
{
System.err.print(e);
}
catch(IOException i)
{
System.err.print(i);
}
compareDICOMSets();
}
public void compareDICOMSets() throws IOException
{
for (Entry<String, String> entry : dicomFile.entrySet())
{
if(dicomTagList.containsKey(entry.getKey()))
Result.put(entry.getKey(), dicomFile.get(element.toString()));
System.out.println(dicomFile.get(element.toString()));
}
SortedSet<String> keys = new TreeSet<String>(Result.keySet());
for (String key : keys) {
String value = Result.get(key);
System.out.println(key);
}
}
}

This line of code looks very wrong
Result.put(entry.getKey(), dicomFile.get(element.toString()));
If you are trying to copy the key/value pair from HashMapOne, then this is not correct.
The value for each key added to Result will be null, because you are calling get method on Map interface on dicomFile. get requires a key as a lookup value, and you are passing in
element.toString()
where element will be the last element that was read from your file.
I think you should be using
Result.put(entry.getKey(), entry.getValue()));