Develop Reference

kafka to hdfs with confluent source code

For the requirement of my project, I need to build a class from the confluent java code to write data from kafka topic to the hdfs filesystem.
It is actually working in CLI with connect-standalone, but I need to do the same thing with the source code which I built successfully.
I have a problem with SinkTask and hdfsConnector classes.
An exception is showing up in the put method.
Here below is my class code:
package io.confluent.connect.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.connect.errors.ConnectException;
import org.apache.kafka.connect.sink.SinkConnector;
import org.apache.kafka.connect.sink.SinkRecord;
import org.apache.kafka.connect.sink.SinkTaskContext;
import io.confluent.connect.avro.AvroData;
import io.confluent.connect.hdfs.avro.AvroFormat;
import io.confluent.connect.hdfs.partitioner.DefaultPartitioner;
import io.confluent.connect.storage.common.StorageCommonConfig;
import io.confluent.connect.storage.partitioner.PartitionerConfig;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.config.ConfigDef;
public class main{
private static Map<String, String> props = new HashMap<>();
protected static final TopicPartition TOPIC_PARTITION = new TopicPartition(TOPIC, PARTITION);
protected static String url = "hdfs://localhost:9000";
protected static SinkTaskContext context;
public static void main(String[] args) {
HdfsSinkConnector hk = new HdfsSinkConnector();
HdfsSinkTask h = new HdfsSinkTask();
props.put(StorageCommonConfig.STORE_URL_CONFIG, url);
props.put(HdfsSinkConnectorConfig.HDFS_URL_CONFIG, url);
props.put(HdfsSinkConnectorConfig.FLUSH_SIZE_CONFIG, "3");
props.put(HdfsSinkConnectorConfig.FORMAT_CLASS_CONFIG, AvroFormat.class.getName());
try {
hk.start(props);
Collection<SinkRecord> sinkRecords = new ArrayList<>();
SinkRecord record = new SinkRecord("test", 0, null, null, null, null, 0);
sinkRecords.add(record);
h.initialize(context);
h.put(sinkRecords);
hk.stop();
} catch (Exception e) {
throw new ConnectException("Couldn't start HdfsSinkConnector due to configuration error", e);
}
}
}

How to run particular Test step of soapUi in java

I want to run particular testStep of my testcase of soap ui using java code. My problem is when I try to run at test step level it need argument of TestCase runner which is anonymous inner type and TestCaseRunContext which is interface. Do I have to implement both to run the same? if yes can please any sample how to do that??
here's my code
package com.testauto.soaprunner.soap.impl;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.eviware.soapui.SoapUI;
import com.eviware.soapui.StandaloneSoapUICore;
import com.eviware.soapui.impl.wsdl.WsdlProject;
import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCaseRunner;
import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
import com.eviware.soapui.model.TestPropertyHolder;
import com.eviware.soapui.model.iface.MessageExchange;
import com.eviware.soapui.model.propertyexpansion.PropertyExpansionUtils;
import com.eviware.soapui.model.testsuite.TestCase;
import com.eviware.soapui.model.testsuite.TestCaseRunContext;
import com.eviware.soapui.model.testsuite.TestProperty;
import com.eviware.soapui.model.testsuite.TestStepResult;
import com.eviware.soapui.model.testsuite.TestSuite;
import com.eviware.soapui.support.types.StringToObjectMap;
import com.eviware.soapui.support.types.StringToStringsMap;
import com.testauto.soaprunner.data.InputData;
import com.testauto.soaprunner.data.ReportData;
public class RunTestImpl{
static Logger logger = LoggerFactory.getLogger(RunTestImpl.class);
List<ReportData> reportDatList=new ArrayList<ReportData>();
public List<ReportData> process(Map<String, String> readDataMap, InputData input, Map<List<String>, String> configurationMap, List<String> configuration, WsdlTestSuite testSuite)
{
List<ReportData> report = new ArrayList<ReportData>();
logger.info("Into the Class for running test cases");
try{
report= getTestSuite(readDataMap,input,configurationMap,configuration,testSuite);
}
catch(Exception e)
{
logger.info(e.getMessage());
}
return report;
}
private List<ReportData> getTestSuite(Map<String, String> readDataMap, InputData input, Map<List<String>, String> configurationMap, List<String> configuration, WsdlTestSuite testSuite) throws Exception {
ReportData report=new ReportData();
logger.info("Into the Class for running test cases");
String suiteName = "";
String reportStr = "";
List<String> testCaseNameList= setPropertyValues(readDataMap,input);
WsdlTestCaseRunner runner = null;
List<TestSuite> suiteList = new ArrayList<TestSuite>();
List<TestCase> caseList = new ArrayList<TestCase>();
SoapUI.setSoapUICore(new StandaloneSoapUICore(true));
System.out.println("testcase name "+ configurationMap.get(configuration));
// WsdlTestCase testCase= testSuite.getTestCaseByName(input.getApiName()+"_"+testCaseName+"_TestCase");
WsdlTestCase testCase= testSuite.getTestCaseByName("my_TESTCASE");
WsdlTestStep tesStep=testCase.getTestStepByName(configurationMap.get(testCaseNameList));
System.out.println("test case name:"+testCase.getName());
report.setTestCase(testCase.getName());
suiteList.add(testSuite);
runner= tesStep.run(?,?);
return reportDatList;
}
private List<String> setPropertyValues(Map<String, String> readDataMap, InputData input) {
String testCaseName="";
TestPropertyHolder holder = PropertyExpansionUtils.getGlobalProperties();
List<String> dataConfigurationList=new ArrayList<String>();
Iterator entries = readDataMap.entrySet().iterator();
while (entries.hasNext()) {
Entry thisEntry = (Entry) entries.next();
String key = (String) thisEntry.getKey();
String value = (String) thisEntry.getValue();
testCaseName+=key;
holder.setPropertyValue(key, holder.getPropertyValue(key));
dataConfigurationList.add(key);
}
System.out.println("testCaseName"+testCaseName);
return dataConfigurationList;
}
}
}

After trying different things I got something like this.
TestCaseRunContext context = new MockTestRunContext(new MockTestRunner(testStep.getTestCase()), testStep);
MockTestRunner runner = new MockTestRunner(testStep.getTestCase());
TestStepResult testStepResult= testStep.run(runner, context);
I don't know how it works this trick worked for me. if someone know the reason behind this please share

How to replace variables in the header and in tables with docx4j?

I'm trying to replace variables in the header of a document and in tables but I don't know how to proceed. I managed to replace variables in the body of the document but this method (using ${}) does not work for the headers and tables.
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.HashMap;
import java.util.List;
import org.docx4j.XmlUtils;
import org.docx4j.customxml.ObjectFactory;
import org.docx4j.dml.wordprocessingDrawing.Inline;
import org.docx4j.jaxb.Context;
import org.docx4j.model.datastorage.migration.VariablePrepare;
import org.docx4j.model.structure.HeaderFooterPolicy;
import org.docx4j.model.structure.SectionWrapper;
import org.docx4j.openpackaging.exceptions.Docx4JException;
import org.docx4j.openpackaging.packages.WordprocessingMLPackage;
import org.docx4j.openpackaging.parts.CustomXmlDataStoragePart;
import org.docx4j.openpackaging.parts.Part;
import org.docx4j.openpackaging.parts.PartName;
import org.docx4j.openpackaging.parts.Parts;
import org.docx4j.openpackaging.parts.WordprocessingML.BinaryPartAbstractImage;
import org.docx4j.openpackaging.parts.WordprocessingML.HeaderPart;
import org.docx4j.openpackaging.parts.WordprocessingML.MainDocumentPart;
import org.docx4j.utils.BufferUtil;
import org.docx4j.wml.Hdr;
import org.docx4j.wml.HdrFtrRef;
import org.docx4j.wml.HeaderReference;
import java.util.Locale;
import javax.xml.bind.JAXBElement;
import java.text.DateFormat;
import org.docx4j.openpackaging.parts.WordprocessingML.HeaderPart;
import org.docx4j.wml.HdrFtrRef;
public class EditInvoice {
private static WordprocessingMLPackage template;
private static ObjectFactory factory;
public static void main (String[] args) throws Exception {
boolean save = true;
String outputfilepath = System.getProperty("user.dir")+ "/InvoiceEdited.docx";
java.util.Date uDate = new java.util.Date();
java.sql.Date sDate = new java.sql.Date(System.currentTimeMillis());
sDate = new java.sql.Date(uDate.getTime());
uDate = new java.util.Date(sDate.getTime());
Locale locale = Locale.getDefault();
DateFormat dateFormat = DateFormat.getDateInstance(DateFormat.FULL, locale);
//System.out.println(dateFormat.format(sDate));
template = WordprocessingMLPackage.load(new FileInputStream(new File("invoice_template_sample.docx")));
VariablePrepare.prepare(template);
List<SectionWrapper> sectionWrappers = template.getDocumentModel().getSections();
MainDocumentPart documentPart = template.getMainDocumentPart();
HashMap<String, String> variables = new HashMap<String, String>();
// populate doc variables
variables.put("Name", "John Doe");
variables.put("Phone", "(123) 456 78 90");
variables.put("CompanyName", "BSI Business Systems Integration AG");
variables.put("Email", "john.doe#bsiag.com");
variables.put("CompanyAddress", "Täfernstrasse 16a, 5405 Baden");
variables.put("InvoiceNo", "No. 2013-007");
variables.put("InvoiceDate", dateFormat.format(sDate));
variables.put("BillingName", "Jane Smith");
variables.put("PayableToName", "John Doe, BSI");
variables.put("SubTotal", "$1,530.00");
variables.put("SalesTax", "$229.50");
variables.put("Shipping", "$250.00");
variables.put("Total", "$2,009.50");
// and content for embedded table
Object[][] orderItems = new Object[][]{
new Object[]{"1", "Table", "$800.00", "$800.00"},
new Object[]{"4", "Chair", "$150.00", "$600.00"},
new Object[]{"1", "Assembling", "$130.00", "$130.00"},
};
try
{
documentPart.variableReplace(variables);
//documentPart.addObject(orderItems);
}
catch (Exception e)
{
System.out.println(e);
}
if (save) {
template.save(new java.io.File(outputfilepath) );
} else {
System.out.println(XmlUtils.marshaltoString(documentPart.getContents(), true, true));
}
}
}

To replace variables in headers, you need to do variable replacement to the relevant header parts. Here, you're only doing it in the main document part.
Regarding tables, the variable replacement stuff isn't designed to duplicate rows (eg one row per invoice line item). In other words, it won't insert rows. So without more code on your part, your Object[][] orderItems won't do anything.
(In contrast, docx4j's XML data binding does handle that, using an OpenDoPE od:repeat)

Java: CSV to XML using Jackson

I'm trying to update the following code to output a CSV file into XML. The code below does a brilliant job in converting the CSV to JSON using the headers in the CSV to make the required JSON.
But I want to output as XML instead, and I can't seem to find any good advice to complete this.
Would appreciate some pointers.
package reader;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
public class JacksonPackage
{
public static void main(String[] args) throws Exception
{
File input = new File("c:\\temp\\data.csv");
File output = new File("c:\\temp\\data.json");
List<Map<?, ?>> data = readObjectsFromCsv(input);
writeAsJson(data, output);
}
public static List<Map<?, ?>> readObjectsFromCsv(File file)
throws IOException
{
CsvSchema bootstrap = CsvSchema.emptySchema().withHeader();
CsvMapper csvMapper = new CsvMapper();
MappingIterator<Map<?, ?>> mappingIterator = csvMapper
.reader(Map.class).with(bootstrap).readValues(file);
return mappingIterator.readAll();
}
public static void writeAsJson(List<Map<?, ?>> data, File file)
throws IOException
{
ObjectMapper mapper = new ObjectMapper();
mapper.writeValue(file, data);
}
}

Isn't it just a matter of replacing ObjectMapper with XmlMapper?

jackson.annotation import cannot resolve

I am trying to write a program that parse JSON in java. Two of the imports that I am using could not be resolved:
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.annotation.JsonMappingException;
I managed to resolve the jackson.core by downloading the version 2.2.3 and adding it to jars in the build path, But I still have the same error for jackson.annotation even after I downloaded jackson-annotations-2.3.3.jar(The latest version as I know) and added it to jars. I would need it to work so I can make my JsonMappingException work.
This is part of the code I have:
package weather.data;
//java object imports
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.annotation.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.net.URL;
import java.io.IOException;
public class usertest {
public static void main(String[] args) throws JsonParseException, JsonMappingException, IOException
{
URL jsonUrl = new URL("http://api.openweathermap.org/data/2.5/forecast?lat=35&lon=139");
ObjectMapper mapper = new ObjectMapper();
Jweather jweather = mapper.readValue(jsonUrl, Jweather.class);
for (int i=0; i<10; i++){
System.out.println(temp[i]);
}
}
}
Any ideas?

After some researches on the web I found out that I do not need
import com.fasterxml.jackson.annotation.JsonMappingException;
for my JsonMappingException to work. but I need
import com.fasterxml.jackson.databind.JsonMappingException;
for it. So my code is like this after importing the databind.JsonMappingException
package weather.data;
//java object imports
import com.fasterxml.jackson.core.JsonParseException;
//import com.fasterxml.jackson.annotation.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.JsonMappingException;
import java.io.File;
import java.net.URL;
import java.io.IOException;
public class usertest {
public static void main(String[] args) throws JsonParseException, JsonMappingException, IOException
{
URL jsonUrl = new URL("http://api.openweathermap.org/data/2.5/forecast?lat=35&lon=139");
ObjectMapper mapper = new ObjectMapper();
Jweather jweather = mapper.readValue(jsonUrl, Jweather.class);
for (int i=0; i<10; i++){
System.out.println(jweather.getList().get(i).getMain().getTemp());
}
}
}