Develop Reference

Create a simple job spring boot

I created a spring boot project.
I use spring data with elastic search.
The whole pipeline: controller -> service -> repository is ready.
I now have a file that represents country objects (name and isoCode) and I want to create a job to insert them all in elastic search.
I read the spring documentation and I find that there's too much configuration for such a simple job.
So I'm trying to do a simple main "job" that reads a csv, creates objects and insert them in elastic search.
But I have a bit of trouble to understand how injection would work in this case:
#Component
public class InsertCountriesJob {
private static final String file = "D:path\\to\\countries.dat";
private static final Logger LOG = LoggerFactory.getLogger(InsertCountriesJob.class);
#Autowired
public CountryService service;
public static void main(String[] args) {
LOG.info("Starting insert countries job");
try {
saveCountries();
} catch (Exception e) {
e.printStackTrace();
}
}
public static void saveCountries() throws Exception {
try (CSVReader csvReader = new CSVReader(new FileReader(file))) {
String[] values = null;
while ((values = csvReader.readNext()) != null) {
String name = values[0];
String iso = values[1].equals("N") ? values[2] : values[1];
Country country = new Country(iso, name);
LOG.info("info: country: {}", country);
//write in db;
//service.save(country); <= can't do this because of the injection
}
}
}
}

based on Simon's comment. Here's how I resolved my problem. Might help people that are getting into spring, and that are trying not to get lost.
Basically, to inject anything in Spring, you'll need a SpringBootApplication
public class InsertCountriesJob implements CommandLineRunner{
private static final String file = "D:path\\to\\countries.dat";
private static final Logger LOG = LoggerFactory.getLogger(InsertCountriesJob.class);
#Autowired
public CountryService service;
public static void main(String[] args) {
LOG.info("STARTING THE APPLICATION");
SpringApplication.run(InsertCountriesJob.class, args);
LOG.info("APPLICATION FINISHED");
}
#Override
public void run(String... args) throws Exception {
LOG.info("Starting insert countries job");
try {
saveCountry();
} catch (Exception e) {
e.printStackTrace();
}
LOG.info("job over");
}
public void saveCountry() throws Exception {
try (CSVReader csvReader = new CSVReader(new FileReader(file))) {
String[] values = null;
while ((values = csvReader.readNext()) != null) {
String name = values[0];
String iso = values[1].equals("N") ? values[2] : values[1];
Country country = new Country(iso, name);
LOG.info("info: country: {}", country);
//write in db;
service.save(country);
}
}
}
}

how to call class from different package

Good Morning all, i have problem when i want call my class in different package. i have use
private void getExecutionReport(quickfix.Message message, String tipeMessage) {
// TODO Auto-generated method stub
try {
System.out.print("Message "+ message);
StringField msgType = message.getHeader().getField(new MsgType());
System.out.print("getExecutionReport "+tipeMessage);
String symbol = message.getString(Symbol.FIELD);
System.out.print("getExecutionReport "+symbol);
String clorid = message.getString(ClOrdID.FIELD);
String orderid = message.getString(OrderID.FIELD);
String execid = message.getString(ExecID.FIELD);
String execTransType = message.getString(ExecTransType.FIELD);
String ordertype = message.getString(OrdType.FIELD);
String exectype = message.getString(ExecType.FIELD);
String orderstatus = message.getString(OrdStatus.FIELD);
int side = message.getInt(Side.FIELD);
Double orderqty = message.getDouble(OrderQty.FIELD);
int leavesqty = message.getInt(LeavesQty.FIELD);
double price = message.getDouble(Price.FIELD);
int cumqty = message.getInt(CumQty.FIELD);
int lastqty = message.getInt(LastQty.FIELD);
//call Class in different package
DAOInsert.newOrderSingleInsert(tipeMessage,symbol,clorid,orderid,execid,execTransType,ordertype,exectype,orderstatus,side,orderqty,leavesqty,price,cumqty,lastqty);
} catch (FieldNotFound e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
but when i want hit this class
public class DAOInsert {
private DAOInsert dAOInsert;
static Session sessionObj;
static SessionFactory sessionFactoryObj;
public final static Logger logger = Logger.getLogger(DAOInsert.class);
// This Method Is Used To Create The Hibernate's SessionFactory Object
private static SessionFactory buildSessionFactory() {
// Creating Configuration Instance & Passing Hibernate Configuration File
Configuration configObj = new Configuration();
configObj.configure("hibernate.cfg.xml");
// Since Hibernate Version 4.x, ServiceRegistry Is Being Used
ServiceRegistry serviceRegistryObj = new StandardServiceRegistryBuilder().applySettings(configObj.getProperties()).build();
// Creating Hibernate SessionFactory Instance
sessionFactoryObj = configObj.buildSessionFactory(serviceRegistryObj);
return sessionFactoryObj;
}
public static void newOrderSingleInsert(String tipeMessage, String symbol, String clorid, String orderid, String execid, String execTransType, String ordertype, String exectype, String orderstatus, int side, Double orderqty, int leavesqty, double price, int cumqty, int lastqty){
int count = 0;
NewOrderSingle newordersingleObj = null;
try {
System.out.print("DAOInsert "+tipeMessage);
// Getting Session Object From SessionFactory
sessionObj = buildSessionFactory().openSession();
// Getting Transaction Object From Session Object
sessionObj.beginTransaction();
// Creating Transaction Entities
// newordersingleObj = new NewOrderSingle();
//newordersingleObj.setFixProtocol("FIX.4.2");
//newordersingleObj.setBodyLength(250);
//sessionObj.save(newordersingleObj);
// Committing The Transactions To The Database
//sessionObj.getTransaction().commit();
//logger.info("\nSuccessfully Created '" + count + "' Records In The Database!\n");
} catch(Exception sqlException) {
if(null != sessionObj.getTransaction()) {
logger.info("\n.......Transaction Is Being Rolled Back.......\n");
sessionObj.getTransaction().rollback();
}
sqlException.printStackTrace();
} finally {
if(sessionObj != null) {
sessionObj.close();
}
}
}
}
i cannot hit this class using my first method. which is the problem when i hit my class ? please help me for fix my problem

Apply regex filter in GET reques use Java code, not SQL

I have a goal - get filtered contacts from database use Java, but not use SQL. Two problems:
Heavy Java code for use Database.
IllegalArgumentException: Invalid character found in the request target. The valid characters are defined in RFC 7230 and RFC 3986, because I need use symbols as ^, [, ] and other.
I can't use regex in SQL, in Java only.
Which best ways for resolve this issues?
My code.
public class Contact {
private int id;
private String name;
public Contact() {
}
public Contact(ResultSet resultSet) throws SQLException {
setId(resultSet.getInt(1));
setName(resultSet.getString(2));
}
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
#Override
public String toString() {
return "Contact [Id=" + id + ", Name=" + name + "]";
}
}
public class ManagementLayer {
private static final Logger LOG = Logger.getLogger(String.valueOf(ManagementLayer.class));
private ManagementLayer() {
}
private static class ManagementHolder{
static final ManagementLayer Instance = new ManagementLayer();
}
public static synchronized ManagementLayer getInstance() {
return ManagementHolder.Instance;
}
public List<Contact> getFilteredContacts(String regex, Connection dbConnection)
throws SQLException, PatternSyntaxException {
Pattern pattern = Pattern.compile(regex);
List<Contact> contacts = new ArrayList<>();
Statement statement = null;
ResultSet resultSet = null;
if (dbConnection == null) {
try {
Context context = new InitialContext();
DataSource dataSource = (DataSource) context.lookup("java:comp/env/jdbc/hello");
dbConnection = dataSource.getConnection();
} catch (NamingException e) {
LOG.info(e.getMessage());
}
}
try {
if (dbConnection != null) {
statement = dbConnection.createStatement();
statement.setFetchSize(1);
resultSet = statement.executeQuery("SELECT * FROM contacts");
while (resultSet.next()) {
String contactNameString = resultSet.getString(2);
Matcher matcher = pattern.matcher(contactNameString);
if (!matcher.matches()) {
Contact contact = new Contact();
contact.setId(resultSet.getInt(1));
contact.setName(contactNameString);
contacts.add(contact);
}
}
}
System.out.println(contacts.toString());
} finally {
if (resultSet != null) {
resultSet.close();
}
if (statement != null) {
statement.close();
}
}
return contacts;
}
}
#Path("contacts")
public class WebServiceLayer {
private static final Logger LOG = Logger.getLogger(String.valueOf(WebServiceLayer.class));
#GET
#Produces(MediaType.APPLICATION_JSON)
public List<Contact> getNameFilter(#QueryParam("nameFilter") String val) {
if (val == null || val.isEmpty()) {
throw new WebApplicationException(Response
.status(HttpURLConnection.HTTP_BAD_REQUEST)
.entity("Error! Parameter is wrong or empty!")
.build());
} else {
List<Contact> contacts = new ArrayList<>();
try {
contacts = ManagementLayer
.getInstance()
.getFilteredContacts(val, null);
} catch (SQLException e) {
LOG.info(e.getMessage());
}
return contacts;
}
}
}

I used ConcurrentHashMap for create cache from database.
In Tomcat server.xml file added configuration (port="8080" protocol="HTTP/1.1" URIEncoding="UTF-8" connectionTimeout="20000" redirectPort="8443" relaxedQueryChars='^{}[]|$*'). Also in servlet added URLDecoder.decode(val) for resolving correct converting encoded symbols in regular expression.

initialize all the metadata at once

I have a Singleton class which connects to Cassandra. I want to initialize processMetadata, procMetadata and topicMetadata all at once not one by one. If they gets initialize all at once then I will see consistent values from all those three not different values for either of them.
In the below code, processMetadata, procMetadata and topicMetadata is initialized for the first time inside initializeMetadata method and then it gets updated every 15 minutes.
public class CassUtil {
private static final Logger LOGGER = Logger.getInstance(CassUtil.class);
private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
// below are my three metedata which I need to update all three at once not one by one
private List<ProcessMetadata> processMetadata = new ArrayList<>();
private List<ProcMetadata> procMetadata = new ArrayList<>();
private List<String> topicMetadata = new ArrayList<>();
private Session session;
private Cluster cluster;
private static class Holder {
private static final CassUtil INSTANCE = new CassUtil();
}
public static CassUtil getInstance() {
return Holder.INSTANCE;
}
private CassUtil() {
List<String> servers = TestUtils.HOSTNAMES;
String username = TestUtils.USERNAME;
String password = TestUtils.PASSWORD;
PoolingOptions opts = new PoolingOptions();
opts.setCoreConnectionsPerHost(HostDistance.LOCAL,
opts.getCoreConnectionsPerHost(HostDistance.LOCAL));
Builder builder = Cluster.builder();
cluster =
builder
.addContactPoints(servers.toArray(new String[servers.size()]))
.withRetryPolicy(DowngradingConsistencyRetryPolicy.INSTANCE)
.withPoolingOptions(opts)
.withReconnectionPolicy(new ConstantReconnectionPolicy(100L))
.withLoadBalancingPolicy(
DCAwareRoundRobinPolicy
.builder()
.withLocalDc(
!TestUtils.isProduction() ? "DC2" : TestUtils.getCurrentLocation()
.get().name().toLowerCase()).build())
.withCredentials(username, password).build();
try {
session = cluster.connect("testkeyspace");
} catch (NoHostAvailableException ex) {
LOGGER.logError("error= ", ExceptionUtils.getStackTrace(ex));
} catch (Exception ex) {
LOGGER.logError("error= " + ExceptionUtils.getStackTrace(ex));
}
}
// start a background thread which runs every 15 minutes
public void startScheduleTask() {
scheduler.scheduleAtFixedRate(new Runnable() {
public void run() {
try {
processMetadata = processMetadata(true);
topicMetadata = listOfTopic(TestUtils.GROUP_ID);
procMetadata = procMetadata();
} catch (Exception ex) {
LOGGER.logError("error= ", ExceptionUtils.getStackTrace(ex));
}
}
}, 0, 15, TimeUnit.MINUTES);
}
// called from main thread to initialize the metadata
// and start the background thread where it gets updated
// every 15 minutes
public void initializeMetadata() {
processMetadata = processMetadata(true);
topicMetadata = listOfTopic(TestUtils.GROUP_ID);
procMetadata = procMetadata();
startScheduleTask();
}
private List<String> listOfTopic(final String consumerName) {
List<String> listOfTopics = new ArrayList<>();
String sql = "select topics from topic_metadata where id=1 and consumerName=?";
try {
// get data from cassandra
} catch (Exception ex) {
LOGGER.logError("error= ", ExceptionUtils.getStackTrace(ex), ", Consumer Name= ",
consumerName);
}
return listOfTopics;
}
private List<ProcessMetadata> processMetadata(final boolean flag) {
List<ProcessMetadata> metadatas = new ArrayList<>();
String sql = "select * from process_metadata where id=1 and is_active=?";
try {
// get data from cassandra
} catch (Exception ex) {
LOGGER.logError("error= ", ExceptionUtils.getStackTrace(ex), ", active= ", flag);
}
return metadatas;
}
private List<ProcMetadata> procMetadata() {
List<ProcMetadata> metadatas = new ArrayList<>();
String sql = "select * from schema where id=1";
try {
// get data from cassandra
} catch (SchemaParseException ex) {
LOGGER.logError("schema parsing error= ", ExceptionUtils.getStackTrace(ex));
} catch (Exception ex) {
LOGGER.logError("error= ", ExceptionUtils.getStackTrace(ex));
}
return metadatas;
}
public List<ProcessMetadata> getProcessMetadata() {
return processMetadata;
}
public List<String> getTopicMetadata() {
return topicMetadata;
}
public List<ProcMetadata> getProcMetadata() {
return procMetadata;
}
}
So from my main thread, I call initializeMetadata method only once which initializes those three metadata and then it starts a background thread which updates them every 15 minutes. Afer that I was using them like below from my multiple threads:
CassUtil.getInstance().getProcessMetadata();
CassUtil.getInstance().getTopicMetadata();
CassUtil.getInstance().getProcMetadata();
Problem Statement:-
Now I want to see same state of processMetadata, topicMetadata and procMetadata. Meaning these three metadata should be updated at same time not one after other bcoz I don't want to see mix state value for them after I do get on them.
How can I avoid this issue? Do I need to create another class which will hold these three metadata as constructor parameter?

The most efficient way to keep a consistent state of your lists can be to use an immutable class that will hold your 3 lists, you will then have a field of this type in your class that you will define volatile to make sure that all threads see the latest update of this field.
Here is for example the immutable class that we use to hold the state of the lists (it could be an ordinary class but as it is implementation specific it could be a static inner class):
private static class State {
private final List<ProcessMetadata> processMetadata;
private final List<ProcMetadata> procMetadata;
private final List<String> topicMetadata;
public State(final List<ProcessMetadata> processMetadata,
final List<ProcMetadata> procMetadata, final List<String> topicMetadata) {
this.processMetadata = new ArrayList<>(processMetadata);
this.procMetadata = new ArrayList<>(procMetadata);
this.topicMetadata = new ArrayList<>(topicMetadata);
}
// Getters
}
Then your class would be something like that:
public class CassUtil {
...
private volatile State state = new State(
new ArrayList<>(), new ArrayList<>(), new ArrayList<>()
);
...
public void startScheduleTask() {
...
this.state = new State(
processMetadata(true), listOfTopic(TestUtils.GROUP_ID),
procMetadata()
);
...
}
...
public void initializeMetadata() {
this.state = new State(
processMetadata(true), listOfTopic(TestUtils.GROUP_ID), procMetadata()
);
startScheduleTask();
}
...
public List<ProcessMetadata> getProcessMetadata() {
return this.state.getProcessMetadata();
}
public List<String> getTopicMetadata() {
return this.state.getTopicMetadata();
}
public List<ProcMetadata> getProcMetadata() {
return this.state.getProcMetadata();
}

easymock expect not working and still calling actual method

I want to test MessageProcessor1.listAllKeyword method, which in turn
calls HbaseUtil1.getAllKeyword method. Initialy, I had to deal with a problem associated with the static initializer and the constructor. The problem was to initialize a HBASE DB connection. I used powerMock to suppress static and constructor calls and it worked fine.
Even though I mocked HbaseUtil1.getAllKeyword method, actual method is being called and executes all HBase code leading to an exception, in which HBASE server is not up.
EasyMock.expect(hbaseUtil.getAllKeyword("msg", "u1")).andReturn(expectedList);
Please give me any idea on how to avoid an actual method call. I tried many ways but none of them worked.
public class MessageProcessor1
{
private static Logger logger = Logger.getLogger("MQ-Processor");
private final static String CLASS_NAME = "MessageProcessor";
private static boolean keywordsTableExists = false;
public static PropertiesLoader props;
HbaseUtil1 hbaseUtil;
/**
* For checking if table exists in HBase. If doesn't exists, will create a
* new table. This runs only once when class is loaded.
*/
static {
props = new PropertiesLoader();
String[] userTablefamilys = {
props.getProperty(Constants.COLUMN_FAMILY_NAME_COMMON_KEYWORDS),
props.getProperty(Constants.COLUMN_FAMILY_NAME_USER_KEYWORDS) };
keywordsTableExists = new HbaseUtil()
.creatTable(props.getProperty(Constants.HBASE_TABLE_NAME),
userTablefamilys);
}
/**
* This will load new configuration every time this class instantiated.
*/
{
props = new PropertiesLoader();
}
public String listAllKeyword(String userId) throws IOException {
HbaseUtil1 util = new HbaseUtil1();
Map<String, List<String>> projKeyMap = new HashMap<String, List<String>>();
//logger.info(CLASS_NAME+": inside listAllKeyword method");
//logger.debug("passed id : "+userId);
List<String> qualifiers = util.getAllKeyword("msg", userId);
List<String> keywords = null;
for (String qualifier : qualifiers) {
String[] token = qualifier.split(":");
if (projKeyMap.containsKey(token[0])) {
projKeyMap.get(token[0]).add(token[1]);
} else {
keywords = new ArrayList<String>();
keywords.add(token[1]);
projKeyMap.put(token[0], keywords);
}
}
List<Project> projects = buildProject(projKeyMap);
Gson gson = new GsonBuilder().excludeFieldsWithoutExposeAnnotation()
.create();
System.out.println("Json projects:::" + gson.toJson(projects));
//logger.debug("list all keyword based on project::::"+ gson.toJson(projects));
//return gson.toJson(projects);
return "raj";
}
private List<Project> buildProject(Map<String, List<String>> projKeyMap) {
List<Project> projects = new ArrayList<Project>();
Project proj = null;
Set<String> keySet = projKeyMap.keySet();
for (String hKey : keySet) {
proj = new Project(hKey, projKeyMap.get(hKey));
projects.add(proj);
}
return projects;
}
//#Autowired
//#Qualifier("hbaseUtil1")
public void setHbaseUtil(HbaseUtil1 hbaseUtil) {
this.hbaseUtil = hbaseUtil;
}
}
public class HbaseUtil1 {
private static Logger logger = Logger.getLogger("MQ-Processor");
private final static String CLASS_NAME = "HbaseUtil";
private static Configuration conf = null;
public HbaseUtil1() {
PropertiesLoader props = new PropertiesLoader();
conf = HBaseConfiguration.create();
conf.set(HConstants.ZOOKEEPER_QUORUM, props
.getProperty(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM));
conf.set(
HConstants.ZOOKEEPER_CLIENT_PORT,
props.getProperty(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENT_PORT));
conf.set("hbase.zookeeper.quorum", props
.getProperty(Constants.HBASE_CONFIGURATION_ZOOKEEPER_QUORUM));
conf.set(
"hbase.zookeeper.property.clientPort",
props.getProperty(Constants.HBASE_CONFIGURATION_ZOOKEEPER_CLIENT_PORT));
}
public List<String> getAllKeyword(String tableName, String rowKey)
throws IOException {
List<String> qualifiers = new ArrayList<String>();
HTable table = new HTable(conf, tableName);
Get get = new Get(rowKey.getBytes());
Result rs = table.get(get);
for (KeyValue kv : rs.raw()) {
System.out.println("KV: " + kv + ", keyword: "
+ Bytes.toString(kv.getRow()) + ", quaifier: "
+ Bytes.toString(kv.getQualifier()) + ", family: "
+ Bytes.toString(kv.getFamily()) + ", value: "
+ Bytes.toString(kv.getValue()));
qualifiers.add(new String(kv.getQualifier()));
}
table.close();
return qualifiers;
}
/**
* Create a table
*
* #param tableName
* name of table to be created.
* #param familys
* Array of the name of column families to be created with table
* #throws IOException
*/
public boolean creatTable(String tableName, String[] familys) {
HBaseAdmin admin = null;
boolean tableCreated = false;
try {
admin = new HBaseAdmin(conf);
if (!admin.tableExists(tableName)) {
HTableDescriptor tableDesc = new HTableDescriptor(tableName);
for (int i = 0; i < familys.length; i++) {
tableDesc.addFamily(new HColumnDescriptor(familys[i]));
}
admin.createTable(tableDesc);
System.out.println("create table " + tableName + " ok.");
}
tableCreated = true;
admin.close();
} catch (MasterNotRunningException e1) {
e1.printStackTrace();
} catch (ZooKeeperConnectionException e1) {
e1.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return tableCreated;
}
}
Below is my Test class.
#RunWith(PowerMockRunner.class)
#PrepareForTest(MessageProcessor1.class)
#SuppressStaticInitializationFor("com.serendio.msg.mqProcessor.MessageProcessor1")
public class MessageProcessorTest1 {
private MessageProcessor1 messageProcessor;
private HbaseUtil1 hbaseUtil;
#Before
public void setUp() {
messageProcessor = new MessageProcessor1();
hbaseUtil = EasyMock.createMock(HbaseUtil1.class);
}
#Test
public void testListAllKeyword(){
List<String> expectedList = new ArrayList<String>();
expectedList.add("raj:abc");
suppress(constructor(HbaseUtil1.class));
//suppress(method(HbaseUtil1.class, "getAllKeyword"));
try {
EasyMock.expect(hbaseUtil.getAllKeyword("msg", "u1")).andReturn(expectedList);
EasyMock.replay();
assertEquals("raj", messageProcessor.listAllKeyword("u1"));
} catch (IOException e) {
e.printStackTrace();
}catch (Exception e) {
e.printStackTrace();
}
}
}

The HbaseUtil1 is instantiated within the listAllKeyword method
public String listAllKeyword(String userId) throws IOException {
HbaseUtil1 util = new HbaseUtil1();
...
So the mock one you create in your test isn't being used at all.
If possible, make the HbaseUtil1 object passable, or settable on the MessageProcessor1 class and then set it in the test class.
Also, and note I'm not 100% familiar with PowerMock, you could include HbaseUtil1 in the prepare for test annotation. I think that will make PowerMock instantiate mocks instead of real objects and then use the expectations you provide in you test.