I am implementing a module to take the screenshot of logged in user after every 3 minutes.
This module works fine
But the problem I am facing is that whenever i logged in the user the job starts as it intends to do, but on the logout I shuts down the scheduler. But now the next time i try to login back now scheduler initialized but job isn't starting. I need to redeploy project again for next run.
Here is my LoginBean.java class from where I am starting the job.
package com.viremp.beans;
import java.awt.AWTException;
import java.io.IOException;
import java.io.Serializable;
import java.sql.SQLException;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.faces.bean.ManagedBean;
import javax.faces.bean.SessionScoped;
import javax.faces.context.ExternalContext;
import javax.faces.context.FacesContext;
import javax.servlet.http.HttpServletRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.viremp.component.HandleHobs;
#ManagedBean
#SessionScoped
public class LoginBean implements Serializable {
/**
*
*/
private static final long serialVersionUID = -8650636789236091591L;
private static Logger LOGGER = LoggerFactory.getLogger(LoginBean.class);
private String username;
private String password;
private String error;
private boolean visible = false;
private HandleHobs handleHobs;
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getError() {
return error;
}
public void setError(String error) {
this.error = error;
}
public boolean isVisible() {
return visible;
}
public void setVisible(boolean visible) {
this.visible = visible;
}
/*
* public LoginBean() { try { if (handleHobs != null &&
* !handleHobs.isJobStoredScreenShotIsStarted()) { handleHobs = new
* HandleHobs(); } } catch (Exception e) { LOGGER.error("error init job", e); }
*
* }
*/
#PostConstruct
public void init() {
try {
if (handleHobs == null || !handleHobs.isJobStoredScreenShotIsStarted()) {
handleHobs = new HandleHobs();
}
} catch (Exception e) {
LOGGER.error("error init job", e);
}
}
public void login() throws ClassNotFoundException, SQLException, IOException, AWTException {
// String un = "a";
// String pw = "b";
FacesContext context = FacesContext.getCurrentInstance();
ExternalContext externalContext = context.getExternalContext();
HttpServletRequest request = (HttpServletRequest) externalContext.getRequest();
// Login login = new Login();
// boolean isLoggedIn = login.LoginUser(username, password);
try {
request.login(username, password);
handleHobs.startJobStoredScreenShot(username);
externalContext
.redirect(externalContext.getRequestContextPath() + "/faces/Success.xhtml?faces-redirect=true");
} catch (Exception e) {
FacesContext fc = FacesContext.getCurrentInstance();
this.error = getErrorParam(fc);
setVisible(true);
System.out.println("not equal.. " + error);
e.printStackTrace();
}
/*
*
* if (isLoggedIn) { System.out.println("equal"); externalContext
* .redirect(externalContext.getRequestContextPath() +
* "/faces/Success.xhtml?faces-redirect=true"); } else {
*
* FacesContext fc = FacesContext.getCurrentInstance(); this.error =
* getErrorParam(fc);
*
* setVisible(true); System.out.println("not equal.. " + error);
*
* }
*/
}
public String getErrorParam(FacesContext fc) {
Map<String, String> params = fc.getExternalContext().getRequestParameterMap();
return params.get("error1");
}
public logout(){
handleHobs.shutdownJobStoredScreenShot();
ExternalContext externalContext = FacesContext.getCurrentInstance().getExternalContext();
externalContext.invalidateSession();
externalContext.redirect(externalContext.getRequestContextPath() + "/faces/login.xhtml?faces-redirect=true");
}
And here is my HandleHobs.java class which is for handling job, [sorry for the wrong class name, it should be HandleJobs.java]
package com.viremp.component;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.SimpleScheduleBuilder.simpleSchedule;
import static org.quartz.TriggerBuilder.newTrigger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
import org.quartz.Trigger;
import org.quartz.impl.StdSchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HandleHobs {
private static Logger LOGGER = LoggerFactory.getLogger(HandleHobs.class);
private static JobDetail job;
private static Scheduler scheduler;
String email;
public HandleHobs(String email) {
this.email = email;
}
private static Trigger trigger;
{
try {
if (job == null && scheduler == null && trigger == null) {
LOGGER.info("initializing job");
job = (JobDetail) newJob(JobStoredScreenShot.class).withIdentity("job1", "group1").build();
trigger = newTrigger().withIdentity("trigger1", "group1").startNow()
.withSchedule(simpleSchedule().withIntervalInSeconds(5).repeatForever()).build();
scheduler = StdSchedulerFactory.getDefaultScheduler();
scheduler.scheduleJob(job, trigger);
LOGGER.info("init successsful");
}
} catch (Exception e) {
LOGGER.error("fail to init variables for job", e);
}
}
public HandleHobs() {
}
public void startJobStoredScreenShot(String email) {
try {
this.email = email;
JobStoredScreenShot jss = new JobStoredScreenShot();
jss.setEmail(email);
if (scheduler != null && !scheduler.isStarted()) {
Scheduler scheduler1 = new StdSchedulerFactory().getScheduler();
System.out.println("here..... " + email);
scheduler1.getContext().put("email", email);
System.out.println("and here..... " + email);
scheduler.start();
}
LOGGER.info("init successsful");
} catch (Exception e) {
LOGGER.error("fail to init job JobStoredScreenShot", e);
}
}
public void shutdownJobStoredScreenShot() {
try {
if (scheduler.isStarted()) {
scheduler.shutdown();
}
LOGGER.info("shutdown successsful");
} catch (Exception e) {
LOGGER.error("fail to init job JobStoredScreenShot", e);
}
}
public boolean isJobStoredScreenShotIsStarted() {
boolean isStarted = false;
try {
if (scheduler != null) {
isStarted = true;
}
} catch (Exception e) {
LOGGER.error("fail get isTarted", e);
}
return isStarted;
}
}
And here it is my JobStoredScreenShot.java class
package com.viremp.component;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.Toolkit;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.InputStream;
import java.time.LocalDate;
import javax.imageio.ImageIO;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.SchedulerContext;
import com.viremp.core.domain.Screenshot;
import com.viremp.core.domain.User;
import com.viremp.core.repository.ScreenshotRepository;
import com.viremp.core.repository.UserRepository;
import com.viremp.core.repository.impl.ScreenshotRepositoryImpl;
import com.viremp.core.repository.impl.UserRepositoryImpl;
public class JobStoredScreenShot implements Job {
String email;
User user = new User();
public void setEmail(String Email) {
this.email = Email;
System.out.println("in job stored class email is: " + email);
user.setEmail(email);
}
#Override
public void execute(JobExecutionContext arg0) throws JobExecutionException {
try {
SchedulerContext schedulerContext = arg0.getScheduler().getContext();
// Below line gets the value from context.
// Just get it and cast it in to correct type
String email = (String) schedulerContext.get("email");
System.out.println(email);
BufferedImage image = new Robot()
.createScreenCapture(new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(image, "png", baos);
baos.flush();
byte[] imageInByte = baos.toByteArray();
baos.close();
System.out.println("in execute " + user.getEmail() + ".................: " + email);
UserRepository userRepository = new UserRepositoryImpl();
user = userRepository.getUserByEmail(email);
System.out.println("USER IS: " + user.getUsername() + " id is : " + user.getId());
// byte[] buffer = (((DataBufferByte)
// (image).getRaster().getDataBuffer()).getData());
InputStream inputStream = new ByteArrayInputStream(imageInByte);
LocalDate localDate = LocalDate.now();
Screenshot screenshot = new Screenshot();
screenshot.setInput(inputStream);
screenshot.setScreenshotName(user.getUsername());
screenshot.setUser(new User());
screenshot.getUser().setId(user.getId());
screenshot.setScreenShotTime(java.sql.Date.valueOf(localDate));
System.out.println("id is " + 1l);
ScreenshotRepository screenshotRepository = new ScreenshotRepositoryImpl();
screenshotRepository.create(screenshot);
ImageIO.write(image, "png", new File("d:\\screenshot.png"));
System.out.println("screenshot taken");
} catch (Exception e) {
e.printStackTrace();
}
}
}
Please Help, Thanks
I have solved this issue by making job and scheduler null after shutting down the scheduler.
Here is the working code of HandleHobs.java
package com.viremp.component;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.SimpleScheduleBuilder.simpleSchedule;
import static org.quartz.TriggerBuilder.newTrigger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
import org.quartz.Trigger;
import org.quartz.impl.StdSchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HandleHobs {
private static Logger LOGGER = LoggerFactory.getLogger(HandleHobs.class);
private static JobDetail job;
private static Scheduler scheduler;
String email;
// SchedulerFactory schedFact = new org.quartz.impl.StdSchedulerFactory();
public HandleHobs(String email) {
this.email = email;
}
private static Trigger trigger;
{
try {
LOGGER.info("initializing job");
job = (JobDetail) newJob(JobStoredScreenShot.class).withIdentity("job1", "group1").build();
trigger = newTrigger().withIdentity("trigger1", "group1").startNow()
.withSchedule(simpleSchedule().withIntervalInSeconds(5).repeatForever()).build();
scheduler = StdSchedulerFactory.getDefaultScheduler();
scheduler.scheduleJob(job, trigger);
LOGGER.info("init successsful 1");
} catch (Exception e) {
LOGGER.error("fail to init variables for job", e);
}
}
public HandleHobs() {
}
public void startJobStoredScreenShot(String email) {
try {
System.out.println("yaar ma aa gaya hun");
this.email = email;
JobStoredScreenShot jss = new JobStoredScreenShot();
jss.setEmail(email);
if (scheduler != null && !scheduler.isStarted()) {
Scheduler scheduler1 = new StdSchedulerFactory().getScheduler();
System.out.println("here..... " + email);
scheduler1.getContext().put("email", email);
System.out.println("and here..... " + email);
scheduler.start();
}
LOGGER.info("init successsful");
} catch (Exception e) {
LOGGER.error("fail to init job JobStoredScreenShot", e);
}
}
public void shutdownJobStoredScreenShot() {
try {
if (scheduler.isStarted()) {
// JobExecutionContext context = new JobExecutionContext();
// JobKey key = context.getJobDetail().getKey();
// String jobId = key.getName();
// System.out.println("```````````````````NAME``````````````````````" + jobId);
// flag = true;
scheduler.shutdown();
scheduler = null; //i made it null here
job = null; // i made it null here
}
LOGGER.info("shutdown successsful");
} catch (Exception e) {
LOGGER.error("fail to init job JobStoredScreenShot", e);
}
}
public boolean isJobStoredScreenShotIsStarted() {
boolean isStarted = false;
try {
if (scheduler != null) {
isStarted = true;
}
} catch (Exception e) {
LOGGER.error("fail get isTarted", e);
}
return isStarted;
}
}
And this is my LoginBean.java from where I am initiating the job.
package com.viremp.beans;
import java.awt.AWTException;
import java.io.IOException;
import java.io.Serializable;
import java.sql.SQLException;
import java.util.Map;
import javax.annotation.PostConstruct;
import javax.faces.bean.ManagedBean;
import javax.faces.bean.SessionScoped;
import javax.faces.context.ExternalContext;
import javax.faces.context.FacesContext;
import javax.servlet.http.HttpServletRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.viremp.component.HandleHobs;
#ManagedBean
#SessionScoped
public class LoginBean implements Serializable {
/**
*
*/
private static final long serialVersionUID = -8650636789236091591L;
private static Logger LOGGER = LoggerFactory.getLogger(LoginBean.class);
private String username;
private String password;
private String error;
private boolean visible = false;
private HandleHobs handleHobs;
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getError() {
return error;
}
public void setError(String error) {
this.error = error;
}
public boolean isVisible() {
return visible;
}
public void setVisible(boolean visible) {
this.visible = visible;
}
/*
* public LoginBean() { try { if (handleHobs != null &&
* !handleHobs.isJobStoredScreenShotIsStarted()) { handleHobs = new
* HandleHobs(); } } catch (Exception e) { LOGGER.error("error init job", e); }
*
* }
*/
#PostConstruct
public void init() {
try {
handleHobs = new HandleHobs();
} catch (Exception e) {
LOGGER.error("error init job", e);
}
}
public void login() throws ClassNotFoundException, SQLException, IOException, AWTException {
// String un = "a";
// String pw = "b";
FacesContext context = FacesContext.getCurrentInstance();
ExternalContext externalContext = context.getExternalContext();
HttpServletRequest request = (HttpServletRequest) externalContext.getRequest();
// Login login = new Login();
// boolean isLoggedIn = login.LoginUser(username, password);
try {
request.login(username, password);
handleHobs.startJobStoredScreenShot(username);
externalContext
.redirect(externalContext.getRequestContextPath() + "/faces/Success.xhtml?faces-redirect=true");
} catch (Exception e) {
FacesContext fc = FacesContext.getCurrentInstance();
this.error = getErrorParam(fc);
setVisible(true);
System.out.println("not equal.. " + error);
e.printStackTrace();
}
/*
*
* if (isLoggedIn) { System.out.println("equal"); externalContext
* .redirect(externalContext.getRequestContextPath() +
* "/faces/Success.xhtml?faces-redirect=true"); } else {
*
* FacesContext fc = FacesContext.getCurrentInstance(); this.error =
* getErrorParam(fc);
*
* setVisible(true); System.out.println("not equal.. " + error);
*
* }
*/
}
public void logout() {
System.out.println("in logotu");
handleHobs.shutdownJobStoredScreenShot();
this.handleHobs = null; // here i also made null
ExternalContext externalContext = FacesContext.getCurrentInstance().getExternalContext();
externalContext.invalidateSession();
try {
externalContext
.redirect(externalContext.getRequestContextPath() + "/faces/Login.xhtml?faces-redirect=true");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public String getErrorParam(FacesContext fc) {
Map<String, String> params = fc.getExternalContext().getRequestParameterMap();
return params.get("error1");
}
}
Actually after shutting down the jobs did shutdown the scheduler but did not empty the job and scheduler, so on next login the scheduler and job keeps the value so that the job was not starting again, so i just made the values null on shutdown.
Related
I was given code that I have to make do something else. When I go to compile my servlet, it doesn't recognize my bean. I've deleted, recompiled, and tried from all different directories. I have no clue why this isn't working.
import BH.*;
import java.io.IOException;
import javax.servlet.*;
import javax.servlet.http.*;
import java.io.*;
import java.util.*;
import java.util.function.*;
import static java.util.Arrays.asList;
import java.text.DateFormat;
import java.util.concurrent.locks.ReentrantLock;
import java.util.Random;
public class sessionServlet extends HttpServlet {
private List<String[]> the_sessions;
private DateFormat df;
public static ReentrantLock thelock = new ReentrantLock();
public void init() throws ServletException {
the_sessions=new ArrayList<String[]>();
df=DateFormat.getDateTimeInstance(DateFormat.LONG,DateFormat.LONG);
}
protected void doGet(HttpServletRequest req, HttpServletResponse res)
throws ServletException, IOException
{
if ((!(req.getParameter("task")==null))&&(req.getParameter("task").trim().equals("deploy"))) {
PrintWriter out = res.getWriter();
out.println("<html>");
out.println("<body>");
out.println("<hr /><center><h1>sessionServlet Deployed</h1></center><hr />");
out.println("</body>");
out.println("</html>");
return;
}
Consumer <String> forwardTo =(s) ->ForwardTo(s,req,res);
boolean is_first_visit=true;
String[] this_session=new String[3];
String manager = req.getParameter("manager");
for (String [] a_session :the_sessions) {
if (a_session[0].equals(manager)) { //Found an active session
is_first_visit=false;
this_session=a_session;
break;
}
}
if ((req.getParameter("task")==null)&&(!is_first_visit)) {
the_sessions.remove(this_session);
is_first_visit=true; // just used http://hoare.cs.umsl.edu/servlet/js_test/sessionServlet
}
req.setAttribute("thesessioncount",the_sessions.size());
if (is_first_visit) {
if (the_sessions.size()==10) {
forwardTo.accept("noSessions.jsp"); //No Available Sessions
return;
}
String randomStr = getRandomString();
String[] new_session = {randomStr,df.format(new Date()),"need a name"};
the_sessions.add(new_session);
this_session=new_session;
req.setAttribute("manager",randomStr);
forwardTo.accept("startSession.jsp");
return;
}
String the_name="";
String the_pw="";
if (this_session[2].equals("need a name")) { //No name given yet
the_name=req.getParameter("whoisit");
the_pw=req.getParameter("passwd");
if ((the_name==null)||(the_name.trim().length()==0)||checkPW(the_name,the_pw)) { //checkPW returns false if correct
the_sessions.remove(this_session);
req.setAttribute("thesessioncount",the_sessions.size());
forwardTo.accept("startSession.jsp");
return; // didn't enter a name in startSession
}
}
this_session[2]=the_name.trim();
req.setAttribute("thename", this_session[2]);
if (tooLong(this_session[1],df.format(new Date()))) { //Has the session timed out?
the_sessions.remove(this_session);
forwardTo.accept("Expired.jsp");
return;
} else {
this_session[1]=df.format(new Date()); //reset the last session activity time
NotesBean thesenotes=new NotesBean();
if(req.getParameter("task").trim().equals("9")){
the_sessions.remove(this_session);
forwardTo.accept("exit.jsp");
return;
}
thelock.lock();
if (!req.getParameter("task").trim().equals("0")) { //add ACC here, also show/update posts
thesenotes.setAll(req.getParameter("java_source"),Integer.parseInt(req.getParameter("version")));
if (req.getParameter("task").trim().equals("2")) {
thesenotes.setNotes(req.getParameter("notes"),req.getParameter("java_source"),Integer.parseInt(req.getParameter("version")));
}
}
req.setAttribute("thesessioncount",the_sessions.size());
req.setAttribute("theBean",thesenotes);
req.setAttribute("manager",manager);
//req.setAttribute("theURL", "http://www.umsl.edu/~siegelj/turing.jpg");
forwardTo.accept("getNotes.jsp");
thelock.unlock();
return;
}
}//end doGet
boolean tooLong(String now,String then){
//Check amount of time that passed
return false;
}
boolean checkPW(String name,String password){
AccountBean theAccount = new AccountBean();
theAccount.setAccount(name);
if(theAccount.isPassword(password))
return false;
return true;
}
public void log(String s){
FileWriter fileWriter = null;
try {
String content =s+" at :"+new Date(System.currentTimeMillis()).toString()+"\n";
File theLogFile = new File("C:/Tomcat/webapps/js_test/session.log");
fileWriter = new FileWriter(theLogFile,true);
fileWriter.write(content);
} catch (IOException ex) {
} finally {
try {
fileWriter.close();
} catch (IOException ex) {
}
}
}
void ForwardTo(String s,HttpServletRequest req, HttpServletResponse res)
{
RequestDispatcher rd= req.getRequestDispatcher(s);
try {
rd.forward(req, res);
} catch (IOException|ServletException is) {
log(" req from "+s+" not forwarded at ");
try {
throw is;
} catch (Exception e) {
}
}
}
public void destroy()
{
log("The instance was destroyed");
}
public String getRandomString(){
byte[] randbyte=new byte[10];
Random rand = new Random(System.currentTimeMillis());
for (int idx = 0; idx <10; ++idx) {
int randomInt = rand.nextInt(26); //0<=randomInt<26
//System.out.println(randomInt);
randbyte[idx]=(byte)(randomInt+65);
}
try {
String rs=new String(randbyte, "UTF-8");
//System.out.println(rs);
return rs;
} catch (Exception e) {
//System.out.println("bad string");
return "bad";
}
}
}
AccountBean.java
package mybeans;
import BH.*;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.ArrayList;
import java.util.Arrays;
import static java.util.Arrays.asList;
import java.sql.*;
public class AccountBean implements java.io.Serializable {
static final String JDBC_DRIVER = "com.mysql.jdbc.Driver";
static final String DB_URL = "jdbc:mysql://localhost/cs4010";
static final String USER = "cs4010";
static final String PASS = "cs4010";
private String account="";
private String password="";
public AccountBean(){
}
public void setAccount(String a)
{
account = a;
}
public String getAccount()
{
return account;
}
public void setPassword(String p)
{
password = p;
}
public String getPassword(){
return password;
}
public void createAccount()
{
try {
Class.forName("com.mysql.jdbc.Driver");
Connection conn = DriverManager.getConnection(DB_URL,USER,PASS);
Statement stmt = conn.createStatement();
// System.out.println("here: "+Bytes_Hex.String2HexString(n));
String this_query="INSERT INTO ries_userpass (username, password) VALUES ("+account+","+password+");";
// System.out.println(this_query);
stmt.executeUpdate(this_query);
stmt.close();
conn.close();
} catch (Exception e) {
}
return ;
}
public boolean isPassword(String p) //make this to use
{
try {
Class.forName("com.mysql.jdbc.Driver");
Connection conn = DriverManager.getConnection(DB_URL,USER,PASS);
Statement stmt = conn.createStatement();
String this_query=" SELECT * from ries_userpass WHERE username="+account+";";
ResultSet rs = stmt.executeQuery(this_query);
while (rs.next()) {
password=Bytes_Hex.HexString2String(rs.getString("posts"));
if(password.equals(p))
{
rs.close();
stmt.close();
conn.close();
return true;
}
}
rs.close();
stmt.close();
conn.close();
}
catch (Exception e) {
return false;
}
return false;
}
}
add this statement import mybeans.*;, since you are using AccountBean but you are not importing that class package;
import BH.*;
import mybeans.*;
import java.io.IOException;
import javax.servlet.*;
import javax.servlet.http.*;
import java.io.*;
import java.util.*;
import java.util.function.*;
import static java.util.Arrays.asList;
import java.text.DateFormat;
import java.util.concurrent.locks.ReentrantLock;
import java.util.Random;
public class sessionServlet extends HttpServlet {
private List<String[]> the_sessions;
private DateFormat df;
public static ReentrantLock thelock = new ReentrantLock();
public void init() throws ServletException {
the_sessions=new ArrayList<String[]>();
df=DateFormat.getDateTimeInstance(DateFormat.LONG,DateFormat.LONG);
}
protected void doGet(HttpServletRequest req, HttpServletResponse res)
throws ServletException, IOException
{
if ((!(req.getParameter("task")==null))&&(req.getParameter("task").trim().equals("deploy"))) {
PrintWriter out = res.getWriter();
out.println("<html>");
out.println("<body>");
out.println("<hr /><center><h1>sessionServlet Deployed</h1></center><hr />");
out.println("</body>");
out.println("</html>");
return;
}
Consumer <String> forwardTo =(s) ->ForwardTo(s,req,res);
boolean is_first_visit=true;
String[] this_session=new String[3];
String manager = req.getParameter("manager");
for (String [] a_session :the_sessions) {
if (a_session[0].equals(manager)) { //Found an active session
is_first_visit=false;
this_session=a_session;
break;
}
}
if ((req.getParameter("task")==null)&&(!is_first_visit)) {
the_sessions.remove(this_session);
is_first_visit=true; // just used http://hoare.cs.umsl.edu/servlet/js_test/sessionServlet
}
req.setAttribute("thesessioncount",the_sessions.size());
if (is_first_visit) {
if (the_sessions.size()==10) {
forwardTo.accept("noSessions.jsp"); //No Available Sessions
return;
}
String randomStr = getRandomString();
String[] new_session = {randomStr,df.format(new Date()),"need a name"};
the_sessions.add(new_session);
this_session=new_session;
req.setAttribute("manager",randomStr);
forwardTo.accept("startSession.jsp");
return;
}
String the_name="";
String the_pw="";
if (this_session[2].equals("need a name")) { //No name given yet
the_name=req.getParameter("whoisit");
the_pw=req.getParameter("passwd");
if ((the_name==null)||(the_name.trim().length()==0)||checkPW(the_name,the_pw)) { //checkPW returns false if correct
the_sessions.remove(this_session);
req.setAttribute("thesessioncount",the_sessions.size());
forwardTo.accept("startSession.jsp");
return; // didn't enter a name in startSession
}
}
this_session[2]=the_name.trim();
req.setAttribute("thename", this_session[2]);
if (tooLong(this_session[1],df.format(new Date()))) { //Has the session timed out?
the_sessions.remove(this_session);
forwardTo.accept("Expired.jsp");
return;
} else {
this_session[1]=df.format(new Date()); //reset the last session activity time
NotesBean thesenotes=new NotesBean();
if(req.getParameter("task").trim().equals("9")){
the_sessions.remove(this_session);
forwardTo.accept("exit.jsp");
return;
}
thelock.lock();
if (!req.getParameter("task").trim().equals("0")) { //add ACC here, also show/update posts
thesenotes.setAll(req.getParameter("java_source"),Integer.parseInt(req.getParameter("version")));
if (req.getParameter("task").trim().equals("2")) {
thesenotes.setNotes(req.getParameter("notes"),req.getParameter("java_source"),Integer.parseInt(req.getParameter("version")));
}
}
req.setAttribute("thesessioncount",the_sessions.size());
req.setAttribute("theBean",thesenotes);
req.setAttribute("manager",manager);
//req.setAttribute("theURL", "http://www.umsl.edu/~siegelj/turing.jpg");
forwardTo.accept("getNotes.jsp");
thelock.unlock();
return;
}
}//end doGet
boolean tooLong(String now,String then){
//Check amount of time that passed
return false;
}
boolean checkPW(String name,String password){
AccountBean theAccount = new AccountBean();
theAccount.setAccount(name);
if(theAccount.isPassword(password))
return false;
return true;
}
public void log(String s){
FileWriter fileWriter = null;
try {
String content =s+" at :"+new Date(System.currentTimeMillis()).toString()+"\n";
File theLogFile = new File("C:/Tomcat/webapps/js_test/session.log");
fileWriter = new FileWriter(theLogFile,true);
fileWriter.write(content);
} catch (IOException ex) {
} finally {
try {
fileWriter.close();
} catch (IOException ex) {
}
}
}
void ForwardTo(String s,HttpServletRequest req, HttpServletResponse res)
{
RequestDispatcher rd= req.getRequestDispatcher(s);
try {
rd.forward(req, res);
} catch (IOException|ServletException is) {
log(" req from "+s+" not forwarded at ");
try {
throw is;
} catch (Exception e) {
}
}
}
public void destroy()
{
log("The instance was destroyed");
}
public String getRandomString(){
byte[] randbyte=new byte[10];
Random rand = new Random(System.currentTimeMillis());
for (int idx = 0; idx <10; ++idx) {
int randomInt = rand.nextInt(26); //0<=randomInt<26
//System.out.println(randomInt);
randbyte[idx]=(byte)(randomInt+65);
}
try {
String rs=new String(randbyte, "UTF-8");
//System.out.println(rs);
return rs;
} catch (Exception e) {
//System.out.println("bad string");
return "bad";
}
}
}
Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 7 years ago.
Improve this question
I am developing a web crawler application. When i run the program i am getting these error messages below:
i've got these errors after running the program for more that 3 hours. I tried to allocate memory by changing eclipse.ini setting to 2048 MB of ram as it was answered in this topic but still get the same errors after 3 hours or less. I should run the program for more that 2-3 days non-stopping to get analyse the results.
Can you tell me what i am missing here to get these error below ?
These are my classes:
seeds.txt
http://www.stanford.edu
http://www.archive.org
WebCrawler.java
package pkg.crawler;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.jsoup.HttpStatusException;
import org.jsoup.UnsupportedMimeTypeException;
import org.joda.time.DateTime;
public class WebCrawler {
public static Queue <LinkNodeLight> queue = new PriorityBlockingQueue <> (); // priority queue
public static final int n_threads = 5; // amount of threads
private static Set<String> processed = new LinkedHashSet <> (); // set of processed urls
private PrintWriter out; // output file
private PrintWriter err; // error file
private static Integer cntIntra = new Integer (0); // counters for intra- links in the queue
private static Integer cntInter = new Integer (0); // counters for inter- links in the queue
private static Integer dub = new Integer (0); // amount of skipped urls
public static void main(String[] args) throws Exception {
System.out.println("Running web crawler: " + new Date());
WebCrawler webCrawler = new WebCrawler();
webCrawler.createFiles();
try (Scanner in = new Scanner(new File ("seeds.txt"))) {
while (in.hasNext()) {
webCrawler.enque(new LinkNode (in.nextLine().trim()));
}
} catch (IOException e) {
e.printStackTrace();
return;
}
webCrawler.processQueue();
webCrawler.out.close();
webCrawler.err.close();
}
public void processQueue(){
/* run in threads */
Runnable r = new Runnable() {
#Override
public void run() {
/* queue may be empty but process is not finished, that's why we need to check if any links are being processed */
while (true) {
LinkNode link = deque();
if (link == null)
continue;
link.setStartTime(new DateTime());
boolean process = processLink(link);
link.setEndTime(new DateTime());
if (!process)
continue;
/* print the data to the csv file */
if (link.getStatus() != null && link.getStatus().equals(LinkNodeStatus.OK)) {
synchronized(out) {
out.println(getOutputLine(link));
out.flush();
}
} else {
synchronized(err) {
err.println(getOutputLine(link));
err.flush();
}
}
}
}
};
/* run n_threads threads which perform dequeue and process */
LinkedList <Thread> threads = new LinkedList <> ();
for (int i = 0; i < n_threads; i++) {
threads.add(new Thread(r));
threads.getLast().start();
}
for (Thread thread : threads) {
try {
thread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
/* returns true if link was actually processed */
private boolean processLink(LinkNode inputLink) {
String url = getUrlGeneralForm(inputLink);
boolean process = true;
synchronized (processed) {
if (processed.contains(url)) {
process = false;
synchronized (dub) {dub++;}
} else
processed.add(url);
}
/* start processing only if the url have not been processed yet or not being processed */
if (process) {
System.out.println("Processing url " + url);
List<LinkNodeLight> outputLinks = parseAndWieghtResults(inputLink);
for (LinkNodeLight outputLink : outputLinks) {
String getUrlGeneralForumOutput = getUrlGeneralForm(outputLink);
/* add the new link to the queue only if it has not been processed yet */
process = true;
synchronized (processed) {
if (processed.contains(getUrlGeneralForumOutput)) {
process = false;
synchronized (dub) {dub++;}
}
}
if (process) {
enque(outputLink);
}
}
return true;
}
return false;
}
void enque(LinkNodeLight link){
link.setEnqueTime(new DateTime());
/* the add method requires implicit priority */
synchronized (queue) {
if (link.interLinks)
synchronized (cntInter) {cntInter++;}
else
synchronized (cntIntra) {cntIntra++;}
//queue.add(link, 100 - (int)(link.getWeight() * 100.f));
queue.add(link);
}
}
/**
* Picks an element from the queue
* #return top element from the queue or null if the queue is empty
*/
LinkNode deque(){
/* link must be checked */
LinkNode link = null;
synchronized (queue) {
link = (LinkNode) queue.poll();
if (link != null) {
link.setDequeTime(new DateTime());
if (link.isInterLinks())
synchronized (cntInter) {cntInter--;}
else
synchronized (cntIntra) {cntIntra--;}
}
}
return link;
}
private void createFiles() {
/* create output file */
try {
out = new PrintWriter(new BufferedWriter(new FileWriter("CrawledURLS.csv", false)));
out.println(generateHeaderFile());
} catch (IOException e) {
System.err.println(e);
}
/* create error file */
try {
err = new PrintWriter(new BufferedWriter(new FileWriter("CrawledURLSERROR.csv", false)));
err.println(generateHeaderFile());
} catch (IOException e) {
System.err.println(e);
}
}
/**
* formats the string so it can be valid entry in csv file
* #param s
* #return
*/
private static String format(String s) {
// replace " by ""
String ret = s.replaceAll("\"", "\"\"");
// put string into quotes
return "\"" + ret + "\"";
}
/**
* Creates the line that needs to be written in the outputfile
* #param link
* #return
*/
public static String getOutputLine(LinkNode link){
StringBuilder builder = new StringBuilder();
builder.append(link.getParentLink()!=null ? format(link.getParentLink().getUrl()) : "");
builder.append(",");
builder.append(link.getParentLink()!=null ? link.getParentLink().getIpAdress() : "");
builder.append(",");
builder.append(link.getParentLink()!=null ? link.getParentLink().linkProcessingDuration() : "");
builder.append(",");
builder.append(format(link.getUrl()));
builder.append(",");
builder.append(link.getDomain());
builder.append(",");
builder.append(link.isInterLinks());
builder.append(",");
builder.append(Util.formatDate(link.getEnqueTime()));
builder.append(",");
builder.append(Util.formatDate(link.getDequeTime()));
builder.append(",");
builder.append(link.waitingInQueue());
builder.append(",");
builder.append(queue.size());
/* Inter and intra links in queue */
builder.append(",");
builder.append(cntIntra.toString());
builder.append(",");
builder.append(cntInter.toString());
builder.append(",");
builder.append(dub);
builder.append(",");
builder.append(new Date ());
/* URL size*/
builder.append(",");
builder.append(link.getSize());
/* HTML file
builder.append(",");
builder.append(link.getFileName());*/
/* add HTTP error */
builder.append(",");
if (link.getParseException() != null) {
if (link.getParseException() instanceof HttpStatusException)
builder.append(((HttpStatusException) link.getParseException()).getStatusCode());
if (link.getParseException() instanceof SocketTimeoutException)
builder.append("Time out");
if (link.getParseException() instanceof MalformedURLException)
builder.append("URL is not valid");
if (link.getParseException() instanceof UnsupportedMimeTypeException)
builder.append("Unsupported mime type: " + ((UnsupportedMimeTypeException)link.getParseException()).getMimeType());
}
return builder.toString();
}
/**
* generates the Header for the file
* #param link
* #return
*/
private String generateHeaderFile(){
StringBuilder builder = new StringBuilder();
builder.append("Seed URL");
builder.append(",");
builder.append("Seed IP");
builder.append(",");
builder.append("Process Duration");
builder.append(",");
builder.append("Link URL");
builder.append(",");
builder.append("Link domain");
builder.append(",");
builder.append("Link IP");
builder.append(",");
builder.append("Enque Time");
builder.append(",");
builder.append("Deque Time");
builder.append(",");
builder.append("Waiting in the Queue");
builder.append(",");
builder.append("QueueSize");
builder.append(",");
builder.append("Intra in queue");
builder.append(",");
builder.append("Inter in queue");
builder.append(",");
builder.append("Dublications skipped");
/* time was printed, but no header was */
builder.append(",");
builder.append("Time");
/* URL size*/
builder.append(",");
builder.append("Size bytes");
/* HTTP errors */
builder.append(",");
builder.append("HTTP error");
return builder.toString();
}
String getUrlGeneralForm(LinkNodeLight link){
String url = link.getUrl();
if (url.endsWith("/")){
url = url.substring(0, url.length() - 1);
}
return url;
}
private List<LinkNodeLight> parseAndWieghtResults(LinkNode inputLink) {
List<LinkNodeLight> outputLinks = HTMLParser.parse(inputLink);
if (inputLink.hasParseException()) {
return outputLinks;
} else {
return URLWeight.weight(inputLink, outputLinks);
}
}
}
HTMLParser.java
package pkg.crawler;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.math.BigInteger;
import java.util.Formatter;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import java.security.*;
import java.nio.file.Path;
import java.nio.file.Paths;
public class HTMLParser {
private static final int READ_TIMEOUT_IN_MILLISSECS = (int) TimeUnit.MILLISECONDS.convert(30, TimeUnit.SECONDS);
private static HashMap <String, Integer> filecounter = new HashMap<> ();
public static List<LinkNodeLight> parse(LinkNode inputLink){
List<LinkNodeLight> outputLinks = new LinkedList<>();
try {
inputLink.setIpAdress(IpFromUrl.getIp(inputLink.getUrl()));
String url = inputLink.getUrl();
if (inputLink.getIpAdress() != null) {
url.replace(URLWeight.getHostName(url), inputLink.getIpAdress());
}
Document parsedResults = Jsoup
.connect(url)
.timeout(READ_TIMEOUT_IN_MILLISSECS)
.get();
inputLink.setSize(parsedResults.html().length());
/* IP address moved here in order to speed up the process */
inputLink.setStatus(LinkNodeStatus.OK);
inputLink.setDomain(URLWeight.getDomainName(inputLink.getUrl()));
if (true) {
/* save the file to the html */
String filename = parsedResults.title();//digestBig.toString(16) + ".html";
if (filename.length() > 24) {
filename = filename.substring(0, 24);
}
filename = filename.replaceAll("[^\\w\\d\\s]", "").trim();
filename = filename.replaceAll("\\s+", " ");
if (!filecounter.containsKey(filename)) {
filecounter.put(filename, 1);
} else {
Integer tmp = filecounter.remove(filename);
filecounter.put(filename, tmp + 1);
}
filename = filename + "-" + (filecounter.get(filename)).toString() + ".html";
filename = Paths.get("downloads", filename).toString();
inputLink.setFileName(filename);
/* use md5 of url as file name */
try (PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(filename)))) {
out.println("<!--" + inputLink.getUrl() + "-->");
out.print(parsedResults.html());
out.flush();
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
String tag;
Elements tagElements;
List<LinkNode> result;
tag = "a[href";
tagElements = parsedResults.select(tag);
result = toLinkNodeObject(inputLink, tagElements, tag);
outputLinks.addAll(result);
tag = "area[href";
tagElements = parsedResults.select(tag);
result = toLinkNodeObject(inputLink, tagElements, tag);
outputLinks.addAll(result);
} catch (IOException e) {
inputLink.setParseException(e);
inputLink.setStatus(LinkNodeStatus.ERROR);
}
return outputLinks;
}
static List<LinkNode> toLinkNodeObject(LinkNode parentLink, Elements tagElements, String tag) {
List<LinkNode> links = new LinkedList<>();
for (Element element : tagElements) {
if(isFragmentRef(element)){
continue;
}
String absoluteRef = String.format("abs:%s", tag.contains("[") ? tag.substring(tag.indexOf("[") + 1, tag.length()) : "href");
String url = element.attr(absoluteRef);
if(url!=null && url.trim().length()>0) {
LinkNode link = new LinkNode(url);
link.setTag(element.tagName());
link.setParentLink(parentLink);
links.add(link);
}
}
return links;
}
static boolean isFragmentRef(Element element){
String href = element.attr("href");
return href!=null && (href.trim().startsWith("#") || href.startsWith("mailto:"));
}
}
Util.java
package pkg.crawler;
import java.util.Date;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
public class Util {
private static DateTimeFormatter formatter;
static {
formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss:SSS");
}
public static String linkToString(LinkNode inputLink){
return String.format("%s\t%s\t%s\t%s\t%s\t%s",
inputLink.getUrl(),
inputLink.getWeight(),
formatDate(inputLink.getEnqueTime()),
formatDate(inputLink.getDequeTime()),
differenceInMilliSeconds(inputLink.getEnqueTime(), inputLink.getDequeTime()),
inputLink.getParentLink()==null?"":inputLink.getParentLink().getUrl()
);
}
public static String linkToErrorString(LinkNode inputLink){
return String.format("%s\t%s\t%s\t%s\t%s\t%s",
inputLink.getUrl(),
inputLink.getWeight(),
formatDate(inputLink.getEnqueTime()),
formatDate(inputLink.getDequeTime()),
inputLink.getParentLink()==null?"":inputLink.getParentLink().getUrl(),
inputLink.getParseException().getMessage()
);
}
public static String formatDate(DateTime date){
return formatter.print(date);
}
public static long differenceInMilliSeconds(DateTime dequeTime, DateTime enqueTime){
return (dequeTime.getMillis()- enqueTime.getMillis());
}
public static int differenceInSeconds(Date enqueTime, Date dequeTime){
return (int)((dequeTime.getTime()/1000) - (enqueTime.getTime()/1000));
}
public static int differenceInMinutes(Date enqueTime, Date dequeTime){
return (int)((dequeTime.getTime()/60000) - (enqueTime.getTime()/60000));
}
}
URLWeight.java
package pkg.crawler;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;
public class URLWeight {
public static List<LinkNodeLight> weight(LinkNode sourceLink, List<LinkNodeLight> links) {
List<LinkNodeLight> interLinks = new LinkedList<>();
List<LinkNodeLight> intraLinks = new LinkedList<>();
for (LinkNodeLight link : links) {
if (isIntraLink(sourceLink, link)) {
intraLinks.add(link);
link.setInterLinks(false);
} else {
interLinks.add(link);
link.setInterLinks(true);
}
}
static boolean isIntraLink(LinkNodeLight sourceLink, LinkNodeLight link){
String parentDomainName = getHostName(sourceLink.getUrl());
String childDomainName = getHostName(link.getUrl());
return parentDomainName.equalsIgnoreCase(childDomainName);
}
public static String getHostName(String url) {
if(url == null){
// System.out.println("Deneme");
return "";
}
String domainName = new String(url);
int index = domainName.indexOf("://");
if (index != -1) {
domainName = domainName.substring(index + 3);
}
for (int i = 0; i < domainName.length(); i++)
if (domainName.charAt(i) == '?' || domainName.charAt(i) == '/') {
domainName = domainName.substring(0, i);
break;
}
/*if (index != -1) {
domainName = domainName.substring(0, index);
}*/
/* have to keep www in order to do replacements with IP */
//domainName = domainName.replaceFirst("^www.*?\\.", "");
return domainName;
}
public static String getDomainName(String url) {
String [] tmp= getHostName(url).split("\\.");
if (tmp.length == 0)
return "";
return tmp[tmp.length - 1];
}
}
PingTaskManager.java
package pkg.crawler;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class PingTaskManager {
private static ExecutorService executor = Executors.newFixedThreadPool(100);
public static void ping (LinkNode e) {
executor.submit(new PingTaks(e));
}
}
class PingTaks implements Runnable {
private LinkNode link;
public PingTaks( LinkNode link ) {
}
#Override
public void run() {
/* link.ping(); */
}
}
LinkNodeStatus.java
package pkg.crawler;
public enum LinkNodeStatus {
OK,
ERROR
}
LinkNodeLight.java
package pkg.crawler;
import org.joda.time.DateTime;
public class LinkNodeLight implements Comparable<LinkNodeLight> {
protected String url;
protected float weight;
protected DateTime enqueTime;
protected boolean interLinks;
public String getUrl() {
return url;
}
public float getWeight() {
return weight;
}
public void setWeight(float weight) {
this.weight = weight;
}
public DateTime getEnqueTime() {
return enqueTime;
}
public LinkNodeLight(String url) {
this.url = url;
}
public void setEnqueTime(DateTime enqueTime) {
this.enqueTime = enqueTime;
}
#Override
public int compareTo(LinkNodeLight link) {
if (this.weight < link.weight) return 1;
else if (this.weight > link.weight) return -1;
return 0;
}
}
LinkNode.java
package pkg.crawler;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.Socket;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.Date;
import org.joda.time.DateTime;
public class LinkNode extends LinkNodeLight{
public LinkNode(String url) {
super(url);
}
private String tag;
private LinkNode parentLink;
private IOException parseException = null; // initialize parse Exception with null
private float weight;
private DateTime dequeTime;
private DateTime startTime;
private DateTime endTime;
private LinkNodeStatus status;
private String ipAdress;
private int size;
private String filename;
private String domain;
public DateTime getStartTime() {
return startTime;
}
public void setStartTime(DateTime startTime) {
this.startTime = startTime;
}
public DateTime getEndTime() {
return endTime;
}
public void setEndTime(DateTime endTime) {
this.endTime = endTime;
}
public DateTime getDequeTime() {
return dequeTime;
}
public String getTag() {
return tag;
}
public LinkNode getParentLink() {
return parentLink;
}
public Exception getParseException() {
return parseException;
}
public boolean hasParseException(){
return parseException!=null;
}
public void setDequeTime(DateTime dequeTime) {
this.dequeTime = dequeTime;
}
public void setTag(String tag) {
this.tag = tag;
}
public void setParentLink(LinkNode parentLink) {
this.parentLink = parentLink;
}
public void setParseException(IOException parseException) {
this.parseException = parseException;
}
#Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LinkNode link = (LinkNode) o;
if (url != null ? !url.equals(link.url) : link.url != null) {
return false;
}
return true;
}
#Override
public int hashCode() {
return url != null ? url.hashCode() : 0;
}
public long waitingInQueue(){
return Util.differenceInMilliSeconds( dequeTime,enqueTime );
}
public long linkProcessingDuration(){
return Util.differenceInMilliSeconds( endTime,startTime );
}
#Override
public String toString() {
StringBuilder sb = new StringBuilder("LinkNode{");
sb.append("url='").append(url).append('\'');
sb.append(", score=").append(weight);
sb.append(", enqueTime=").append(enqueTime);
sb.append(", dequeTime=").append(dequeTime);
sb.append(", tag=").append(tag);
if(parentLink!=null) {
sb.append(", parentLink=").append(parentLink.getUrl());
}
sb.append('}');
return sb.toString();
}
public void setStatus(LinkNodeStatus status) {
this.status = status;
}
public LinkNodeStatus getStatus(){
if (status == null) {
status = LinkNodeStatus.ERROR;
}
return status;
}
// check server link is it exist or not
/* this method gives fake errors
public LinkNodeStatus ping () {
boolean reachable = false;
String sanitizeUrl = url.replaceFirst("^https", "http");
try {
HttpURLConnection connection = (HttpURLConnection) new URL(sanitizeUrl).openConnection();
connection.setConnectTimeout(1000);
connection.setRequestMethod("HEAD");
int responseCode = connection.getResponseCode();
System.err.println(url + " " + responseCode);
reachable = (200 <= responseCode && responseCode <= 399);
} catch (IOException exception) {
}
return reachable?LinkNodeStatus.OK: LinkNodeStatus.ERROR;
}*/
public String getIpAdress() {
return ipAdress;
}
public void setIpAdress(String ipAdress) {
this.ipAdress = ipAdress;
}
/* methods for controlling url size */
public void setSize(int size) {
this.size = size;
}
public int getSize() {
return this.size;
}
public void setFileName(String filename) {
this.filename = filename;
}
public String getFileName() {
return this.filename;
}
public String getDomain() {
return domain;
}
public void setDomain(String domain) {
this.domain = domain;
}
}
I tried to allocate memory by changing eclipse.ini setting to 2048 MB of ram as it was answered in this topic but still get the same errors after 3 hours or less.
I hate to repeat myself(*), but in eclipse.ini you set up the memory for Eclipse, which has nothing to do with the memory for your crawler.
When using command line, you need to start it via java -Xmx2G pkg.crawler.WebCrawler.
When starting from Eclipse, you need to add -Xmx2G to the run configuration ("VM arguments" rather than "Program arguments").
(*) Link to a deleted question; requires some reputation to view.
I've created users and assigned them different roles using wso2im. Using these I managed to restrict access to a .jsp file so roles seem to be working right.
The problem lies when I need to display different things to different roles within the same JSP (For instance, role AAA can do xxx and yyy, role BBB can do zzz), I'm trying to check for roles using request.isUserInRole("role") but it always returns null, both when trying from the .jsp itself and the servlet that handles authentication.
Finally managed to get it to work. Getting the roles with the servlet and storing them in a cookie. Neither safe nor pretty but does the job:
package foo;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.PrintWriter;
import javax.servlet.RequestDispatcher;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpSession;
import org.apache.axis2.transport.http.HttpTransportProperties;
import org.apache.axis2.client.Options;
import org.apache.axis2.transport.http.HTTPConstants;
import org.wso2.carbon.um.ws.api.stub.RemoteUserStoreManagerServiceStub;
/**
* Servlet implementation class LoginServlet
*/
#WebServlet("/LoginServlet")
public class LoginServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private final String basicAuthUserID = "admin";
private final String basicAuthPassword = "admin";
private final String serverUrl = "https://localhost:9444/services/";
private RemoteUserStoreManagerServiceStub stub = null;
protected void doPost(HttpServletRequest request,
HttpServletResponse response) throws ServletException, IOException {
// get request parameters for userID and password
String user = request.getParameter("user");
String pwd = request.getParameter("pwd");
try {
if(authenticate(user,pwd)){
HttpSession session = request.getSession();
session.setAttribute("user", user);
//setting session to expiry in 30 mins
session.setMaxInactiveInterval(30*60);
Cookie userName = new Cookie("user", user);
userName.setMaxAge(30*60);
String[] roles = getRoleListOfUser(user);
String rolesTodos = null;
for (String s: roles){
if (!s.equals("Internal/everyone")) {
if (rolesTodos == null){
rolesTodos = s;
} else {
//System.out.println("Rol: " + s);
rolesTodos = rolesTodos + "," + s;
}
}
}
//System.out.println("Roles: " + rolesTodos);
Cookie rolesCookie = new Cookie("roles", rolesTodos);
rolesCookie.setMaxAge(30*60);
response.addCookie(userName);
response.addCookie(rolesCookie);
response.sendRedirect("index.jsp");
}else{
RequestDispatcher rd = getServletContext().getRequestDispatcher("/login.html");
PrintWriter out= response.getWriter();
out.println("<font color=red>Either user name or password is wrong.</font>");
rd.include(request, response);
}
} catch (Exception e) {
e.printStackTrace();
}
}
private boolean authenticate(String userName, Object credential) throws Exception {
if (!(credential instanceof String)) {
throw new Exception("Unsupported type of password");
}
try {
if(stub == null) {
stub = new RemoteUserStoreManagerServiceStub(null, serverUrl
+ "RemoteUserStoreManagerService");
HttpTransportProperties.Authenticator basicAuth = new HttpTransportProperties.Authenticator();
basicAuth.setUsername(basicAuthUserID);
basicAuth.setPassword(basicAuthPassword);
basicAuth.setPreemptiveAuthentication(true);
final Options clientOptions = stub._getServiceClient().getOptions();
clientOptions.setProperty(HTTPConstants.AUTHENTICATE, basicAuth);
stub._getServiceClient().setOptions(clientOptions);
}
return stub.authenticate(userName, (String) credential);
} catch (Exception e) {
handleException(e.getMessage(), e);
}
return false;
}
private String[] handleException(String msg, Exception e) throws Exception {
System.out.println(e.getMessage() + e);
throw new Exception(msg, e);
}
public String[] getRoleListOfUser(String userName) throws Exception {
try {
return stub.getRoleListOfUser(userName);
} catch (Exception e) {
System.out.println(e.getMessage() + e);
}
return null;
}
}
I'm getting this warning from #PostConstruct annotated init method
Nis 18, 2014 2:46:10 PM com.sun.faces.vendor.WebContainerInjectionProvider getAnnotatedMethodForMethodArr
WARNING: JSF1047: Method 'public void com.revir.managed.bean.PickListBean.init() throws java.lang.Exception' marked with the 'javax.annotation.PostConstruct' annotation cannot declare any checked exceptions. This method will be ignored.
So my method is ignored, what do I have to do to fix this problem ?
package com.revir.managed.bean;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.PostConstruct;
import javax.faces.application.FacesMessage;
import javax.faces.bean.ManagedBean;
import javax.faces.bean.ViewScoped;
import javax.faces.context.FacesContext;
import org.primefaces.event.TransferEvent;
import org.primefaces.model.DualListModel;
import org.springframework.beans.factory.annotation.Autowired;
#ManagedBean(name = "pickListBean")
#ViewScoped
public class PickListBean implements Serializable {
/**
*
*/
private static final long serialVersionUID = 1L;
private DualListModel<TrvrTani> tanis;
private DualListModel<TrvrIlac> ilacs;
int tanisize = 0;
String taniadi = null;
Long taniidp = null;
public Long getTaniidp() {
return taniidp;
}
public void setTaniidp(Long taniidp) {
this.taniidp = taniidp;
}
String tanikodu = null;
#Autowired(required=false)
private TrvrTaniDAO tanidao;
public TrvrTaniDAO getTanidao() {
return tanidao;
}
public void setTanidao(TrvrTaniDAO tanidao) {
this.tanidao = tanidao;
}
List<TrvrTani> sourcetani;
List<TrvrTani> targettani;
List<TrvrIlac> sourceilac;
List<TrvrIlac> targetilac;
#PostConstruct
public void init(){
try {
sourcetani = new ArrayList<TrvrTani>();
targettani = new ArrayList<TrvrTani>();
tanidao = new TrvrTaniDAO();
List<TrvrTani> taniList = tanidao.findAll();
for (TrvrTani tani : taniList) {
Long taniid = tani.getTaniid();
sourcetani.add(new TrvrTani(taniid, tani.getTaniadi(), tani
.getTanikodu()));
}
tanis = new DualListModel<TrvrTani>(sourcetani, targettani);
sourceilac = new ArrayList<TrvrIlac>();
targetilac = new ArrayList<TrvrIlac>();
ilacdao = new TrvrIlacDAO();
List<TrvrIlac> ilacList = ilacdao.findAll();
for (TrvrIlac ilac : ilacList) {
sourceilac.add(new TrvrIlac(ilac.getIlacid(), ilac.getIlacad(),
ilac.getBarkod(), null));
}
ilacs = new DualListModel<TrvrIlac>(sourceilac, targetilac);
} catch (Exception e) {
System.out.println("Hata mesajı : " +e);
throw e;
}
}
public DualListModel<TrvrIlac> getIlacs() {
return ilacs;
}
public void setIlacs(DualListModel<TrvrIlac> ilacs) {
this.ilacs = ilacs;
}
public DualListModel<TrvrTani> getTanis() {
return tanis;
}
public void setTanis(DualListModel<TrvrTani> tanis) {
this.tanis = tanis;
}
public void onTransferTani(TransferEvent event) {
StringBuilder builder = new StringBuilder();
for (Object item : event.getItems()) {
builder.append(((TrvrTani) item).getTaniadi()).append("<br />");
targetlist(tanisize, taniadi, taniidp, tanikodu);
}
FacesMessage msgtani = new FacesMessage();
msgtani.setSeverity(FacesMessage.SEVERITY_INFO);
msgtani.setSummary("Tanı Eklendi");
msgtani.setDetail(builder.toString());
FacesContext.getCurrentInstance().addMessage(null, msgtani);
}
public void targetlist(int tanisize, String taniadi, Long taniidp,
String tanikodu) {
tanisize = tanis.getTarget().size();
System.out.println(" ************target************* : "
+ tanis.getTarget().size());
for (int h = 0; h < tanisize; h++) {
/* elemanin adi, id si ve kodu */
taniadi = tanis.getTarget().get(h).getTaniadi();
System.out.println(" ************taniadi1************* : "
+ taniadi);
taniidp = tanis.getTarget().get(h).getTaniid();
System.out.println(" ************taniid2************* : "
+ taniidp);
tanikodu = tanis.getTarget().get(h).getTanikodu();
System.out.println(" ************tanikodu3************* : "
+ tanikodu);
}
}
public void onTransferIlac(TransferEvent event) {
StringBuilder builder = new StringBuilder();
for (Object item : event.getItems()) {
builder.append(((TrvrIlac) item).getIlacad()).append("<br />");
}
FacesMessage msgilac = new FacesMessage();
msgilac.setSeverity(FacesMessage.SEVERITY_INFO);
msgilac.setSummary("İlaç Eklendi");
msgilac.setDetail(builder.toString());
FacesContext.getCurrentInstance().addMessage(null, msgilac);
}
}
Remove the throws Exception from the init method. Use try catch to prevent any exception from being thrown. Once you remove the exception declaration, the compiler will show you of any exceptions that may be thrown, so add try catch there.
how to add this
public class JavaHttpsExample
{
public static void main(String[] args)
throws Exception
{
String httpsURL = "https://localhost/send.php&json=somevalue";
URL myurl = new URL(httpsURL);
HttpsURLConnection con = (HttpsURLConnection)myurl.openConnection();
InputStream ins = con.getInputStream();
InputStreamReader isr = new InputStreamReader(ins);
BufferedReader in = new BufferedReader(isr);
String inputLine;
while ((inputLine = in.readLine()) != null)
{
System.out.println(inputLine);
}
in.close();
}
}
to
Jforex maven sdk for eclipse:
http://www.dukascopy.com/client/jforexlib/JForex-SDK.zip
and
http://www.dukascopy.com/wiki/#IClient_functionality
package singlejartest_old;
import com.dukascopy.api.system.ISystemListener;
import com.dukascopy.api.system.IClient;
import com.dukascopy.api.system.ClientFactory;
import com.dukascopy.api.*;
import java.util.HashSet;
import java.util.Scanner;
import java.util.Set;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
/**
* This small program demonstrates how to initialize Dukascopy client and start a strategy
*/
public class MainStopFromConsole {
private static final Logger LOGGER = LoggerFactory.getLogger(MainStopFromConsole.class);
private static String jnlpUrl = "https://www.dukascopy.com/client/demo/jclient/jforex.jnlp";
private static String userName = "";
private static String password = "";
public static void main(String[] args) throws Exception {
//get the instance of the IClient interface
final IClient client = ClientFactory.getDefaultInstance();
//set the listener that will receive system events
client.setSystemListener(new ISystemListener() {
private int lightReconnects = 3;
#Override
public void onStart(long processId) {
LOGGER.info("Strategy started: " + processId);
}
#Override
public void onStop(long processId) {
LOGGER.info("Strategy stopped: " + processId);
if (client.getStartedStrategies().size() == 0) {
System.exit(0);
}
}
#Override
public void onConnect() {
LOGGER.info("Connected");
lightReconnects = 3;
}
#Override
public void onDisconnect() {
LOGGER.warn("Disconnected");
if (lightReconnects > 0) {
LOGGER.error("TRY TO RECONNECT, reconnects left: " + lightReconnects);
client.reconnect();
--lightReconnects;
} else {
try {
//sleep for 10 seconds before attempting to reconnect
Thread.sleep(10000);
} catch (InterruptedException e) {
//ignore
}
try {
client.connect(jnlpUrl, userName, password);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
}
});
LOGGER.info("Connecting...");
//connect to the server using jnlp, user name and password
client.connect(jnlpUrl, userName, password);
//wait for it to connect
int i = 10; //wait max ten seconds
while (i > 0 && !client.isConnected()) {
Thread.sleep(1000);
i--;
}
if (!client.isConnected()) {
LOGGER.error("Failed to connect Dukascopy servers");
System.exit(1);
}
//subscribe to the instruments
Set<Instrument> instruments = new HashSet<Instrument>();
instruments.add(Instrument.EURUSD);
LOGGER.info("Subscribing instruments...");
client.setSubscribedInstruments(instruments);
//start the strategy
LOGGER.info("Starting strategy");
final long strategyId = client.startStrategy(new IStrategy(){
public Instrument instrument = Instrument.EURUSD;
private IConsole console;
public void onStart(IContext context) throws JFException {
console = context.getConsole();
}
public void onBar(Instrument instrument, Period period, IBar askBar, IBar bidBar) throws JFException {
if ( instrument == this.instrument){
console.getOut().println(" bar: " + period + " " + askBar);
}
}
public void onTick(Instrument instrument, ITick tick) throws JFException { }
public void onMessage(IMessage message) throws JFException { }
public void onAccount(IAccount account) throws JFException { }
public void onStop() throws JFException { }
});
//now it's running
//every second check if "stop" had been typed in the console - if so - then stop the strategy
Thread thread = new Thread(new Runnable() {
#Override
public void run() {
Scanner s = new Scanner(System.in);
while(true){
while(s.hasNext()){
String str = s.next();
if(str.equalsIgnoreCase("stop")){
System.out.println("Strategy stop by console command.");
client.stopStrategy(strategyId);
}
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
});
thread.start();
}
}
i need send some values(account balance or open positions) to WWW serwer from strategy
Thanks
zix
package master;
import com.dukascopy.api.system.ISystemListener;
import com.dukascopy.api.system.IClient;
import com.dukascopy.api.system.ClientFactory;
import com.dukascopy.api.*;
import java.awt.Color;
import java.awt.List;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.Writer;
import java.lang.reflect.Array;
import java.net.URL;
import java.nio.file.Files;
import java.text.DateFormat;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Scanner;
import java.util.Set;
import javax.net.ssl.HttpsURLConnection;
import javax.swing.JFrame;
import org.slf4j.LoggerFactory;
import org.slf4j.Logger;
import singlejartest.JavaHttpsExample;
public class MainStopFromConsole {
private static final Logger LOGGER = LoggerFactory.getLogger(MainStopFromConsole.class);
private static String jnlpUrl = "https://www.dukascopy.com/client/demo/jclient/jforex.jnlp";
private static String userName = "DEMO2PRFwj";
private static String password = "PRFwj";
//static String[] columns = {"Open Time", "Id", "Label", "Comment", "Instrument", "Side", "Amount", "Original Amount", "Open Price", "Stop Loss", "Take Profit", "Profit (Pips)", "Profit Currency", "Profit in USD", "Commission", "Commission USD"};
static String[] columns = {"Id", "Instrument", "Side", "Amount", "Open Price", "Stop Loss", "Take Profit"};
//static String[][] data = {{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16"},
static String[][] data = new String[50][11];
//static String[][] data = null;
//static JFrame jf = new JFrame();
public static void main(String[] args) throws Exception {
//get the instance of the IClient interface
final IClient client = ClientFactory.getDefaultInstance();
//set the listener that will receive system events
client.setSystemListener(new ISystemListener() {
private int lightReconnects = 3;
public void onStart(long procid) {
//IConsole console = context.getConsole();
LOGGER.info("Strategy started: ");
}
public void onStop(long processId) {
LOGGER.info("Strategy stopped: " + processId);
if (client.getStartedStrategies().size() == 0) {
System.exit(0);
}
}
#Override
public void onConnect() {
LOGGER.info("Connected");
lightReconnects = 3;
}
#Override
public void onDisconnect() {
LOGGER.warn("Disconnected");
if (lightReconnects > 0) {
LOGGER.error("TRY TO RECONNECT, reconnects left: " + lightReconnects);
client.reconnect();
--lightReconnects;
} else {
try {
//sleep for 10 seconds before attempting to reconnect
Thread.sleep(10000);
} catch (InterruptedException e) {
//ignore
}
try {
client.connect(jnlpUrl, userName, password);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
}
});
LOGGER.info("Connecting...");
//connect to the server using jnlp, user name and password
client.connect(jnlpUrl, userName, password);
//wait for it to connect
int i = 10; //wait max ten seconds
while (i > 0 && !client.isConnected()) {
Thread.sleep(1000);
i--;
}
if (!client.isConnected()) {
LOGGER.error("Failed to connect Dukascopy servers");
System.exit(1);
}
//subscribe to the instruments
Set<Instrument> instruments = new HashSet<Instrument>();
instruments.add(Instrument.EURUSD);
LOGGER.info("Subscribing instruments...");
client.setSubscribedInstruments(instruments);
//start the strategy
LOGGER.info("Starting strategy");
final long strategyId = client.startStrategy(new IStrategy(){
public Instrument instrument = Instrument.EURUSD;
private IConsole console;
private IEngine engine;
public void onStart(IContext context) throws JFException {
console = context.getConsole();
engine = context.getEngine();
}
public void onBar(Instrument instrument, Period period, IBar askBar, IBar bidBar) throws JFException {
if ( instrument == this.instrument){
//console.getOut().println(" bar: " + period + " " + askBar);
}
}
public void onTick(Instrument instrument, ITick tick) throws JFException {
try {
int xx = 0;
//data[0][1] = "1";
for(IOrder o : engine.getOrders()){
if(o.getProfitLossInUSD() != 987654231){
console.getOut().println("Order: " + o.getInstrument() + " " + o.getProfitLossInPips() + " " + o.getOrderCommand());
// Copy orders to array
String userHash = "1";
String positionOpenTime = "" + o.getFillTime();
String positionId = "" + o.getId();
String positionInstrument = "" + o.getInstrument();
String positionIsbuy = "" + o.getOrderCommand().isLong();
String positionVolume = "" + o.getAmount();
String positionOpen = "" + o.getOpenPrice();
String positionSl = "" + o.getStopLossPrice();
String positionTp = "" + o.getTakeProfitPrice();
String positionComment = "" + o.getComment();
data[xx][0] = userHash;
data[xx][1] = positionOpenTime;
data[xx][2] = positionId;
data[xx][3] = positionInstrument;
data[xx][4] = positionIsbuy;
data[xx][5] = positionVolume;
data[xx][6] = positionOpen;
data[xx][7] = positionSl;
data[xx][8] = positionTp;
data[xx][9] = positionComment;
xx++;
}
}
xx=0;
/*
int i = 0;
int j = 0;
String string = "";
for(j = 0; j<10; j++){
for(i = 0; i<10; i++){
string += data[j][i] + "#";
}
string += "$$";
}
}
console.getOut().println("=====================================================================================" );
console.getOut().println(string);
*/
console.getOut().println("=====================================================================================" );
String txt ="";
txt = Arrays.deepToString(data);
txt = txt.replaceAll(",", "aa");
txt = txt.replaceAll(" ", "zz");
System.out.println(txt);
console.getOut().println("=====================================================================================" );
//=================================================================== send get == need commerciall ssl like startssl.com and serveralias and servername in virtualhost
URL hp = new URL("https://breakermind.com/index.php?line="+txt);
HttpsURLConnection hpCon = (HttpsURLConnection) hp.openConnection();
boolean isProxy = hpCon.usingProxy();
System.out.println("is using proxy " + isProxy);
InputStream obj = hpCon.getInputStream();
BufferedReader br = new BufferedReader(new InputStreamReader(obj));
String s;
while ((s = br.readLine()) != null) {
s = s.replaceAll("zz", " ");
s = s.replaceAll("aa", ",");
System.out.println(">>>" + s);
}
//===================================================================== end
} catch (Exception e) {
console.getErr().println(e.getMessage());
e.printStackTrace(console.getErr());
// context.stop();
}
console.getOut().println("=====================================================================================" );
}
public void onMessage(IMessage message) throws JFException { }
public void onAccount(IAccount account) throws JFException { }
public void onStop() throws JFException { }
});
//now it's running
//every second check if "stop" had been typed in the console - if so - then stop the strategy
Thread thread = new Thread(new Runnable() {
#Override
public void run() {
Scanner s = new Scanner(System.in);
while(true){
while(s.hasNext()){
String str = s.next();
if(str.equalsIgnoreCase("stop")){
System.out.println("Strategy stop by console command.");
client.stopStrategy(strategyId);
}
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
});
thread.start();
}
}