I am using SequenceR to parse my Java code files. It uses Spoon library of Java which is a popular code analysis library. What I need is, I will give a buggy line number to the analyzer, it will detect the line and find out the method or class of it and it will remove all the comments on that class or method.
I have tried in several ways to achieve that. But can't really get the desired output.
This is my input Java file.
public class CWE23_Relative_Path_Traversal__connect_tcp_01 extends AbstractTestCase
{
/* uses badsource and badsink */
public void bad() throws Throwable
{
String data;
data = ""; /* Initialize data */
/* Read data using an outbound tcp connection */
{
Socket socket = null;
BufferedReader readerBuffered = null;
InputStreamReader readerInputStream = null;
try
{
/* Read data using an outbound tcp connection */
socket = new Socket("host.example.org", 39544);
/* read input from socket */
readerInputStream = new InputStreamReader(socket.getInputStream(), "UTF-8");
readerBuffered = new BufferedReader(readerInputStream);
/* POTENTIAL FLAW: Read data using an outbound tcp connection */
data = readerBuffered.readLine();
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error with stream reading", exceptIO);
}
finally
{
/* clean up stream reading objects */
try
{
if (readerBuffered != null)
{
readerBuffered.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing BufferedReader", exceptIO);
}
try
{
if (readerInputStream != null)
{
readerInputStream.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing InputStreamReader", exceptIO);
}
/* clean up socket objects */
try
{
if (socket != null)
{
socket.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing Socket", exceptIO);
}
}
}
String root;
if(System.getProperty("os.name").toLowerCase().indexOf("win") >= 0)
{
/* running on Windows */
root = "C:\\uploads\\";
}
else
{
/* running on non-Windows */
root = "/home/user/uploads/";
}
if (data != null)
{
/* POTENTIAL FLAW: no validation of concatenated value */
File file = new File(root + data);
FileInputStream streamFileInputSink = null;
InputStreamReader readerInputStreamSink = null;
BufferedReader readerBufferdSink = null;
if (file.exists() && file.isFile())
{
try
{
streamFileInputSink = new FileInputStream(file);
readerInputStreamSink = new InputStreamReader(streamFileInputSink, "UTF-8");
readerBufferdSink = new BufferedReader(readerInputStreamSink);
IO.writeLine(readerBufferdSink.readLine());
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error with stream reading", exceptIO);
}
finally
{
/* Close stream reading objects */
try
{
if (readerBufferdSink != null)
{
readerBufferdSink.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing BufferedReader", exceptIO);
}
try
{
if (readerInputStreamSink != null)
{
readerInputStreamSink.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing InputStreamReader", exceptIO);
}
try
{
if (streamFileInputSink != null)
{
streamFileInputSink.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing FileInputStream", exceptIO);
}
}
}
}
}
public void good() throws Throwable
{
goodG2B();
}
/* goodG2B() - uses goodsource and badsink */
private void goodG2B() throws Throwable
{
String data;
/* FIX: Use a hardcoded string */
data = "foo";
String root;
if(System.getProperty("os.name").toLowerCase().indexOf("win") >= 0)
{
/* running on Windows */
root = "C:\\uploads\\";
}
else
{
/* running on non-Windows */
root = "/home/user/uploads/";
}
if (data != null)
{
/* POTENTIAL FLAW: no validation of concatenated value */
File file = new File(root + data);
FileInputStream streamFileInputSink = null;
InputStreamReader readerInputStreamSink = null;
BufferedReader readerBufferdSink = null;
if (file.exists() && file.isFile())
{
try
{
streamFileInputSink = new FileInputStream(file);
readerInputStreamSink = new InputStreamReader(streamFileInputSink, "UTF-8");
readerBufferdSink = new BufferedReader(readerInputStreamSink);
IO.writeLine(readerBufferdSink.readLine());
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error with stream reading", exceptIO);
}
finally
{
/* Close stream reading objects */
try
{
if (readerBufferdSink != null)
{
readerBufferdSink.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing BufferedReader", exceptIO);
}
try
{
if (readerInputStreamSink != null)
{
readerInputStreamSink.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing InputStreamReader", exceptIO);
}
try
{
if (streamFileInputSink != null)
{
streamFileInputSink.close();
}
}
catch (IOException exceptIO)
{
IO.logger.log(Level.WARNING, "Error closing FileInputStream", exceptIO);
}
}
}
}
}
/* Below is the main(). It is only used when building this testcase on
* its own for testing or for building a binary to use in testing binary
* analysis tools. It is not used when compiling all the testcases as one
* application, which is how source code analysis tools are tested.
*/
public static void main(String[] args) throws ClassNotFoundException,
InstantiationException, IllegalAccessException
{
mainFromParent(args);
}
}
What I have done in my analyzer code is this -
public static void generateAbstraction(File buggy_file, int buggy_line, File working_dir) {
Launcher launcher = new Launcher();
launcher.getEnvironment().setAutoImports(true);
launcher.getEnvironment().setNoClasspath(true);
launcher.getEnvironment().setCommentEnabled(true);
launcher.addInputResource(buggy_file.toString());
try {
launcher.buildModel();
} catch (Exception e) {
}
CtModel model = launcher.getModel();
CtMethod topLevelmethod = null;
CtElement buggy_ctElement = null;
CtElement tmp_ctElement = null;
CtPath buggy_ctElement_ctPath = null;
// This is the main part
for (CtType<?> ctType : model.getAllTypes()) {
for (Iterator<CtElement> desIter = ctType.descendantIterator(); desIter.hasNext(); ) {
tmp_ctElement = desIter.next();
try {
// Main problem resides here
if (tmp_ctElement.getPosition().getLine() == buggy_line && !(tmp_ctElement instanceof CtComment)) {
buggy_ctElement = tmp_ctElement;
buggy_ctElement_ctPath = tmp_ctElement.getPath();
topLevelmethod = getTopLevelMethod(buggy_ctElement);
List<CtComment> comments = topLevelmethod.getElements(
new TypeFilter<CtComment>(CtComment.class)
);
for(CtComment c: comments){
topLevelmethod.removeComment(c);
}
break;
}
} catch (java.lang.UnsupportedOperationException e) {
continue;
}
}
// ......
}
}
public static CtMethod getTopLevelMethod(CtElement ctElement) {
CtMethod topLevelMethod = null;
topLevelMethod = ctElement.getParent(CtMethod.class);
while (topLevelMethod != null && topLevelMethod.getParent(CtMethod.class) != null) {
System.out.println();
topLevelMethod = topLevelMethod.getParent(CtMethod.class);
}
return topLevelMethod;
}
The output goes like this -
public class CWE23_Relative_Path_Traversal__connect_tcp_01 extends AbstractTestCase {
public void bad() throws Throwable {
String data;
data = "";/* Initialize data */
/* Read data using an outbound tcp connection */
{
Socket socket = null;
BufferedReader readerBuffered = null;
InputStreamReader readerInputStream = null;
try {
/* Read data using an outbound tcp connection */
socket = new Socket("host.example.org", 39544);
/* read input from socket */
readerInputStream = new InputStreamReader(socket.getInputStream(), "UTF-8");
readerBuffered = new BufferedReader(readerInputStream);
/* POTENTIAL FLAW: Read data using an outbound tcp connection */
data = readerBuffered.readLine();
} catch (IOException exceptIO) {
logger.log(Level.WARNING, "Error with stream reading", exceptIO);
} finally {
/* clean up stream reading objects */
try {
if (readerBuffered != null) {
readerBuffered.close();
}
} catch (IOException exceptIO) {
logger.log(Level.WARNING, "Error closing BufferedReader", exceptIO);
}
try {
if (readerInputStream != null) {
readerInputStream.close();
}
} catch (IOException exceptIO) {
logger.log(Level.WARNING, "Error closing InputStreamReader", exceptIO);
}
/* clean up socket objects */
try {
if (socket != null) {
socket.close();
}
} catch (IOException exceptIO) {
logger.log(Level.WARNING, "Error closing Socket", exceptIO);
}
}
}
String root;
if ((System.getProperty("os.name").toLowerCase().indexOf("win")) >= 0) {
/* running on Windows */
root = "C:\\uploads\\";
}else {
/* running on non-Windows */
root = "/home/user/uploads/";
}
if (data != null) {
/* POTENTIAL FLAW: no validation of concatenated value */
File file = new File((root + data));
FileInputStream streamFileInputSink = null;
InputStreamReader readerInputStreamSink = null;
BufferedReader readerBufferdSink = null;
if ((file.exists()) && (file.isFile())) {
try {
streamFileInputSink = new FileInputStream(file);
readerInputStreamSink = new InputStreamReader(streamFileInputSink, "UTF-8");
readerBufferdSink = new BufferedReader(readerInputStreamSink);
IO.writeLine(readerBufferdSink.readLine());
} catch (IOException exceptIO) {
logger.log(Level.WARNING, "Error with stream reading", exceptIO);
} finally {
/* Close stream reading objects */
try {
if (readerBufferdSink != null) {
readerBufferdSink.close();
}
} catch (IOException exceptIO) {
logger.log(Level.WARNING, "Error closing BufferedReader", exceptIO);
}
try {
if (readerInputStreamSink != null) {
readerInputStreamSink.close();
}
} catch (IOException exceptIO) {
logger.log(Level.WARNING, "Error closing InputStreamReader", exceptIO);
}
try {
if (streamFileInputSink != null) {
streamFileInputSink.close();
}
} catch (IOException exceptIO) {
logger.log(Level.WARNING, "Error closing FileInputStream", exceptIO);
}
}
}
}
}
public void good() throws Throwable {
}
/* goodG2B() - uses goodsource and badsink */
private void goodG2B() throws Throwable {
}
/* Below is the main(). It is only used when building this testcase on
its own for testing or for building a binary to use in testing binary
analysis tools. It is not used when compiling all the testcases as one
application, which is how source code analysis tools are tested.
*/
public static void main(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
}
}
If you look at it closely, you can see that only one comment was removed. That is -
/* uses badsource and badsink */
Other comments are intact. But I need to remove all the comments in a class or method. What am I doing wrong? I am totally new to spoon. Any help would be appreciated.
I accidentally found the answer a few minutes ago. I am not gonna delete this answer because someone might find this useful.
To remove all the comments of the file, you just need to turn a flag off.
This line did the magic -
launcher.getEnvironment().setCommentEnabled(false);
What it does is, it sets the setCommentEnabled value false and comments are ignored totally. So, you don't need to worry about them at all. This simple solution took me 6 hours to figure out.
Related
I have to create a simple rotating proxy application where 100 requests get evenly distributed to 10 devices. I've got the following structure:
WebServer with a Java-SocketServer running. All Android devices are connected to this Socket-Server to be able to know which devices are currently online and for determining which device should be used for the next request.
10 Android devices in different networks. They are connected to the Socket Server and are waiting for requests that should be forwarded to the remote address and then sent back to the SocketServer.
In easy words: I basically have to create an application similar like Honeygain, Peer2Profit or IPRoyal Pawns so that I can later do requests like this:
//Use "-x" to set Proxy-IP and Proxy-Port
curl -x ANDROID_DEVICE_IP:PORT -L https://www.google.com
I managed to have an always running proxy service in an Android application. It basically looks like this and just forwards HTTP-Requests from Port 1440 to the desired remote address and then sends the response back to the original client. The Proxy basically works fine.
public class ProxyServerThread extends Thread {
public static void main(String[] args) {
(new ProxyServerThread()).run();
}
public ProxyServerThread() {
super("Server Thread");
}
#Override
public void run() {
try (ServerSocket serverSocket = new ServerSocket(1440)) {
Socket socket;
try {
while ((socket = serverSocket.accept()) != null) {
(new Handler(socket)).start();
}
} catch (IOException e) {
e.printStackTrace(); // TODO: implement catch
}
} catch (IOException e) {
e.printStackTrace(); // TODO: implement catch
return;
}
}
public static class Handler extends Thread {
public static final Pattern CONNECT_PATTERN = Pattern.compile("CONNECT (.+):(.+) HTTP/(1\\.[01])", Pattern.CASE_INSENSITIVE);
private final Socket clientSocket;
private boolean previousWasR = false;
public Handler(Socket clientSocket) {
this.clientSocket = clientSocket;
}
#Override
public void run() {
try {
String request = readLine(clientSocket);
System.out.println(request);
Matcher matcher = CONNECT_PATTERN.matcher(request);
if (matcher.matches()) {
String header;
do {
header = readLine(clientSocket);
} while (!"".equals(header));
OutputStreamWriter outputStreamWriter = new OutputStreamWriter(clientSocket.getOutputStream(), "ISO-8859-1");
final Socket forwardSocket;
try {
forwardSocket = new Socket(matcher.group(1), Integer.parseInt(matcher.group(2)));
System.out.println(forwardSocket);
} catch (IOException | NumberFormatException e) {
e.printStackTrace(); // TODO: implement catch
outputStreamWriter.write("HTTP/" + matcher.group(3) + " 502 Bad Gateway\r\n");
outputStreamWriter.write("Proxy-agent: Simple/0.1\r\n");
outputStreamWriter.write("\r\n");
outputStreamWriter.flush();
return;
}
try {
outputStreamWriter.write("HTTP/" + matcher.group(3) + " 200 Connection established\r\n");
outputStreamWriter.write("Proxy-agent: Simple/0.1\r\n");
outputStreamWriter.write("\r\n");
outputStreamWriter.flush();
Thread remoteToClient = new Thread() {
#Override
public void run() {
forwardData(forwardSocket, clientSocket);
}
};
remoteToClient.start();
try {
if (previousWasR) {
int read = clientSocket.getInputStream().read();
if (read != -1) {
if (read != '\n') {
forwardSocket.getOutputStream().write(read);
}
forwardData(clientSocket, forwardSocket);
} else {
if (!forwardSocket.isOutputShutdown()) {
forwardSocket.shutdownOutput();
}
if (!clientSocket.isInputShutdown()) {
clientSocket.shutdownInput();
}
}
} else {
forwardData(clientSocket, forwardSocket);
}
} finally {
try {
remoteToClient.join();
} catch (InterruptedException e) {
e.printStackTrace(); // TODO: implement catch
}
}
} finally {
forwardSocket.close();
}
}
} catch (IOException e) {
e.printStackTrace(); // TODO: implement catch
} finally {
try {
clientSocket.close();
} catch (IOException e) {
e.printStackTrace(); // TODO: implement catch
}
}
}
private static void forwardData(Socket inputSocket, Socket outputSocket) {
try {
InputStream inputStream = inputSocket.getInputStream();
try {
OutputStream outputStream = outputSocket.getOutputStream();
try {
byte[] buffer = new byte[4096];
int read;
do {
read = inputStream.read(buffer);
if (read > 0) {
outputStream.write(buffer, 0, read);
if (inputStream.available() < 1) {
outputStream.flush();
}
}
} while (read >= 0);
} finally {
if (!outputSocket.isOutputShutdown()) {
outputSocket.shutdownOutput();
}
}
} finally {
if (!inputSocket.isInputShutdown()) {
inputSocket.shutdownInput();
}
}
} catch (IOException e) {
e.printStackTrace(); // TODO: implement catch
}
}
private String readLine(Socket socket) throws IOException {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
int next;
readerLoop:
while ((next = socket.getInputStream().read()) != -1) {
if (previousWasR && next == '\n') {
previousWasR = false;
continue;
}
previousWasR = false;
switch (next) {
case '\r':
previousWasR = true;
break readerLoop;
case '\n':
break readerLoop;
default:
byteArrayOutputStream.write(next);
break;
}
}
return byteArrayOutputStream.toString("ISO-8859-1");
}
}
}
Here comes the Problem:
Everything works fine but only on the local network. I cannot manage to get this to work without port forwarding. Since all devices are on their mobile cellular data I need a way to be able to connect to the device anyway.
How do the mentioned apps manage to connect to the devices?
my code has to read in two different Object Types (Bestellung, AKunde) through a ObjectOutputStream and save it in a csv file, which works.
But when i try to read them from the file it doesn't work.
Here is the code:
OutputStream:
LinkedList<Bestellung> bestellListe = verwaltungBestell.getBestellListe();
try {
fileOutputStream = new FileOutputStream(file);
outputStream = new ObjectOutputStream(fileOutputStream);
for (AKunde kunde : kundenliste) {
outputStream.writeObject(kunde);
}
for (Bestellung bestellung : bestellListe) {
outputStream.writeObject(bestellung);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (fileOutputStream != null) {
fileOutputStream.close();
}
if (outputStream != null) {
outputStream.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
InputStream:
ArrayList<AKunde> kundenImport = new ArrayList<AKunde>();
ArrayList<Bestellung> bestellungenImport = new ArrayList<Bestellung>();
boolean cont = true;
try {
ObjectInputStream objectStream = new ObjectInputStream(new FileInputStream(directorie));
while (cont) {
AKunde kunde = null;
try {
kunde = (AKunde) objectStream.readObject();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
if (kunde != null) {
kundenImport.add(kunde);
} else {
cont = false;
}
}
while (cont) {
Bestellung bestellung = null;
try {
bestellung = (Bestellung) objectStream.readObject();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
if (bestellung != null) {
bestellungenImport.add(bestellung);
} else {
cont = false;
}
}
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
}
But it won't read the "Bestellungen" and won't save them into "bestellungenImport".
Anyone has a solution???
Your code never reaches the Bestellung reader part.
You have a false assumption that kunde =(AKunde)objectStream.readObject(); returns null.
Instead, it throws exception.
Oneway you can do is cast it like #luk2302.
Another way is to add a object count when writing your object stream:
outputStream.writeInt(kundenliste.size());
for (AKunde kunde : kundenliste) {
outputStream.writeObject(kunde);
}
outputStream.writeInt(bestellListe.size());
for (Bestellung bestellung : bestellListe) {
outputStream.writeObject(bestellung);
}
Then replace your while(cont) loop with a for each loop:
int kundeCount = objectStream.readInt();
for (int i = 0; i < kundeCount; i++) {
// Read and import kunde
}
You need to change your logic for reading objects. There are two main issues:
you never reset cont so the second while loop will never do anything
even if you did that you would always skip the first Bestellung since it was already read when the second loop is reached
I would propose something along the lines of:
Object object = objectStream.readObject();
if (object instanceof AKunde) {
kundenImport.add((AKunde) object);
} else if (object instanceof Bestellung) {
bestellungenImport.add((Bestellung) object);
} else {
// something else was read
}
You simply need to loop over this code and add proper error handling where needed.
I would suggest, you change the way you write your objects to ObjectOutputStream in the first place:
Directly write the kundenListe and bestellListe objects, so you dont't have to worry about types or number of elements when reading the objects again. Your stream of object then always contains two objects, the two lists.
// use try-with-resources if you're on Java 7 or newer
try (ObjectOutputStream outputStream = new ObjectOutputStream(new FileOutputStream(file))) {
// write the complete list of objects
outputStream.writeObject(kundenliste);
outputStream.writeObject(bestellListe);
} catch (IOException e) {
e.printStackTrace(); //TODO proper exception handling
}
Then you could read it just like that:
ArrayList<AKunde> kundenImport = new ArrayList<>();
ArrayList<Bestellung> bestellungenImport = new ArrayList<>();
//again try-with-resources
try (ObjectInputStream inputStream = new ObjectInputStream(new FileInputStream(file))) {
kundenImport.addAll((List) inputStream.readObject());
bestellungenImport.addAll((List) inputStream.readObject());
} catch (IOException | ClassNotFoundException e) { //multi-catch, if Java 7 or newer
e.printStackTrace(); //TODO proper exception handling
}
Further reads:
The try-with-resources Statement
Catching Multiple Exception Types (...)
I am trying to read a line in a file, in the spout, and then send it to the bolts but i keep getting a stream closed error. Do I close it wrong or what is the problem here?
public class InputSpout extends BaseRichSpout {
private SpoutOutputCollector collector;
public void
declareOutputFields( OutputFieldsDeclarer declarer) {
declarer.declare( new Fields("logfile"));
}
private FileReader fileReader;
private boolean completed = false;
private TopologyContext context;
#Override
public void open( Map config, TopologyContext context, SpoutOutputCollector collector) {
try {
this.context = context;
this.fileReader = new FileReader(("logfile.txt").toString());
} catch (FileNotFoundException e) {
throw new RuntimeException("Error reading file "
+ ("logfile"));
}
this.collector = collector;
}
public void nextTuple() {
if (completed) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
}
String str;
BufferedReader reader = new BufferedReader(fileReader);
str = null;
try {
str = reader.readLine();
} catch (IOException e1) {
e1.printStackTrace();
}
try {
while (str != null) {
this.collector.emit(new Values(str));
str = reader.readLine();
}
} catch (Exception e) {
throw new RuntimeException("Error reading typle", e);
} finally {
completed = true;
}
try {
reader.close();
fileReader.close();
} catch (IOException e) {
e.printStackTrace();
}
This is the error I am getting:
java.io.IOException: Stream closed
at sun.nio.cs.StreamDecoder.ensureOpen(StreamDecoder.java:46)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:147)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.fill(BufferedReader.java:154)
at java.io.BufferedReader.readLine(BufferedReader.java:317)
at java.io.BufferedReader.readLine(BufferedReader.java:382)
at myStorm.InputSpout.nextTuple(InputSpout.java:52)
at backtype.storm.daemon.executor$fn__4654$fn__4669$fn__4698.invoke(executor.clj:565)
at backtype.storm.util$async_loop$fn__458.invoke(util.clj:463)
at clojure.lang.AFn.run(AFn.java:24)
at java.lang.Thread.run(Thread.java:745)
You're closing the InputStream used by fileReader in nextTuple making it unavailable for subsequent calls. There's no need to close this Reader - just close a single BufferedReader when all data has been read.
I am programatically trying to connect to ssh running remotely. I am running a tomcat server instance. Whenever i need, from the code, i create a session, connect and execute a few commands that are needed within a try block and then close off the connection that was created as part of the finally block at all the places. Things work well and fine, but at some cases when i execute a w or netstat command on the ssh server, I see a few connections that are idle for more than a few hours and the ip address of those connections shows the connection to be from my application, but my java heap dump does not show any instance of my class in the memory, but i see ganymed related class instances in the heap.
I am using ganymed-ssh-260 library to connect to my server.
Is this something that someone has already seen?
Attaching the code snippet that connectes the ssh to the server
public class SSHExecutor{
private OutputStream stdin;
private InputStream stdout;
private InputStream stderr;
private Session sess;
private Connection conn;
public void createConnection(String hostname, int port, String userName, String password) throws Exception {
try {
conn = new Connection(hostname, port);
final boolean isAuthenticated = publicKeyAccess(hostname, userName, password);
if (!isAuthenticated) {
throw new IOException("Authentication failed.");
}
sess = conn.openSession();
final int xWidth = 90;
final int yWidth = 80;
sess.requestPTY("dumb", xWidth, yWidth, 0, 0, null);
sess.startShell();
stdin = sess.getStdin();
stdout = sess.getStdout();
stderr = sess.getStderr();
isConnectionActive = true;
final String response = getResponse();
if (response != null && response.toLowerCase().contains(ObjectConstants.CURRENTLY_NOT_AVAILABLE)) {
throw new IOException("Account is currently not available.");
}
} catch (Exception e) {
log.error("Problem in CreateConnection", e);
isConnectionActive = false;
throw e;
}
}
public String getResponse() {
final StringBuffer responseData = new StringBuffer();
try {
final int byteValue = 8192;
final byte[] buffer = new byte[byteValue];
try {
while (true) {
if ((stdout.available() == 0) && (stderr.available() == 0)) {
int conditions = 1;
if (promptString != null && promptString.length() > 0) {
final int fiveThousand = 5000;
conditions = sess.waitForCondition(ChannelCondition.STDOUT_DATA
| ChannelCondition.STDERR_DATA | ChannelCondition.EOF, fiveThousand);
} else {
conditions = sess.waitForCondition(ChannelCondition.STDOUT_DATA
| ChannelCondition.STDERR_DATA | ChannelCondition.EOF,
ObjectConstants.THOUSAND_FIVE_HUNDRED);
}
if ((conditions & ChannelCondition.TIMEOUT) != 0) {
break;
}
if ((conditions & ChannelCondition.EOF) != 0) {
if ((conditions & (ChannelCondition.STDOUT_DATA | ChannelCondition.STDERR_DATA)) == 0) {
break;
}
}
}
while (stdout.available() > 0) {
final int len = stdout.read(buffer);
if (len > 0) {
responseData.append(new String(buffer, 0, len));
}
}
while (stderr.available() > 0) {
final int len = stderr.read(buffer);
if (len > 0) {
responseData.append(new String(buffer, 0, len));
}
}
if (promptString != null && promptString.length() > 0) {
if (responseData.indexOf(promptString) != -1) {
break;
}
}
}
} catch (Exception e) {
log.error("Read Error :", e);
}
} catch (Exception e) {
log.error("getResponse Error ", e);
}
return responseData.toString();
}
public String executeCommand(String command) throws IOException {
String response = null;
if (isConnectionActive && stdin != null) {
try {
stdin.write(command.getBytes());
stdin.flush();
response = getResponse();
} catch (IOException ie) {
throw ie;
} catch (Exception e) {
log.error("Exception in executeCommandForPage()", e);
response = e.getMessage();
}
} else {
response = "Connection not active.";
}
return response;
}
public void closeConnection() {
if (stderr != null) {
try {
stderr.close();
} catch (Exception e) {
log.error("Exception in closeConnection()", e);
}
}
if (stdout != null) {
try {
stdout.close();
} catch (Exception e) {
log.error("Exception in closeConnection()", e);
}
}
if (stdin != null) {
try {
stdin.close();
} catch (Exception e) {
log.error("Exception in closeConnection()", e);
}
}
if (sess != null) {
try {
sess.close();
} catch (Exception e) {
log.error("Exception in closeConnection()", e);
}
}
if (conn != null) {
try {
conn.close();
} catch (Exception e) {
log.error("Exception in closeConnection()", e);
}
}
}
}
You're creating a local Connection variable but you're testing what must be a member variable, which is always null, so you're never closing it.
If authentication fails you're leaking the connection.
I have the following method that I run from my map task in a multithreaded execution , however this works fine in a standalone mod e, but when I runt this in Hadoop YARN it runs out of the physical memory of 1GB and the virtual memory also shoots up.
I need to know if I am doing anything wrong from a programming perspective, I think I am closing all the streams that I am opening ASAP , so I see no reason for a memory leak to happen . Please advise.
Thanks.
public static void manageTheCurrentURL(String url) {
logger.trace("Entering the method manageTheCurrentURL ");
InputStream stream = null;
InputStream is = null;
ByteArrayOutputStream out = null;
WebDriver driver = null;
try {
if (StringUtils.isNotBlank(url)) {
caps.setJavascriptEnabled(true); // not really needed: JS
// enabled by default
caps.setCapability(
PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY,
"/usr/local/bin/phantomjs");
// Launch driver (will take care and ownership of the phantomjs
// process)
driver = new PhantomJSDriver(caps);
driver.get(url);
String htmlContent = driver.getPageSource();
if (htmlContent != null) {
is = new ByteArrayInputStream(htmlContent.getBytes());
ByteArrayDocumentSource byteArrayDocumentSource = new ByteArrayDocumentSource(
is, url, "text/html");
Any23 runner = new Any23();
runner.setHTTPUserAgent("test-user-agent");
out = new ByteArrayOutputStream();
TripleHandler handler = new NTriplesWriter(out);
try {
runner.extract(byteArrayDocumentSource, handler);
} catch (ExtractionException e) {
} finally {
if (driver != null) {
driver.quit();
//driver.close();
}
try {
handler.close();
} catch (TripleHandlerException e) {
}
if (is != null) {
try {
is.close();
} catch (IOException e) {
}
}
}
if (out != null) {
stream = new ByteArrayInputStream(out.toByteArray());
Iterator<Node[]> it = new DeltaParser(stream);
if (it != null) {
SolrCallbackForNXParser callback = new SolrCallbackForNXParser(
url);
callback.startStory();
while (it.hasNext()) {
Node[] abc = it.next();
callback.processStory(abc);
}
callback.endStory();
}
}
}
}
} catch (IOException e) {
return;
}
finally {
if (stream != null) {
try {
stream.close();
} catch (IOException e) {
}
}
if (out != null) {
try {
out.close();
} catch (IOException e) {
}
}
}
logger.trace("Exiting the method manageTheCurrentURL ");
}