Develop Reference

Java - How to call python classes using processbuilder

How do I call and execute python class methods from java. My current java code works, but only if I write:
if __name__ == '__main__':
print("hello")
But I want to execute a class method, regardless of if __name__ == '__main__':
Example python class method I would like to run:
class SECFileScraper:
def __init__(self):
self.counter = 5
def tester_func(self):
return "hello, this test works"
Essentially I would want to run SECFileScraper.tester_func() in java.
My Java code:
try {
ProcessBuilder pb = new ProcessBuilder(Arrays.asList(
"python", pdfFileScraper));
Process p = pb.start();
BufferedReader bfr = new BufferedReader(new InputStreamReader(p.getInputStream()));
String line = "";
System.out.println("Running Python starts: " + line);
int exitCode = p.waitFor();
System.out.println("Exit Code : " + exitCode);
line = bfr.readLine();
System.out.println("First Line: " + line);
while ((line = bfr.readLine()) != null) {
System.out.println("Python Output: " + line);
}
} catch (Exception e) {
e.printStackTrace();
}
pdfFileScraper is the file path to my python script.
I've tried jython, but my python files use pandas and sqlite3, which can't be implemented using jython.

So if I understand your requirement, you want to invoke a class method in pdfFileScraper.py. The basics of doing this from the shell would be something akin to:
scraper=path/to/pdfFileScraper.py
dir_of_scraper=$(dirname $scraper)
export PYTHONPATH=$dir_of_scraper
python -c 'import pdfFileScraper; pdfFileScraper.ClassInScraper()'
What we do is get the directory of pdfFileScraper, and add it to the PYTHONPATH, then we run python with a command that imports the pdfFileScraper file as a module, which exposes all the methods and classes in the class in the namespace pdfFileScraper, and then construct a class ClassInScraper().
In java, something like:
import java.io.*;
import java.util.*;
public class RunFile {
public static void main(String args[]) throws Exception {
File f = new File(args[0]); // .py file (e.g. bob/script.py)
String dir = f.getParent(); // dir of .py file
String file = f.getName(); // name of .py file (script.py)
String module = file.substring(0, file.lastIndexOf('.'));
String command = "import " + module + "; " + module + "." + args[1];
List<String> items = Arrays.asList("python", "-c", command);
ProcessBuilder pb = new ProcessBuilder(items);
Map<String, String> env = pb.environment();
env.put("PYTHONPATH", dir);
pb.redirectErrorStream();
Process p = pb.start();
BufferedReader bfr = new BufferedReader(new InputStreamReader(p.getInputStream()));
String line = "";
System.out.println("Running Python starts: " + line);
int exitCode = p.waitFor();
System.out.println("Exit Code : " + exitCode);
line = bfr.readLine();
System.out.println("First Line: " + line);
while ((line = bfr.readLine()) != null) {
System.out.println("Python Output: " + line);
}
}
}

You can also call Python lib directly via JNI. This way, you don't start new process, you can share context between script calls, etc.
Take a look here for a sample:
https://github.com/mkopsnc/keplerhacks/tree/master/python

This is my java class that worked for me.
class PythonFileReader {
private String path;
private String fileName;
private String methodName;
PythonFileReader(String path, String fileName, String methodName) throws Exception {
this.path = path;
this.fileName = fileName;
this.methodName = methodName;
reader();
}
private void reader() throws Exception {
StringBuilder input_result = new StringBuilder();
StringBuilder output_result = new StringBuilder();
StringBuilder error_result = new StringBuilder();
String line;
String module = fileName.substring(0, fileName.lastIndexOf('.'));
String command = "import " + module + "; " + module + "." + module + "." + methodName;
List<String> items = Arrays.asList("python", "-c", command);
ProcessBuilder pb = new ProcessBuilder(items);
pb.directory(new File(path));
Process p = pb.start();
BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));
BufferedReader out = new BufferedReader(new InputStreamReader(p.getInputStream()));
BufferedReader error = new BufferedReader(new InputStreamReader(p.getErrorStream()));
while ((line = in.readLine()) != null)
input_result.append("\n").append(line);
if (input_result.length() > 0)
System.out.println(fileName + " : " + input_result);
while ((line = out.readLine()) != null)
output_result.append(" ").append(line);
if (output_result.length() > 0)
System.out.println("Output : " + output_result);
while ((line = error.readLine()) != null)
error_result.append(" ").append(line);
if (error_result.length() > 0)
System.out.println("Error : " + error_result);
}}
and this is the way that you can use this class
public static void main(String[] args) throws Exception {
String path = "python/path/file";
String pyFileName = "python_name.py";
String methodeName = "test('stringInput' , 20)";
new PythonFileReader(path, pyFileName, methodeName );
}
and this is my python class
class test:
def test(name, count):
print(name + " - " + str([x for x in range(count)]))

Not able to import csv file into mongodb available in Remote location from java?

Runtime r = Runtime.getRuntime();
String cmd = "C:\\Program Files (x86)\\MongoDB\\Server\\3.0\\bin\\mongoimport -d dummydb -c Employee --type csv --file /home/mongodb/one.csv --headerline";
r.exec(cmd);
When I ran same command in linux machine, csv file imported. But, from java standalone I am not able to insert.
Could you please help on this.

Please try the below code after changing the collection name, database name and file path. It should work.
Note:-
Need to mention mongoimport.exe - as present in the below code.
Code:-
import java.io.BufferedReader;
import java.io.InputStreamReader;
public class MongoImportUtil {
public static void main(String[] args) {
String db = "test";
String col = "Account";
String Host = "localhost";
String Port = "27017";
String fileName = "D:/files/sample.csv";
String command = "C:\\Program Files\\MongoDB\\Server\\3.4\\bin\\mongoimport.exe --host " + Host + " --port "
+ Port + " --db " + db + " --collection " + col + " --headerline --type=csv --file " + fileName;
try {
System.out.println(command);
Process process = Runtime.getRuntime().exec(command);
int waitFor = process.waitFor();
System.out.println("waitFor:: " + waitFor);
BufferedReader success = new BufferedReader(new InputStreamReader(process.getInputStream()));
BufferedReader error = new BufferedReader(new InputStreamReader(process.getErrorStream()));
String s = "";
while ((s = success.readLine()) != null) {
System.out.println(s);
}
while ((s = error.readLine()) != null) {
System.out.println("Std ERROR : " + s);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

What's the difference between InputStream and StreamGobbler?

In the code below, the author uses Runtime to call a sub-process. But I don't understand why he uses StreamGobbler. What will happen if replace it by InputStream? Please help me, thanks!
public class GoodWindowsExec
{
public static void main(String args[])
{
try
{
String osName = System.getProperty("os.name" );
System.out.println("osName: " + osName);
String[] cmd = new String[3];
if(osName.equals("Windows XP") ||osName.equals("Windows 2000"))
{
cmd[0] = "cmd.exe" ;
cmd[1] = "/C" ;
cmd[2] = args[0];
}
else if( osName.equals( "Windows 98" ) )
{
cmd[0] = "command.com" ;
cmd[1] = "/C" ;
cmd[2] = args[0];
}
Runtime rt = Runtime.getRuntime();
System.out.println("Execing " + cmd[0] + " " + cmd[1]+ " " + cmd[2]);
Process proc = rt.exec(cmd);
// any error message?
StreamGobbler errorGobbler = new StreamGobbler(proc.getErrorStream(), "ERROR"); //Can I replace StreamGobbler by InputStream?
// any output?
StreamGobbler outputGobbler = new StreamGobbler(proc.getInputStream(), "OUTPUT"); //As above
// kick them off
errorGobbler.start();
outputGobbler.start();
// any error???
int exitVal = proc.waitFor();
System.out.println("ExitValue: " + exitVal);
} catch (Throwable t){
t.printStackTrace();
}
}
}
BufferedReader brNormal = new BufferedReader(new InputStreamReader(process.getInputStream()));
String s = null;
while ((s = brNormal.readLine()) != null) {
logger.info("RawToRgbConverter.exe", s);
}
brNormal.close();
BufferedReader brError = new BufferedReader(new InputStreamReader(process.getErrorStream()));
while ((s = brError.readLine()) != null) {
logger.error("RawToRgbConverter.exe", s);
}
brError.close();

Even as I don't know the StreamGobbler class, this is obviously a threaded implementation to copy the output and error streams to a given target. Thus, no you cannot simply replace it, as the multi-threading is indeed necessary. A simple input stream would just sit there, but not actually do anything.
Note however, that this complete solution is outdated since Java 1.5 which introduced the ProcessBuilder and the automatic redirects. See the javadoc at https://docs.oracle.com/javase/8/docs/api/

Size limit on BufferedReader exceeded?

In a java 6 webapp, I am attempting to retrieve a large amount of output from an executed command. I've "borrowed/stolen/based" it on the javaworld article. The problem I am facing is that the length appears to exceed a size limit since the output is lopped off. I've output the data to a file so I can see the size of what is returned, and that is exactly 32K (32768). I've experimented with changing the default size of the buffer (see BufferedReader constructor), but I have not observed any change to the length of the data returned no matter what value I have for the buffered-size (very small to very large).
Any advice would be very much appreciated!
public class StreamGobbler extends Thread {
private InputStream is;
private String type;
private List<String> output;
public StreamGobbler(InputStream is, String type) {
this.is = is;
this.type = type;
}
#Override
public void run() {
try {
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
String line = null;
this.output = new ArrayList<String>();
while ((line = br.readLine()) != null) {
this.getOutput().add(line + "\n");
System.out.println(type + ">" + line);
}
br.close();
} catch (IOException ioe) {
System.err.println("ERROR: " + ioe.getMessage());
}
}
/**
* #return the output
*/
public List<String> getOutput() {
return output;
}
}
public class JobClassAds {
private String CONDOR_HISTORY = "condor_history";
private String CONDOR_HISTORY_XML = CONDOR_HISTORY + " -xml";
private String CONDOR_HISTORY_LONG = CONDOR_HISTORY + " -long";
public String getHistory() {
try {
Runtime runtime = Runtime.getRuntime();
String exec = CONDOR_HISTORY_LONG;
Process process = runtime.exec(exec);
System.out.println("Running " + exec + " ...");
// Error message
StreamGobbler errGobbler = new StreamGobbler(process.getErrorStream(), "ERROR");
// Output
StreamGobbler outGobbler = new StreamGobbler(process.getInputStream(), "OUTPUT");
Thread outThread = new Thread(outGobbler);
Thread errThread = new Thread(errGobbler);
outThread.start();
errThread.start();
outThread.join();
errThread.join();
/*
String line = null;
while ((line = input.readLine()) != null) {
System.out.println(line);
content.append(line);
}
*
*/
int exitVal = process.waitFor();
List<String> output = outGobbler.getOutput();
String inputString = "";
for (String o : output) {
inputString += o;
}
System.out.println(exec + " Exited with error code " + exitVal);
BufferedWriter out = new BufferedWriter(new FileWriter("/tmp/history_result.xml"));
out.write(inputString);
out.close();
return inputString;
} catch (Exception e) {
System.err.println(e.getMessage());
return null;
}
}

The problem is not with the BufferedReader's buffer size.
I think that the real cause is something that the external command is doing. I suspect that it is bailing out without flushing its stdout stream. Note that you are "gobbling" but not outputting the command's stderr stream. That's where you may find the evidence pointing to the real cause of the problem.
By the way, you are using the StreamGobbler class in a suboptimal fashion. It extends Thread so the intended way to use is:
SteamGobbler sg = new StreamGobbler(...);
sg.start();
sg.join();
but you are effectively doing this:
SteamGobbler sg = new StreamGobbler(...);
Thread th = new Thread(sg);
th.start();
th.join();
It works ... but only because a Thread is-a Runnable.

Runtime.getRuntime().exec(cmd); appending previous output with each new execution?

I am running into this problem.
my program invokes Runtime.getRuntime().exec(cmd); on a windows platform. I read the error and output stream and do something with this output. This method gets called in a loop after each 4-5 seconds and it goes on till the program is terminated.
Now what happens, each time i read the output, the previous output is appended to the new output and as such with each iteration the result grows bigger and bigger. Is there anyway to stop this thing. The command executed is "tasklist" with some filtering parameters.
I have made a method (which returns String output) for this Runtime.getTuntime().exec(cmd) in which i am also closing the process after execution but when it is called from within the loop, each time previous output is appended to the new one.
Here is the code:
class Track implements Runnable {
static int size = 0;
public void run() {
String cmd1 = "tasklist /fo list /fi \"imagename eq java.exe\"";
String cmd2 = "tasklist /fo list /fi \"imagename eq javaw.exe\"";
String text = "";
int i=1, j=0;
while(size < 100000){
try{
text = fList.pList(cmd2, 1);
if (text.indexOf("javaw.exe")== -1){
text = fList.pList(cmd1, 1);
}
if(j==22) System.out.println(text);
if (text.charAt(0)!= '0') continue;
i = text.lastIndexOf("Mem Usage: ")+14;
text = text.substring(i);
text = text.substring(0,text.lastIndexOf(" K"));
text = text.replaceFirst(",", "");
size = Integer.parseInt(text);
System.out.println(size);
Thread.sleep(3000);
j++;
} catch(Exception e){
System.out.println(e);
}
}
System.out.println("Memory utlization exceeded the permissible limit");
System.out.println("Now terminating the Program\n");
System.exit(1);
}
static void memoryCheck(int size) throws Exception{
(new Thread(new Track())).start();
}
}
in class fList is the method pList:
static String pList(String cmd, int eval) throws Exception{ //can execute external command
String out = "";
int val = 5; // should not be zero, to verify returned code zero for normal exec.
try
{
//String osName = System.getProperty("os.name" );
Runtime rt = Runtime.getRuntime();
Process proc = rt.exec(cmd);
// any error message?
eProcList error = new eProcList(proc.getErrorStream());
// any output?
eProcList output = new eProcList(proc.getInputStream());
// kick them off
error.start();
output.start();
// any error???
int exitVal = proc.waitFor();
out = eProcList.procList();
val = exitVal;
proc.destroy();
proc.getInputStream().close();
proc.getErrorStream().close();
} catch (Throwable t)
{
t.printStackTrace();
}
if (eval==1) return val + out;
return out;
}
class eProcList extends Thread
{
InputStream iStream;
static String oPut = "";
eProcList(InputStream iStream)
{
this.iStream = iStream;
}
public void run()
{
try
{
InputStreamReader isr = new InputStreamReader(iStream);
BufferedReader br = new BufferedReader(isr);
String line=null;
while ( (line = br.readLine()) != null)
oPut = oPut + line+"\n";
} catch (IOException e)
{
e.printStackTrace();
}
}
static public String procList(){
return oPut;
}
}
you asked so iv'e copied all here.

You made oPut a static field - its initialized to "" once when the class is loaded and then shared between every new instance of eProcList, i.e. never cleared of the previous run. Either don't make it static (why is it static?) or clear it in the constructor.