Pyspark: python worker failed to connect back/socketTimeOut - java

I am very new to pyspark and still somewhat of a beginner with python. I'm taking an online course to better learn pyspark, but I cannot get the following code to run. I've tried both collect and take to see if i can test the initial result after parsing lines, but that doesn't seem to work. I have attached the error message provided by jupyter notebook on my local windows machine.
import findspark
findspark.init()
from pyspark import SparkConf, SparkContext
conf = SparkConf().setMaster("local").setAppName("MinTemperatures")
conf.set("spark.network.timeout", "600s")
sc = SparkContext.getOrCreate()
def parseLine(line):
fields = line.split(',')
stationID = fields[0]
entryType = fields[2]
temperature = fields[3]
return (stationID, entryType, temperature)
file="C:/sparkCourse/1800.csv"
lines = sc.textFile(file)
parsedLines = lines.map(parseLine)
minTemps = parsedLines.filter(lambda x: "TMIN" in x[1])
stationTemps = minTemps.map(lambda x: (x[0], x[2]))
minTemps = stationTemps.reduceByKey(lambda x, y: min(x,y))
lines.take(5)
#results = minTemps.collect();
#for result in results:
# print(result[0] + "\t{:.2f}F".format(result[1]))
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
<ipython-input-4-6a97a8b6e79c> in <module>
21 minTemps = stationTemps.reduceByKey(lambda x, y: min(x,y))
22
---> 23 lines.take(5)
24 #results = minTemps.collect();
25
C:\spark\python\pyspark\rdd.py in take(self, num)
1358
1359 p = range(partsScanned, min(partsScanned + numPartsToTry, totalParts))
-> 1360 res = self.context.runJob(self, takeUpToNumLeft, p)
1361
1362 items += res
C:\spark\python\pyspark\context.py in runJob(self, rdd, partitionFunc, partitions, allowLocal)
1049 # SparkContext#runJob.
1050 mappedRDD = rdd.mapPartitions(partitionFunc)
-> 1051 sock_info = self._jvm.PythonRDD.runJob(self._jsc.sc(), mappedRDD._jrdd, partitions)
1052 return list(_load_from_socket(sock_info, mappedRDD._jrdd_deserializer))
1053
C:\spark\python\lib\py4j-0.10.7-src.zip\py4j\java_gateway.py in __call__(self, *args)
1255 answer = self.gateway_client.send_command(command)
1256 return_value = get_return_value(
-> 1257 answer, self.gateway_client, self.target_id, self.name)
1258
1259 for temp_arg in temp_args:
C:\spark\python\pyspark\sql\utils.py in deco(*a, **kw)
61 def deco(*a, **kw):
62 try:
---> 63 return f(*a, **kw)
64 except py4j.protocol.Py4JJavaError as e:
65 s = e.java_exception.toString()
C:\spark\python\lib\py4j-0.10.7-src.zip\py4j\protocol.py in get_return_value(answer, gateway_client, target_id, name)
326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
--> 328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.runJob.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 1.0 failed 1 times, most recent failure: Lost task 0.0 in stage 1.0 (TID 1, localhost, executor driver): org.apache.spark.SparkException: Python worker failed to connect back.
at org.apache.spark.api.python.PythonWorkerFactory.createSimpleWorker(PythonWorkerFactory.scala:170)
at org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:97)
at org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:117)
at org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:108)
at org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:65)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:402)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:408)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.lang.Thread.run(Unknown Source)
Caused by: java.net.SocketTimeoutException: Accept timed out
at java.net.DualStackPlainSocketImpl.waitForNewConnection(Native Method)
at java.net.DualStackPlainSocketImpl.socketAccept(Unknown Source)
at java.net.AbstractPlainSocketImpl.accept(Unknown Source)
at java.net.PlainSocketImpl.accept(Unknown Source)
at java.net.ServerSocket.implAccept(Unknown Source)
at java.net.ServerSocket.accept(Unknown Source)
at org.apache.spark.api.python.PythonWorkerFactory.createSimpleWorker(PythonWorkerFactory.scala:164)
... 14 more
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1887)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1875)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1874)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1874)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:926)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2108)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2057)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2046)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:737)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2061)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2082)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2101)
at org.apache.spark.api.python.PythonRDD$.runJob(PythonRDD.scala:153)
at org.apache.spark.api.python.PythonRDD.runJob(PythonRDD.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Unknown Source)
Caused by: org.apache.spark.SparkException: Python worker failed to connect back.
at org.apache.spark.api.python.PythonWorkerFactory.createSimpleWorker(PythonWorkerFactory.scala:170)
at org.apache.spark.api.python.PythonWorkerFactory.create(PythonWorkerFactory.scala:97)
at org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:117)
at org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:108)
at org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:65)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:402)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:408)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
... 1 more
Caused by: java.net.SocketTimeoutException: Accept timed out
at java.net.DualStackPlainSocketImpl.waitForNewConnection(Native Method)
at java.net.DualStackPlainSocketImpl.socketAccept(Unknown Source)
at java.net.AbstractPlainSocketImpl.accept(Unknown Source)
at java.net.PlainSocketImpl.accept(Unknown Source)
at java.net.ServerSocket.implAccept(Unknown Source)
at java.net.ServerSocket.accept(Unknown Source)
at org.apache.spark.api.python.PythonWorkerFactory.createSimpleWorker(PythonWorkerFactory.scala:164)
... 14 more

You created SparkConf, but then you didn't pass it to SparkContext. I think it should look something like this:
conf = SparkConf().setMaster("local").setAppName("MinTemperatures").set("spark.network.timeout", "600s")
sc = SparkContext(conf=conf)

Related

: java.lang.NoSuchMethodError: 'scala.collection.mutable.ArrayOps scala.Predef$.refArrayOps(java.lang.Object[])'

I was trying to implement the DBSCAN model in PySpark framework. Github link of complete implementation . I just copied and run the code from 'README.md' segment. The code is
import dbscan
from sklearn.datasets import make_blobs
from pyspark.sql import types as T, SparkSession
from scipy.spatial import distance
spark = SparkSession \
.builder \
.appName("DBSCAN") \
.config("spark.jars.packages", "graphframes:graphframes:0.7.0-spark2.3-s_2.11") \
.config('spark.driver.host', '127.0.0.1') \
.getOrCreate()
X, labels_true = make_blobs(n_samples=750, centers=centers, cluster_std=0.4, random_state=5)
data = [(i, [float(item) for item in X[i]]) for i in range(X.shape[0])]
schema = T.StructType([T.StructField("id", T.IntegerType(), False),
T.StructField("value", T.ArrayType(T.FloatType()), False)])
df = spark.createDataFrame(data, schema=schema)
df_clusters = dbscan.process(spark, df, .2, 10, distance.euclidean, 2, "checkpoint")
which shows me the error message as
-
**> --------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call
last)
in ()
12 T.StructField("value", T.ArrayType(T.FloatType()), False)])
13 df = spark.createDataFrame(data, schema=schema)
---> 14 df_clusters = dbscan.process(spark, df, .2, 10, distance.euclidean, 2, "checkpoint")
4 frames
/usr/local/lib/python3.6/dist-packages/py4j/protocol.py in
get_return_value(answer, gateway_client, target_id, name)
326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
--> 328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
Py4JJavaError: An error occurred while calling o185.createGraph. :
java.lang.NoSuchMethodError: 'scala.collection.mutable.ArrayOps
scala.Predef$.refArrayOps(java.lang.Object[])' at
org.graphframes.GraphFrame$.apply(GraphFrame.scala:676) at
org.graphframes.GraphFramePythonAPI.createGraph(GraphFramePythonAPI.scala:10)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method) at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566) at
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at
py4j.Gateway.invoke(Gateway.java:282) at
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79) at
py4j.GatewayConnection.run(GatewayConnection.java:238) at
java.base/java.lang.Thread.run(Thread.java:834)**
I really don't know anything on Java or scala ; also beginner at pyspark . Kindly help me to solve out this problem , I need it badly!

Why wont Java run this code?

I have a ProcessBuilder named pb. The class has the following Constructor:
public Server(JSONObject root) {
this.jv = new JarVersion( (JSONObject) root.get("jarV") );
this.ram = Integer.parseInt(root.get("ram").toString());
this.name = root.get("name").toString();
this.unit = size.valueOf( root.get("size") .toString().toUpperCase() );
String wpath = Core.serverpath + fls + this.name;
String cmd = "java -Xmx"+this.ram;
cmd += this.unit.getShort(this.unit) + " ";
cmd += "-jar ";
cmd += wpath + fls + jv.getFullVersion() + ".jar";
pb = new ProcessBuilder(cmd);
pb.directory(new File( wpath ));
}
My Current pb has this command: "java -Xmx1024M -jar C:\Users[USERNAME]\Desktop\SLV2_DATA\server\test\spigot-1.11.1.jar".
The path is available and this command is working from CMD.exe i don't know why it won't in the class ?
Why does this command cannot be executed?
This is the error I get:
java.io.IOException: Cannot run program "java -Xmx1024M -jar C:\Users\RicheeNektar\Desktop\SLV2_DATA\server\test\spigot-1.11.1.jar" (in directory "C:\Users\RicheeNektar\Desktop\SLV2_DATA\server\test"): CreateProcess error=2, Das System kann die angegebene Datei nicht finden
at java.base/java.lang.ProcessBuilder.start(Unknown Source)
at java.base/java.lang.ProcessBuilder.start(Unknown Source)
at org.RicheeNektar.Resources.Server.start(Server.java:90)
at org.RicheeNektar.frames.Console.<init>(Console.java:176)
at org.RicheeNektar.ServerPanel.Core$5.actionPerformed(Core.java:345)
at java.desktop/javax.swing.AbstractButton.fireActionPerformed(Unknown Source)
at java.desktop/javax.swing.AbstractButton$Handler.actionPerformed(Unknown Source)
at java.desktop/javax.swing.DefaultButtonModel.fireActionPerformed(Unknown Source)
at java.desktop/javax.swing.DefaultButtonModel.setPressed(Unknown Source)
at java.desktop/javax.swing.plaf.basic.BasicButtonListener.mouseReleased(Unknown Source)
at java.desktop/java.awt.Component.processMouseEvent(Unknown Source)
at java.desktop/javax.swing.JComponent.processMouseEvent(Unknown Source)
at java.desktop/java.awt.Component.processEvent(Unknown Source)
at java.desktop/java.awt.Container.processEvent(Unknown Source)
at java.desktop/java.awt.Component.dispatchEventImpl(Unknown Source)
at java.desktop/java.awt.Container.dispatchEventImpl(Unknown Source)
at java.desktop/java.awt.Component.dispatchEvent(Unknown Source)
at java.desktop/java.awt.LightweightDispatcher.retargetMouseEvent(Unknown Source)
at java.desktop/java.awt.LightweightDispatcher.processMouseEvent(Unknown Source)
at java.desktop/java.awt.LightweightDispatcher.dispatchEvent(Unknown Source)
at java.desktop/java.awt.Container.dispatchEventImpl(Unknown Source)
at java.desktop/java.awt.Window.dispatchEventImpl(Unknown Source)
at java.desktop/java.awt.Component.dispatchEvent(Unknown Source)
at java.desktop/java.awt.EventQueue.dispatchEventImpl(Unknown Source)
at java.desktop/java.awt.EventQueue.access$500(Unknown Source)
at java.desktop/java.awt.EventQueue$3.run(Unknown Source)
at java.desktop/java.awt.EventQueue$3.run(Unknown Source)
at java.base/java.security.AccessController.doPrivileged(Native Method)
at java.base/java.security.ProtectionDomain$JavaSecurityAccessImpl.doIntersectionPrivilege(Unknown Source)
at java.base/java.security.ProtectionDomain$JavaSecurityAccessImpl.doIntersectionPrivilege(Unknown Source)
at java.desktop/java.awt.EventQueue$4.run(Unknown Source)
at java.desktop/java.awt.EventQueue$4.run(Unknown Source)
at java.base/java.security.AccessController.doPrivileged(Native Method)
at java.base/java.security.ProtectionDomain$JavaSecurityAccessImpl.doIntersectionPrivilege(Unknown Source)
at java.desktop/java.awt.EventQueue.dispatchEvent(Unknown Source)
at java.desktop/java.awt.EventDispatchThread.pumpOneEventForFilters(Unknown Source)
at java.desktop/java.awt.EventDispatchThread.pumpEventsForFilter(Unknown Source)
at java.desktop/java.awt.EventDispatchThread.pumpEventsForHierarchy(Unknown Source)
at java.desktop/java.awt.EventDispatchThread.pumpEvents(Unknown Source)
at java.desktop/java.awt.EventDispatchThread.pumpEvents(Unknown Source)
at java.desktop/java.awt.EventDispatchThread.run(Unknown Source)
Caused by: java.io.IOException: CreateProcess error=2, Das System kann die angegebene Datei nicht finden
at java.base/java.lang.ProcessImpl.create(Native Method)
at java.base/java.lang.ProcessImpl.<init>(Unknown Source)
at java.base/java.lang.ProcessImpl.start(Unknown Source)
... 41 more

Failed to access the WSDL

Hello I am trying setting up a simple WSDL program in Java but I get the following error, any advices? I am fairly new at this, and this program is my first one of sorts.
Thank you,
package de.vs.webServices.clientSide;
import java.net.URL;
import java.util.Scanner;
import javax.xml.namespace.QName;
import javax.xml.ws.Service;
import de.vs.webServices.serverSide.ZeichenketteInt;
public class ZeichenketteClient {
public static void main(String[] args) throws Exception {
URL url = new URL("http://localhost:9999/vs/Zeichenkette?wsdl");
QName qname = new QName("http://serverSide.webServices.vs.de/","ZeichenketteImplService");
Service service = Service.create(url, qname);
ZeichenketteInt zeichenketteInt = service.getPort(ZeichenketteInt.class);
Scanner sc = new Scanner(System.in);
int nr=0;
do {
System.out.println("Waehlen Sie die Nummer eines Befehls oder 0 um das Programm zu beenden: ");
System.out.println("1. Zeichen Doppeln");
System.out.println("2. Zeichenkette Verdoppeln");
System.out.println("3. ZeichenketteSpiegeln");
System.out.println("4. ZeichenketteLange");
nr = sc.nextInt();
String zeichenkette = null;
switch (nr) {
case 1:
System.out
.println("Schreiben Sie die gewunschte Zeichenkette: ");
zeichenkette = sc.next();
System.out.println("Ergebnis: "
+ zeichenketteInt.zeichenDoppeln(zeichenkette));
break;
case 2:
System.out
.println("Schreiben Sie die gewunschte Zeichenkette: ");
zeichenkette = sc.next();
System.out.println("Ergebnis: "
+ zeichenketteInt.zeichenKetteVerdoppeln(zeichenkette));
break;
case 3:
System.out
.println("Schreiben Sie die gewunschte Zeichenkette: ");
zeichenkette = sc.next();
System.out.println("Ergebnis: "
+ zeichenketteInt.zeichenKetteSpiegeln(zeichenkette));
break;
case 4:
System.out
.println("Schreiben Sie die gewunschte Zeichenkette: ");
zeichenkette = sc.next();
System.out.println("Ergebnis: "
+ zeichenketteInt.zeichenKetteLange(zeichenkette));
break;
}
} while (nr!=0);
sc.close();
}}
This is the error that I get:
Exception in thread "main" javax.xml.ws.WebServiceException: Failed to access the WSDL at: http://localhost:9999/vs/Zeichenkette?wsdl. It failed with:
Connection refused: connect.
at com.sun.xml.internal.ws.wsdl.parser.RuntimeWSDLParser.tryWithMex(Unknown Source)
at com.sun.xml.internal.ws.wsdl.parser.RuntimeWSDLParser.parse(Unknown Source)
at com.sun.xml.internal.ws.wsdl.parser.RuntimeWSDLParser.parse(Unknown Source)
at com.sun.xml.internal.ws.wsdl.parser.RuntimeWSDLParser.parse(Unknown Source)
at com.sun.xml.internal.ws.client.WSServiceDelegate.parseWSDL(Unknown Source)
at com.sun.xml.internal.ws.client.WSServiceDelegate.<init>(Unknown Source)
at com.sun.xml.internal.ws.client.WSServiceDelegate.<init>(Unknown Source)
at com.sun.xml.internal.ws.client.WSServiceDelegate.<init>(Unknown Source)
at com.sun.xml.internal.ws.client.WSServiceDelegate.<init>(Unknown Source)
at com.sun.xml.internal.ws.spi.ProviderImpl.createServiceDelegate(Unknown Source)
at javax.xml.ws.Service.<init>(Unknown Source)
at javax.xml.ws.Service.create(Unknown Source)
at de.vs.webServices.clientSide.ZeichenketteClient.main(ZeichenketteClient.java:19)
Caused by: java.net.ConnectException: Connection refused: connect
at java.net.DualStackPlainSocketImpl.connect0(Native Method)
at java.net.DualStackPlainSocketImpl.socketConnect(Unknown Source)
at java.net.AbstractPlainSocketImpl.doConnect(Unknown Source)
at java.net.AbstractPlainSocketImpl.connectToAddress(Unknown Source)
at java.net.AbstractPlainSocketImpl.connect(Unknown Source)
at java.net.PlainSocketImpl.connect(Unknown Source)
at java.net.SocksSocketImpl.connect(Unknown Source)
at java.net.Socket.connect(Unknown Source)
at java.net.Socket.connect(Unknown Source)
at sun.net.NetworkClient.doConnect(Unknown Source)
at sun.net.www.http.HttpClient.openServer(Unknown Source)
at sun.net.www.http.HttpClient.openServer(Unknown Source)
at sun.net.www.http.HttpClient.<init>(Unknown Source)
at sun.net.www.http.HttpClient.New(Unknown Source)
at sun.net.www.http.HttpClient.New(Unknown Source)
at sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(Unknown Source)
at sun.net.www.protocol.http.HttpURLConnection.plainConnect0(Unknown Source)
at sun.net.www.protocol.http.HttpURLConnection.plainConnect(Unknown Source)
at sun.net.www.protocol.http.HttpURLConnection.connect(Unknown Source)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(Unknown Source)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(Unknown Source)
at java.net.URL.openStream(Unknown Source)
at com.sun.xml.internal.ws.wsdl.parser.RuntimeWSDLParser.createReader(Unknown Source)
at com.sun.xml.internal.ws.wsdl.parser.RuntimeWSDLParser.resolveWSDL(Unknown Source)
... 12 more
The part of the error that says: "refused: connect"
is telling you that the server (localhost in this case) rejected your connection request.
I would check that the URI: http://localhost:9999/vs/Zeichenkette?wsdl actually returns a WSDL document. Just open it in a web browser and see what you get back. If the browser can't open it, Java can't either. Once the browser will open it, Java should have no problem with it (though it occasionally happens that the browser reads it fine and permissions in Java stop you, this doesn't appear to be one of those cases).
Then I suggest looking to make certain that the server port is open (IPTables, AV, things like that can block it). Note that this is not likely your problem, since the error is reading the WSDL. I'm looking ahead to after the WSDL loads.

Javasist throws javassist.CannotCompileException when instrumenting setString method of org.h2.jdbc.JdbcPreparedStatement

um trying to instrument jdbc methods while a server is running. I have tried it by instrumenting setString, setInt methods and executeQuery method while a simple mysql query is running as it is given in JDBC examples. It works totally fine when i instrument that setString method by injecting following line.
private void injectSetVariableMethods(CtMethod method) {
if (isInEnum(method.getName().toUpperCase(), SetMethods.class)) {
try {
method.insertAt(1, true,
"javaagent.JDBCPublisher.fillArrayList(String.valueOf($2), " +
"Thread.currentThread().getStackTrace()[1].getMethodName().toUpperCase());"
);
} catch (CannotCompileException e) {
e.printStackTrace();
}
}
}
But now, when i run it with the server which use h2, it gives the following exception.
javassist.CannotCompileException: by javassist.bytecode.BadBytecode: setString (ILjava/lang/String;)V in org.h2.jdbc.JdbcPreparedStatement: failed to resolve types
at javassist.CtBehavior.insertAt(CtBehavior.java:1210)
at javaagent.JDBCTransformer.injectSetVariableMethods(JDBCClassTransformer.java:212)
at javaagent.JDBCTransformer.transform(JDBCClassTransformer.java:99)
at sun.instrument.TransformerManager.transform(TransformerManager.java:188)
at sun.instrument.InstrumentationImpl.transform(InstrumentationImpl.java:424)
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:800)
at org.eclipse.osgi.internal.baseadaptor.DefaultClassLoader.defineClass(DefaultClassLoader.java:188)
at org.eclipse.osgi.baseadaptor.loader.ClasspathManager.defineClassHoldingLock(ClasspathManager.java:638)
at org.eclipse.osgi.baseadaptor.loader.ClasspathManager.defineClass(ClasspathManager.java:613)
at org.eclipse.osgi.baseadaptor.loader.ClasspathManager.findClassImpl(ClasspathManager.java:574)
at org.eclipse.osgi.baseadaptor.loader.ClasspathManager.findLocalClassImpl(ClasspathManager.java:492)
at org.eclipse.osgi.baseadaptor.loader.ClasspathManager.findLocalClass(ClasspathManager.java:465)
at org.eclipse.osgi.internal.baseadaptor.DefaultClassLoader.findLocalClass(DefaultClassLoader.java:216)
at org.eclipse.osgi.internal.loader.BundleLoader.findLocalClass(BundleLoader.java:395)
at org.eclipse.osgi.internal.loader.BundleLoader.findClassInternal(BundleLoader.java:464)
at org.eclipse.osgi.internal.loader.BundleLoader.findClass(BundleLoader.java:421)
at org.eclipse.osgi.internal.loader.BundleLoader.findClass(BundleLoader.java:412)
at org.eclipse.osgi.internal.baseadaptor.DefaultClassLoader.loadClass(DefaultClassLoader.java:107)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
at org.h2.jdbc.JdbcConnection.prepareStatement(JdbcConnection.java:234)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.tomcat.jdbc.pool.ProxyConnection.invoke(ProxyConnection.java:126)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:109)
at org.carbon.ndatasource.rdbms.ConnectionRollbackOnReturnInterceptor.invoke(ConnectionRollbackOnReturnInterceptor.java:51)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:109)
at org.apache.tomcat.jdbc.pool.interceptor.AbstractCreateStatementInterceptor.invoke(AbstractCreateStatementInterceptor.java:67)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:109)
at org.apache.tomcat.jdbc.pool.interceptor.ConnectionState.invoke(ConnectionState.java:153)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:109)
at org.apache.tomcat.jdbc.pool.TrapException.invoke(TrapException.java:41)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:109)
at org.apache.tomcat.jdbc.pool.DisposableConnectionFacade.invoke(DisposableConnectionFacade.java:80)
at com.sun.proxy.$Proxy18.prepareStatement(Unknown Source)
at org.carbon.user.core.claim.dao.ClaimDAO.getDialectCount(ClaimDAO.java:160)
at org.carbon.user.core.common.DefaultRealm.populateProfileAndClaimMaps(DefaultRealm.java:429)
at org.carbon.user.core.common.DefaultRealm.init(DefaultRealm.java:105)
at org.carbon.user.core.common.DefaultRealmService.initializeRealm(DefaultRealmService.java:230)
at org.wso2.carbon.user.core.common.DefaultRealmService.<init>(DefaultRealmService.java:96)
at org.wso2.carbon.user.core.common.DefaultRealmService.<init>(DefaultRealmService.java:109)
at org.carbon.user.core.internal.Activator.startDeploy(Activator.java:68)
at org.wso2.carbon.user.core.internal.BundleCheckActivator.start(BundleCheckActivator.java:61)
at org.eclipse.osgi.framework.internal.core.BundleContextImpl$1.run(BundleContextImpl.java:711)
at java.security.AccessController.doPrivileged(Native Method)
at org.eclipse.osgi.framework.internal.core.BundleContextImpl.startActivator(BundleContextImpl.java:702)
at org.eclipse.osgi.framework.internal.core.BundleContextImpl.start(BundleContextImpl.java:683)
at org.eclipse.osgi.framework.internal.core.BundleHost.startWorker(BundleHost.java:381)
at org.eclipse.osgi.framework.internal.core.AbstractBundle.resume(AbstractBundle.java:390)
at org.eclipse.osgi.framework.internal.core.Framework.resumeBundle(Framework.java:1176)
at org.eclipse.osgi.framework.internal.core.StartLevelManager.resumeBundles(StartLevelManager.java:559)
at org.eclipse.osgi.framework.internal.core.StartLevelManager.resumeBundles(StartLevelManager.java:544)
at org.eclipse.osgi.framework.internal.core.StartLevelManager.incFWSL(StartLevelManager.java:457)
at org.eclipse.osgi.framework.internal.core.StartLevelManager.doSetStartLevel(StartLevelManager.java:243)
at org.eclipse.osgi.framework.internal.core.StartLevelManager.dispatchEvent(StartLevelManager.java:438)
at org.eclipse.osgi.framework.internal.core.StartLevelManager.dispatchEvent(StartLevelManager.java:1)
at org.eclipse.osgi.framework.eventmgr.EventManager.dispatchEvent(EventManager.java:230)
at org.eclipse.osgi.framework.eventmgr.EventManager$EventThread.run(EventManager.java:340)
Caused by: javassist.bytecode.BadBytecode: setString (ILjava/lang/String;)V in org.h2.jdbc.JdbcPreparedStatement: failed to resolve types
at javassist.bytecode.stackmap.MapMaker.make(MapMaker.java:111)
at javassist.bytecode.MethodInfo.rebuildStackMap(MethodInfo.java:423)
at javassist.bytecode.MethodInfo.rebuildStackMapIf6(MethodInfo.java:405)
at javassist.CtBehavior.insertAt(CtBehavior.java:1200)
... 59 more
Caused by: javassist.bytecode.BadBytecode: failed to resolve types
at javassist.bytecode.stackmap.MapMaker.make(MapMaker.java:169)
at javassist.bytecode.stackmap.MapMaker.make(MapMaker.java:108)
... 62 more
Caused by: javassist.NotFoundException: org.h2.value.ValueNull
at javassist.ClassPool.get(ClassPool.java:450)
at javassist.bytecode.stackmap.TypeData$TypeVar.fixTypes2(TypeData.java:345)
at javassist.bytecode.stackmap.TypeData$TypeVar.fixTypes(TypeData.java:330)
at javassist.bytecode.stackmap.TypeData$TypeVar.dfs(TypeData.java:274)
at javassist.bytecode.stackmap.MapMaker.fixTypes(MapMaker.java:394)
at javassist.bytecode.stackmap.MapMaker.make(MapMaker.java:167)
... 63 more
What i am doing with my fillArrayList method is, um passing those values into a ArrayList by checking the method name and adding '' for values set with setString. But it looks like it is instrumenting that method at somepoint, because i am getting the reimplemented queries with '?' replaced with respective values (strings with '' and ints as normal). But once the server has started it throws another set of exceptions which also involve h2.
[2015-10-13 17:18:56,600] ERROR {org.carbon.registry.core.jdbc.dao.JDBCLogsDAO} - Failed to get logs. General error: "java.lang.IndexOutOfBoundsException: Index: 1, Size: 1" [50000-140]
org.h2.jdbc.JdbcSQLException: General error: "java.lang.IndexOutOfBoundsException: Index: 1, Size: 1" [50000-140]
at org.h2.message.DbException.getJdbcSQLException(DbException.java:327)
at org.h2.message.DbException.get(DbException.java:156)
at org.h2.message.DbException.convert(DbException.java:279)
at org.h2.message.DbException.toSQLException(DbException.java:252)
at org.h2.message.TraceObject.logAndConvert(TraceObject.java:386)
at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:104)
at org.carbon.registry.core.jdbc.dao.JDBCLogsDAO.internalGetLogs(JDBCLogsDAO.java:427)
at org.carbon.registry.core.jdbc.dao.JDBCLogsDAO.getLogList(JDBCLogsDAO.java:317)
at org.carbon.registry.core.jdbc.EmbeddedRegistry.getLogs(EmbeddedRegistry.java:2332)
at org.carbon.registry.core.caching.CacheBackedRegistry.getLogs(CacheBackedRegistry.java:402)
at org.carbon.registry.core.session.UserRegistry.getLogsInternal(UserRegistry.java:1806)
at org.carbon.registry.core.session.UserRegistry.access$3600(UserRegistry.java:60)
at org.carbon.registry.core.session.UserRegistry$37.run(UserRegistry.java:1777)
at org.carbon.registry.core.session.UserRegistry$37.run(UserRegistry.java:1774)
at java.security.AccessController.doPrivileged(Native Method)
at org.carbon.registry.core.session.UserRegistry.getLogs(UserRegistry.java:1774)
at org.carbon.registry.indexing.ResourceSubmitter.submitResource(ResourceSubmitter.java:119)
at org.carbon.registry.indexing.ResourceSubmitter.run(ResourceSubmitter.java:76)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IndexOutOfBoundsException: Index: 1, Size: 1
at java.util.ArrayList.rangeCheck(ArrayList.java:635)
at java.util.ArrayList.get(ArrayList.java:411)
at javaagent.JDBCPublisher.getArrayList(JDBCAgentPublisher.java:151)
at javaagent.JDBCPublisher.modifyOriginalQuery(JDBCAgentPublisher.java:351)
at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:84)
... 19 more
[2015-10-13 17:18:56,601] WARN {org.carbon.registry.indexing.ResourceSubmitter} - An error occurred while submitting resources for indexing
org.carbon.registry.core.exceptions.RegistryException: Failed to get logs. General error: "java.lang.IndexOutOfBoundsException: Index: 1, Size: 1" [50000-140]
at org.carbon.registry.core.jdbc.dao.JDBCLogsDAO.internalGetLogs(JDBCLogsDAO.java:465)
at org.carbon.registry.core.jdbc.dao.JDBCLogsDAO.getLogList(JDBCLogsDAO.java:317)
at org.carbon.registry.core.jdbc.EmbeddedRegistry.getLogs(EmbeddedRegistry.java:2332)
at org.carbon.registry.core.caching.CacheBackedRegistry.getLogs(CacheBackedRegistry.java:402)
at org.carbon.registry.core.session.UserRegistry.getLogsInternal(UserRegistry.java:1806)
at org.carbon.registry.core.session.UserRegistry.access$3600(UserRegistry.java:60)
at org.wso2.carbon.registry.core.session.UserRegistry$37.run(UserRegistry.java:1777)
at org.carbon.registry.core.session.UserRegistry$37.run(UserRegistry.java:1774)
at java.security.AccessController.doPrivileged(Native Method)
at org.carbon.registry.core.session.UserRegistry.getLogs(UserRegistry.java:1774)
at org.carbon.registry.indexing.ResourceSubmitter.submitResource(ResourceSubmitter.java:119)
at org.carbon.registry.indexing.ResourceSubmitter.run(ResourceSubmitter.java:76)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.h2.jdbc.JdbcSQLException: General error: "java.lang.IndexOutOfBoundsException: Index: 1, Size: 1" [50000-140]
at org.h2.message.DbException.getJdbcSQLException(DbException.java:327)
at org.h2.message.DbException.get(DbException.java:156)
at org.h2.message.DbException.convert(DbException.java:279)
at org.h2.message.DbException.toSQLException(DbException.java:252)
at org.h2.message.TraceObject.logAndConvert(TraceObject.java:386)
at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:104)
at org.wso2.carbon.registry.core.jdbc.dao.JDBCLogsDAO.internalGetLogs(JDBCLogsDAO.java:427)
... 18 more
Caused by: java.lang.IndexOutOfBoundsException: Index: 1, Size: 1
at java.util.ArrayList.rangeCheck(ArrayList.java:635)
at java.util.ArrayList.get(ArrayList.java:411)
at javaagent.JDBCPublisher.getArrayList(JDBCAgentPublisher.java:151)
at javaagent.JDBCPublisher.modifyOriginalQuery(JDBCAgentPublisher.java:351)
at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:84)
... 19 more
It throws bunch of IndexOutOfBoundExceptions repeatedly with a properly assigned sql query. What could have making this issue..... what should i do to correct this?
Find below a stripped down example, which prints the values of each invocation of the method org.h2.jdbc.JdbcPreparedStatement.setString(int, String).
Following directory structure and content is assumed:
./instrumented/
h2-1.4.186.jar
javassist-3.7.ga.jar
SetStringDemo.java
execution of the example
javac -cp javassist-3.7.ga.jar;h2-1.4.186.jar SetStringDemo.java
java -cp .;instrumented/;javassist-3.7.ga.jar;h2-1.4.186.jar SetStringDemo
output
instrument class org.h2.jdbc.JdbcPreparedStatement
create test database and insert some rows
idx: 2 value: 'name 0'
idx: 2 value: 'name 1'
idx: 2 value: 'name 2'
idx: 2 value: 'name 3'
idx: 2 value: 'name 4'
So the problem is most probably in the way you instrument the class.
Code used for the example.
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import javassist.ClassPool;
import javassist.CtClass;
import javassist.CtMethod;
public class SetStringDemo {
// exception handling left out for the PoC
public static void main(String[] args) throws Exception {
if (Files.deleteIfExists(
Paths.get("instrumented/org/h2/jdbc/JdbcPreparedStatement.class")
)) {
System.out.println("previously instrumented class removed");
}
System.out.println("instrument class org.h2.jdbc.JdbcPreparedStatement");
ClassPool pool = ClassPool.getDefault();
CtClass clazz = pool.get("org.h2.jdbc.JdbcPreparedStatement");
CtMethod method = clazz.getDeclaredMethod("setString");
method.insertAt(1,
"System.out.println(\"idx: \" + $1 + \" value: '\" + $2 + \"'\");"
);
clazz.writeFile("instrumented/");
System.out.println("create test database and insert some rows");
try (Connection conn = DriverManager.getConnection("jdbc:h2:mem:", "sa", "")) {
String createTable = "CREATE TABLE TEST_TABLE(ID INT, NAME VARCHAR(1024))";
conn.createStatement().executeUpdate(createTable);
String insertSql = "INSERT INTO TEST_TABLE VALUES(?, ?)";
try (PreparedStatement insertStmnt = conn.prepareStatement(insertSql)) {
for (int i = 0; i < 5; i++) {
insertStmnt.setInt(1, i);
insertStmnt.setString(2, "name " + i);
insertStmnt.executeUpdate();
}
}
}
}
}

MalformedInputException in Stream<String> handling

This code seems to be returning a java.nio.charset.MalformedInputException: Input length = 1 error and for the life of me I can't figure out why.
Files.lines(Paths.get(fileName))
.forEachOrdered(m -> hashmap.put(LocalDateTime.MIN, m));
hashmap is a HashMap<LocalDateTime, String>
Full stack error:
Exception in thread "main" java.io.UncheckedIOException: java.nio.charset.MalformedInputException: Input length = 1
at java.io.BufferedReader$1.hasNext(Unknown Source)
at java.util.Iterator.forEachRemaining(Unknown Source)
at java.util.Spliterators$IteratorSpliterator.forEachRemaining(Unknown Source)
at java.util.stream.ReferencePipeline$Head.forEachOrdered(Unknown Source)
at FileHandler.inputSavedDonations(FileHandler.java:92)
at StreamHelper.main(StreamHelper.java:36)
Caused by: java.nio.charset.MalformedInputException: Input length = 1
at java.nio.charset.CoderResult.throwException(Unknown Source)
at sun.nio.cs.StreamDecoder.implRead(Unknown Source)
at sun.nio.cs.StreamDecoder.read(Unknown Source)
at java.io.InputStreamReader.read(Unknown Source)
at java.io.BufferedReader.fill(Unknown Source)
at java.io.BufferedReader.readLine(Unknown Source)
at java.io.BufferedReader.readLine(Unknown Source)
... 6 more
Line 92 is .forEachOrdered(m -> hashmap.put(LocalDateTime.MIN, m));
fileName declaration: private static String fileName = "file";

Categories

Resources