Develop Reference

PySpark SparkConf() equivalent of spark command option "--jars"

I'd like to run some PySpark script on JupyterLab, and create custom UDF from JAR packages. To do so I need to broadcast these JAR packages to executor nodes. This answer has showed the command line interface approach (invoking --jars option in spark-submit). But I'd like to know the SparkConf() approach. On my JupyterLab sc.version=3.3.0-SNAPSHOT.
I'm very new to Spark.. your help will be highly appreciated!
Code:
import findspark
findspark.init()
findspark.find()
import pyspark
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
import os
# ------------ create spark session ------------
app_name = 'PySpark_Example'
path = os.getcwd()
conf = SparkConf().setAppName(os.environ.get('JUPYTERHUB_USER').replace(" ", "") + "_" + app_name).setMaster(
'spark://spark-master-svc.spark:7077')
command = os.popen("hostname -i")
hostname = command.read().split("\n")[0]
command.close()
conf.set("spark.scheduler.mode","FAIR")
conf.set("spark.deployMode","client")
conf.set("spark.driver.host",hostname)
conf.set('spark.extraListeners','sparkmonitor.listener.JupyterSparkMonitorListener')
conf.set("spark.jars", "{path}/my_func.jar,{path}/javabuilder.jar".format(path=path))
conf.set("spark.executor.extraClassPath", "{path}/".format(path=path))
sc = pyspark.SparkContext(conf=conf)
spark = SparkSession(sc)
spark._jsc.addJar("{}/my_func.jar".format(path))
spark._jsc.addJar("{}/javabuilder.jar".format(path))
# ------------- create sample dataframe ---------
sdf = spark.createDataFrame(
[
(1, 2.),
(2, 3.),
(3, 5.),
],
["col1", "col2"]
)
sdf.createOrReplaceTempView("temp_table")
# -------------- create UDF ----------------------
create_udf_from_jar = "CREATE OR REPLACE FUNCTION my_func AS 'my_func.Class1' " + \
"USING JAR '{}/my_func.jar'".format(path)
spark.sql(create_udf_from_jar)
spark.sql("SHOW USER FUNCTIONS").show()
# -------------- test ----------------------------
spark.sql("SELECT my_func(col1) FROM temp_table").show()
Error:
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
~tmp/ipykernel_4398/644670379.py in <cell line: 1>()
----> 1 spark.sql("SELECT my_func(col1) FROM temp_table").show()
~opt/spark/python/pyspark/sql/session.py in sql(self, sqlQuery, **kwargs)
1033 sqlQuery = formatter.format(sqlQuery, **kwargs)
1034 try:
-> 1035 return DataFrame(self._jsparkSession.sql(sqlQuery), self._wrapped)
1036 finally:
1037 if len(kwargs) > 0:
~opt/spark/python/lib/py4j-0.10.9.3-src.zip/py4j/java_gateway.py in __call__(self, *args)
1319
1320 answer = self.gateway_client.send_command(command)
-> 1321 return_value = get_return_value(
1322 answer, self.gateway_client, self.target_id, self.name)
1323
~opt/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
188 def deco(*a: Any, **kw: Any) -> Any:
189 try:
--> 190 return f(*a, **kw)
191 except Py4JJavaError as e:
192 converted = convert_exception(e.java_exception)
~opt/spark/python/lib/py4j-0.10.9.3-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
325 if answer[1] == REFERENCE_TYPE:
--> 326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
328 format(target_id, ".", name), value)
Py4JJavaError: An error occurred while calling o3813.sql.
: java.lang.NoClassDefFoundError: com/mathworks/toolbox/javabuilder/internal/MWComponentInstance

spark.jars is the one you're looking for (Doc)

JavaObject from Netlogo has no length using py4j?

I am running nl4py (a python module for NetLogo) in Jupyter notebook. I am trying to get import a list from netlogo into python, but the import is in a Java format. However, when I try to convert the JavaObject to a python format using py4j I get an error of: JavaObject has no len(). Is there a better way to convert JavaObject in python? Thanks.
python 3.8, ipython 7.10.0, nl4py 0.5.0, jdk 15.0.2, Netlogo 6.0, MacOS Catalina 10.15.7
#start of code for nl4py
import nl4py
nl4py.startServer("/Applications/NetLogo 6.0/")
n = nl4py.NetLogoApp()
n.openModel('/Users/tracykuper/Desktop/Netlogo models/Mucin project/1_21_20/PA_metabolite_model_1_21.nlogo')
n.command("setup")
#run abm model for n number of times
#change patch variable under a specific turtle
for i in range(1):
n.command("repeat 10 [go]")
#A = np.array([1,2,3,4],[3,2,-1,-6])) #turtle number, metabolite diff.
#run simulation of metabolic network to get biomass and metabolite values
#change patch variable under a specific turtle
names = ["1", "2", "3"] #turtle names
patch_values = ["-0.5", "50", "-0.5"] #metabolite values
for i in range(len(names)):
x = ('ask turtle {} [ask patch-here [set succinate succinate + {}]]'.format(names[i],patch_values[i]))
n.command(x)
#set new bacteria mass values
values = ["5", "30", "5"] #biomass values
y = ('ask turtle {} [set m m + {}]'.format(names[i],values[i]))
n.command(y)
n.command("ask turtle {} [set color red]".format(names[i]))
import py4j
mass = n.report("mass-list")
print(mass)
self = n.report("self-list")
type(mass)
s = py4j.protocol.get_return_value(mass, object)
[[0.69], [0.8], [0.73], [0.71], [0.5], [0.51], [0.54], [0.82], [0.72], [0.88]]
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-31-0b04d0127b47> in <module>
11 #map(mass + mass,mass)
12
---> 13 s = py4j.protocol.get_return_value(mass, object)
~/opt/anaconda3/envs/netlogo4/lib/python3.6/site-packages/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
319 (e.g., *hello* in `object1.hello()`). Optional.
320 """
--> 321 if is_error(answer)[0]:
322 if len(answer) > 1:
323 type = answer[1]
~/opt/anaconda3/envs/netlogo4/lib/python3.6/site-packages/py4j/protocol.py in is_error(answer)
372
373 def is_error(answer):
--> 374 if len(answer) == 0 or answer[0] != SUCCESS:
375 return (True, None)
376 else:
TypeError: object of type 'JavaObject' has no len()

Record Screen using ffmpeg doesn't work in tomcat server(Could not find codec parameters for stream 0)

I have the following problem,when I executed the below FFMPEG terminal command in command it prompt successfully.
C:\\ffmpeg\\bin\\ffmpeg.exe -f gdigrab -i desktop -c:v libx264 -analyzeduration 15M -probesize 15M -pix_fmt yuv420p -loglevel 99 C:\\ffmpeg\\bin\\video.mp4
but I have a problem while running a command from a java web application.I used runtime.exec(). It's working fine at Netbeans. But not working at all after deploying on tomcat server.
Here is my code :
String command = "C:\\ffmpeg\\bin\\"
+ "ffmpeg.exe -f gdigrab -i desktop -c:v libx264 -analyzeduration 15M -probesize 15M -pix_fmt yuv420p -loglevel 99 C:\\ffmpeg\\bin\\video.mp4";
System.out.println("command = " + command);
try {
Process p = Runtime.getRuntime().exec(command);
final InputStreamReader isr
= new InputStreamReader(p.getErrorStream());
Thread th = new Thread() {
public void run() {
try {
BufferedReader br = new BufferedReader(isr);
String line = null;
while ((line = br.readLine()) != null) {
System.out.println(line);
}
} catch (Exception ex) {
}
}
};
th.start();
} catch (IOException ex) {
System.out.println("Error 1 : " + ex.getMessage());
} catch (Exception ex) {
System.out.println("Error 2 : " + ex.getMessage());
}
And here is log for error
ffmpeg version 4.1.1 Copyright (c) 2000-2019 the FFmpeg developers
built with gcc 8.2.1 (GCC) 20190212
configuration: --enable-gpl --enable-version3 --enable-sdl2 --enable-fontconfig --enable-gnutls --enable-iconv --enable-libass --enable-libbluray --enable-libfreetype --enable-libmp3lame --enable-libopencore-amrnb --enable-libopencore-amrwb --enable-libopenjpeg --enable-libopus --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libtheora --enable-libtwolame --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx264 --enable-libx265 --enable-libxml2 --enable-libzimg --enable-lzma --enable-zlib --enable-gmp --enable-libvidstab --enable-libvorbis --enable-libvo-amrwbenc --enable-libmysofa --enable-libspeex --enable-libxvid --enable-libaom --enable-libmfx --enable-amf --enable-ffnvcodec --enable-cuvid --enable-d3d11va --enable-nvenc --enable-nvdec --enable-dxva2 --enable-avisynth
libavutil 56. 22.100 / 56. 22.100
libavcodec 58. 35.100 / 58. 35.100
libavformat 58. 20.100 / 58. 20.100
libavdevice 58. 5.100 / 58. 5.100
libavfilter 7. 40.101 / 7. 40.101
libswscale 5. 3.100 / 5. 3.100
libswresample 3. 3.100 / 3. 3.100
libpostproc 55. 3.100 / 55. 3.100
Splitting the commandline.
Reading option '-f' ... matched as option 'f' (force format) with argument 'gdigrab'.
Reading option '-i' ... matched as input url with argument 'desktop'.
Reading option '-c:v' ... matched as option 'c' (codec name) with argument 'libx264'.
Reading option '-analyzeduration' ... matched as AVOption 'analyzeduration'
with argument '15M'.
Reading option '-probesize' ... matched as AVOption 'probesize' with argument '15M'.
Reading option '-pix_fmt' ... matched as option 'pix_fmt' (set pixel format) with argument 'yuv420p'.
Reading option '-loglevel' ... matched as option 'loglevel' (set logging level) with argument '99'.
Reading option 'C:\ffmpeg\bin\video.mp4' ... matched as output url.
Finished splitting the commandline.
Parsing a group of options: global .
Applying option loglevel (set logging level) with argument 99.
Successfully parsed a group of options.
Parsing a group of options: input url desktop.
Applying option f (force format) with argument gdigrab.
Successfully parsed a group of options.
Opening an input file: desktop.
[gdigrab # 0000007c2a72ad00] Capturing whole desktop as 1024x768x32 at (0,0)
[gdigrab # 0000007c2a72ad00] Failed to capture image (error 5)
[gdigrab # 0000007c2a72ad00] stream 0: start_time: -9223372036854.775 duration: -9223372036854.775
[gdigrab # 0000007c2a72ad00] format: start_time: -9223372036854.775 duration: -9223372036854.775 bitrate=754233 kb/s
[gdigrab # 0000007c2a72ad00] Could not find codec parameters for stream 0 (Video: bmp, 1 reference frame, none, 754233 kb/s): unspecified size
Consider increasing the value for the 'analyzeduration' and 'probesize' options
Input #0, gdigrab, from 'desktop':
Duration: N/A, bitrate: 754233 kb/s
Stream #0:0, 0, 1/1000000: Video: bmp, 1 reference frame, none, 754233 kb/s, 29.97 fps, 1000k tbr, 1000k tbn, 1000k tbc
Successfully opened the file.
Parsing a group of options: output url C:\ffmpeg\bin\video.mp4.
Applying option c:v (codec name) with argument libx264.
Applying option pix_fmt (set pixel format) with argument yuv420p.
Successfully parsed a group of options.
Opening an output file: C:\ffmpeg\bin\video.mp4.
Output #0, mp4, to 'C:\ffmpeg\bin\video.mp4':
Output file #0 does not contain any stream
Can anyone help me to get out of this issue.
Thanks in advance.

How can I call a Go function from Java using the Java native interface?

It is possible to call C methods through the JNA interface in Java. How can I reach the same functionality with Go?
package main
import "fmt"
import "C"
//export Add
func Add(x, y int) int {
fmt.Printf("Go says: adding %v and %v\n", x, y)
return x + y
}

After review of the documentation about Go Shared Libraries:
It is possible to integrate the call of Go functions from Java Spring Batch. Below is a short example:
Go function:
package main
import "fmt"
import "C"
//export Add
func Add(x, y int) int {
fmt.Printf("Go says: adding %v and %v\n", x, y)
return x + y
}
After that, execute the command to generate the binary files:
go build -buildmode=c-shared -o bin/lib-cqm-transformer.so src/cqm_transformer.go
This generates the binary files:
ls -la bin/
total 2860
drwxrwxr-x 2 dmotta dmotta 4096 abr 23 01:13 .
drwxrwxr-x 5 dmotta dmotta 4096 abr 23 00:35 ..
-rw-r--r-- 1 root root 1558 abr 23 01:13 lib-cqm-transformer.h
-rw-r--r-- 1 root root 2915112 abr 23 01:13 lib-cqm-transformer.so
Finally, create the JNA class:
package com.XX.XX.batch.engine.transformer;
import com.sun.jna.Library;
import com.sun.jna.Native;
public class GoEngineTransformerTest {
static GoCqmTransformer GO_CQM_TRANSFORMER;
static {
String os = System.getProperty("os.name").toLowerCase();
String libExtension;
if (os.contains("mac os")) {
libExtension = "dylib";
} else if (os.contains("windows")) {
libExtension = "dll";
} else {
libExtension = "so";
}
String pwd = System.getProperty("user.dir");
String lib = pwd + "/golang/bin/lib-cqm-transformer." + libExtension;
GO_CQM_TRANSFORMER = (GoCqmTransformer) Native.loadLibrary(lib, GoCqmTransformer.class);
}
public interface GoCqmTransformer extends Library {
long Add(long x, long y);
}
public static void main(String[] args) {
System.out.println("Java says: about to call Go ..");
long total = GO_CQM_TRANSFORMER.Add(30, 12);
System.out.println("Java says: result is " + total);
}
}
After that, execute from the main Java class. Results:
Java HotSpot(TM) 64-Bit Server VM warning: You have loaded library /tmp/jna1412558273325390219.tmp which might have disabled stack guard. The VM will try to fix the stack guard now.
It's highly recommended that you fix the library with 'execstack -c <libfile>', or link it with '-z noexecstack'.
Java says: about to call Go ..
Go says: adding 30 and 12
Java says: result is 42

Updating module version when updating version in dependencies (multi-module maven )

My problem: versions-maven-plugin helps me to up version in some module (let's call it A) in my multi-module maven project.
Some modules (let's call it B and C) in this project have in dependencies module A. I need to up versions for this modules (B and C) too. Sometimes, i also need to up version in other module (B-parent) where B (or C) in dependencies (A version up -> B version up -> B-parent version up). Other problem is the modules can be at different levels of nesting.
Example:
root:
---B-parent:
---B (A in dependencies)
---C-parent
---C (A in dependencies)
---A-parent:
---A
Process: A version up -> A-parent version up, C version-up -> C-parent version-up, B version-up -> B-parent version up.
This plugin can't do this.
Is there any idea how this can be done?
Or my strategy of updating versions is not good enough?

I've made a script for increasing version numbers in all dependent modules recursively with a versions-maven-plugin.
Algorithm is as follows:
Run versions:set in target module
Run versions:set in all modules which have been updated by versions:set from previous step. If the module has been already processed - skip it.
Repeat step 2
Python 2.7 code
#!/usr/bin/env python
# -*- coding: utf-8 -*- #
# How To
#
# Run script and pass module path as a first argument.
# Or run it without arguments in module dir.
#
# Script will request the new version number for each module.
# If no version provided - last digit will be incremented (1.0.0 -> 1.0.1).
# cd <module-path>
# <project-dir>/increment-version.py
# ...
# review changes and commit
from subprocess import call, Popen, PIPE, check_output
import os
import re
import sys
getVersionCommand = "mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate " \
"-Dexpression=project.version 2>/dev/null | grep -v '\['"
def getCurrentModuleVersion():
return check_output(getVersionCommand, shell=True).decode("utf-8").split("\n")[0]
def incrementLastDigit(version):
digits = version.split(".")
lastDigit = int(digits[-1])
digits[-1] = str(lastDigit+1)
return ".".join(digits)
def isUpdatedVersionInFile(version, file):
return "<version>" + version + "</version>" in \
check_output("git diff HEAD --no-ext-diff --unified=0 --exit-code -a --no-prefix {} "
"| egrep \"^\\+\"".format(file), shell=True).decode("utf-8")
def runVersionSet(version):
process = Popen(["mvn", "versions:set", "-DnewVersion="+version, "-DgenerateBackupPoms=false"], stdout=PIPE)
(output, err) = process.communicate()
exitCode = process.wait()
if exitCode is not 0:
print "Error setting the version"
exit(1)
return output, err, exitCode
def addChangedPoms(version, dirsToVisit, visitedDirs):
changedFiles = check_output(["git", "ls-files", "-m"]) \
.decode("utf-8").split("\n")
changedPoms = [f for f in changedFiles if f.endswith("pom.xml")]
changedDirs = [os.path.dirname(os.path.abspath(f)) for f in changedPoms if isUpdatedVersionInFile(version, f)]
changedDirs = [d for d in changedDirs if d not in visitedDirs and d not in dirsToVisit]
print "New dirs to visit:", changedDirs
return changedDirs
if __name__ == "__main__":
visitedDirs = []
dirsToVisit = []
if len(sys.argv) > 1:
if os.path.exists(os.path.join(sys.argv[1], "pom.xml")):
dirsToVisit.append(os.path.abspath(sys.argv[1]))
else:
print "Error. No pom.xml file in dir", sys.argv[1]
exit(1)
else:
dirsToVisit.append(os.path.abspath(os.getcwd()))
pattern = re.compile("aggregation root: (.*)")
while len(dirsToVisit) > 0:
dirToVisit = dirsToVisit.pop()
print "Visiting dir", dirToVisit
os.chdir(dirToVisit)
currentVersion = getCurrentModuleVersion()
defaultVersion = incrementLastDigit(currentVersion)
version = raw_input("New version for {}:{} ({}):".format(dirToVisit, currentVersion, defaultVersion))
if not version.strip():
version = defaultVersion
print "New version:", version
output, err, exitcode = runVersionSet(version)
rootDir = pattern.search(output).group(1)
visitedDirs = visitedDirs + [dirToVisit]
os.chdir(rootDir)
print "Adding new dirs to visit"
dirsToVisit = dirsToVisit + addChangedPoms(version, dirsToVisit, visitedDirs)