Develop Reference

Using Stax2 to escape special characters in Scala

I am trying to use stax2 in order to write xml files with escaping special characters for the attributes.
When I am trying to achieve is an exact output like this:
<elem1 att1="This
That" />
But when I use the usual XMLStreamWriter is this:
<elem1 att1="This &#x0A; That" />
So I tried the following with stax2 :
import org.codehaus.stax2.{XMLOutputFactory2}
import org.scalatest.funsuite.AnyFunSuite
import java.io.{File, FileOutputStream}
import javax.xml.stream.{XMLOutputFactory, XMLStreamWriter}
class testStreamXML extends AnyFunSuite{
val file = new File("stax2test.xml")
val fileOutputStream = new FileOutputStream(file)
val outputFactory: XMLOutputFactory2 = XMLOutputFactory.newInstance().asInstanceOf[XMLOutputFactory2]
//outputFactory.setProperty(XMLOutputFactory2.P_ATTR_VALUE_ESCAPER, true)
val writer= outputFactory.createXMLStreamWriter(fileOutputStream)
writer.writeStartDocument()
writer.writeStartElement("elem1")
writer.writeAttribute("att1", "This
That")
writer.writeEndElement()
writer.writeEndDocument()
}
And whenever i try to set the property P_ATTR_VALUE_ESCAPER to true or false,I receive this error:
An exception or error caused a run to abort: class java.lang.Boolean cannot be cast to class org.codehaus.stax2.io.EscapingWriterFactory (java.lang.Boolean is in module java.base of loader 'bootstrap'; org.codehaus.stax2.io.EscapingWriterFactory is in unnamed module of loader 'app')
java.lang.ClassCastException: class java.lang.Boolean cannot be cast to class org.codehaus.stax2.io.EscapingWriterFactory (java.lang.Boolean is in module java.base of loader 'bootstrap'; org.codehaus.stax2.io.EscapingWriterFactory is in unnamed module of loader 'app')
at com.ctc.wstx.api.WriterConfig.setProperty(WriterConfig.java:401)
at com.ctc.wstx.api.CommonConfig.setProperty(CommonConfig.java:100)
at com.ctc.wstx.stax.WstxOutputFactory.setProperty(WstxOutputFactory.java:153)
at testStreamXML3.<init>(testStreamXML3.scala:10)
at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:64)
at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:500)
at java.base/java.lang.reflect.ReflectAccess.newInstance(ReflectAccess.java:128)
at java.base/jdk.internal.reflect.ReflectionFactory.newInstance(ReflectionFactory.java:350)
at java.base/java.lang.Class.newInstance(Class.java:645)
at org.scalatest.tools.Runner$.genSuiteConfig(Runner.scala:1402)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$8(Runner.scala:1199)
at scala.collection.immutable.List.map(List.scala:246)
at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1198)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:993)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:971)
at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1480)
at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:971)
at org.scalatest.tools.Runner$.run(Runner.scala:798)
at org.scalatest.tools.Runner.run(Runner.scala)
at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2or3(ScalaTestRunner.java:38)
at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:25)
Any suggestion how to use resolve this? or to achieve my goal of escaping special characters in attribute?

The property you are referring to does require a class of EscapingWriterFactory. Here are the docs:
Property that can be set if a custom output escaping for attribute value content is needed. The value set needs to be of type
EscapingWriterFactory. When set, the factory will be used to create a
per-writer instance used to escape all attribute values written, both
via explicit XMLStreamWriter.writeAttribute(java.lang.String,
java.lang.String) methods, and via copy methods
(XMLStreamWriter2.copyEventFromReader(org.codehaus.stax2.XMLStreamReader2,
boolean)). [1]
Regarding your question of how to achieve an 'individual' escaping an implementation of this factory would do the job. Here is a simple implementation (inspired by [2]) using the given writer without applying any escaping - this might be your starting point for any special use case you want to address:
class CustomXmlEscapingWriterFactory extends EscapingWriterFactory{
override def createEscapingWriterFor(writer: Writer, s: String): Writer =
new Writer(){
override def write(cbuf: Array[Char], off: Int, len: Int): Unit =
writer.write(cbuf, off, len)
override def flush(): Unit = writer.flush()
override def close(): Unit = writer.close()
}
override def createEscapingWriterFor(outputStream: OutputStream, s: String): Writer =
throw IllegalArgumentException("not supported")
}
class TestStreamXML extends AnyFunSuite{
val file = new File("stax2test.xml")
val fileOutputStream = new FileOutputStream(file)
val oprovider: OutputFactoryProviderImpl = new OutputFactoryProviderImpl()
val outputFactory: XMLOutputFactory2 = oprovider.createOutputFactory()
// your factory implementation goes here as property
outputFactory.setProperty(XMLOutputFactory2.P_ATTR_VALUE_ESCAPER, CustomXmlEscapingWriterFactory())
val writer= outputFactory.createXMLStreamWriter(fileOutputStream)
writer.writeStartDocument()
writer.writeStartElement("elem1")
writer.writeAttribute("att1", "This
That")
writer.writeEndElement()
writer.writeEndDocument()
}
The resulting output looks like this:
<?xml version='1.0' encoding='UTF-8'?><elem1 att1="This
That"/>
[1] https://fasterxml.github.io/stax2-api/javadoc/4.0.0/org/codehaus/stax2/XMLOutputFactory2.html#P_ATTR_VALUE_ESCAPER
[2] Escaping quotes using jackson-dataformat-xml

how to parse postman-collection?

Goal:
I am trying to parse the postman_echo collection json and persist the result into a new json copy on disk, resulting the same file as the original one.
I prefer built-in data structure from the language, but using json library should be good too. Not sure Antlr4 is a better way.
follow-up question
is it possible to allow any valid nested json in the body of post request?
update:
https://github.com/chakpongchung/postman-parser
In the end we come up with this satisfactory solution.

An alternative to what zoran mentioned is to create a case class if the structure is not too dynamic (with Play JSON). This would make it easier to compare the result.
case class MyObject(
queryString: List[KeyValue],
method: String,
url: String,
httpVersion: String
) ... and so on
object MyObject {
implicit val format: Format[MyObject] = Json.format
}
case class KeyValue(name: String, value: String)
object KeyValue {
implicit val format: Format[KeyValue] = Json.format
}
Then, you just need to do:
object JsonParser extends App {
val postman_collections = "./scala_input.json"
val jsonifiedString = scala.io.Source.fromFile(postman_collections).mkString
val myJsonData = Try(Json.parse(jsonifiedString)).map(_.as[MyObject])
myJsonData match {
case Success(myValue) => // compare your case class here
case Failure(err) => println("none")
}
}

I'm not sure if I understand your question well, but if you are trying to iterate over json string, you might try something like this:
import play.api.libs.json.{JsObject, JsValue, Json}
import scala.util.{Failure, Success, Try}
object JsonParser extends App {
val postman_coolections = "./resources/scala_input.json"
val jsonifiedString = scala.io.Source.fromFile(postman_coolections).mkString
val json: JsValue = Try(Json.parse(jsonifiedString)) match {
case Success(js) => js
case Failure(ex) => throw new Exception("Couldn't parse json", ex)
}
json.asInstanceOf[JsObject].fields.foreach{
case (key: String, value: JsValue)=>
println(s"Key:$key value:${value.toString}")
writeFile(s"$key.json", Json.prettyPrint(value))
}
//writing the whole postman input as a single file
writeFile("postmanInputFormatted.json", Json.prettyPrint(json))
writeFile("postmanInput.json", Json.stringify(json))
// To access individual property one option is to use this approach
val lookedValue = json \ "postData" \ "params" \ 1 \ "hello" \ "test"
lookedValue match {
case JsDefined(value) => println(s"Test value is $value")
case JsUndefined() => println("Didn't find test value")
}
// or
val lookedValueAlt = (json \ "postData" \ "params" \ 1 \ "hello" \ "test").getOrElse(throw SomeException)

There are multiple problems in your parser and most of them are that you are trying to use default parser to handle Json object as string. For example, in Request you are handling header as Seq[String] when it's actually Seq of (key, value) pairs. For this particular case, you should change it to something like this:
case class Request(
method: String,
header: Seq[HeaderItem], // header: []
url: Option[Url] = None,
description: String = ""
)
object Request {
implicit val format: Format[Request] = Json.using[Json.WithDefaultValues].format
case class HeaderItem(key: String, value: String)
object HeaderItem {
implicit val format: Format[HeaderItem] = Json.format
}
You can convert header to Seq[String] if you want, but you will have to write custom Read for that.
In the above case you also have cases when description is missing, so you want to handle that with default values.
You have such problems to handle in few other places as well, e.g. "response".
Another problem that I've noticed is a way how you handle "type" property from Json string. Type is reserved keyword, and you can handle it by wrapping it in ``, e.g.
case class Script(
`type`: String,
exec: Seq[String]
)

A satisfactory solution is posted in the github link above in the question.

UDF to extract only the file name from path in Spark SQL

There is input_file_name function in Apache Spark which is used by me to add new column to Dataset with the name of file which is currently being processed.
The problem is that I'd like to somehow customize this function to return only file name, ommitting the full path to it on s3.
For now, I am doing replacement of the path on the second step using map function:
val initialDs = spark.sqlContext.read
.option("dateFormat", conf.dateFormat)
.schema(conf.schema)
.csv(conf.path).withColumn("input_file_name", input_file_name)
...
...
def fromFile(fileName: String): String = {
val baseName: String = FilenameUtils.getBaseName(fileName)
val tmpFileName: String = baseName.substring(0, baseName.length - 8) //here is magic conversion ;)
this.valueOf(tmpFileName)
}
But I'd like to use something like
val initialDs = spark.sqlContext.read
.option("dateFormat", conf.dateFormat)
.schema(conf.schema)
.csv(conf.path).withColumn("input_file_name", **customized_input_file_name_function**)

In Scala:
#register udf
spark.udf
.register("get_only_file_name", (fullPath: String) => fullPath.split("/").last)
#use the udf to get last token(filename) in full path
val initialDs = spark.read
.option("dateFormat", conf.dateFormat)
.schema(conf.schema)
.csv(conf.path)
.withColumn("input_file_name", get_only_file_name(input_file_name))
Edit: In Java as per comment
#register udf
spark.udf()
.register("get_only_file_name", (String fullPath) -> {
int lastIndex = fullPath.lastIndexOf("/");
return fullPath.substring(lastIndex, fullPath.length - 1);
}, DataTypes.StringType);
import org.apache.spark.sql.functions.input_file_name
#use the udf to get last token(filename) in full path
Dataset<Row> initialDs = spark.read()
.option("dateFormat", conf.dateFormat)
.schema(conf.schema)
.csv(conf.path)
.withColumn("input_file_name", get_only_file_name(input_file_name()));

Borrowing from a related question here, the following method is more portable and does not require a custom UDF.
Spark SQL Code Snippet: reverse(split(path, '/'))[0]
Spark SQL Sample:
WITH sample_data as (
SELECT 'path/to/my/filename.txt' AS full_path
)
SELECT
full_path
, reverse(split(full_path, '/'))[0] as basename
FROM sample_data
Explanation:
The split() function breaks the path into it's chunks and reverse() puts the final item (the file name) in front of the array so that [0] can extract just the filename.
Full Code example here :
spark.sql(
"""
|WITH sample_data as (
| SELECT 'path/to/my/filename.txt' AS full_path
| )
| SELECT
| full_path
| , reverse(split(full_path, '/'))[0] as basename
| FROM sample_data
|""".stripMargin).show(false)
Result :
+-----------------------+------------+
|full_path |basename |
+-----------------------+------------+
|path/to/my/filename.txt|filename.txt|
+-----------------------+------------+

commons io is natural/easiest import in spark means(no need to add additional dependency...)
import org.apache.commons.io.FilenameUtils
getBaseName(String fileName)
Gets the base name, minus the full path and extension, from a full fileName.
val baseNameOfFile = udf((longFilePath: String) => FilenameUtils.getBaseName(longFilePath))
Usage is like...
yourdataframe.withColumn("shortpath" ,baseNameOfFile(yourdataframe("input_file_name")))
.show(1000,false)

Calling Java library method with (Object ...) arg list from Scala , retaining type sanity (Shapeless / Scalaz)

I am using the datastax java drver for Cassandra from scala (2.10.4)
to build batches of prepared statements
but have hit the following problem.
Table definition in CQL
use ks;
drop table bc_test;
create table bc_test(
id TEXT,
id_1 BIGINT,
c1 COUNTER,
c2 COUNTER,
PRIMARY KEY(id, id_1)
);
First problem is implicit numeric widening error
import com.datastax.driver.core.{ResultSet, ProtocolOptions, Cluster, Session}
import com.datastax.driver.core.RegularStatement
import com.datastax.driver.core.PreparedStatement
import com.datastax.driver.core.BoundStatement
import com.datastax.driver.core.BatchStatement
val c = Cluster.builder().addContactPoint("127.0.0.1").withPort(9042).build
val sess = c.connect("ks")
val p_bc_test_c1: PreparedStatement =
sess.prepare("UPDATE bc_test SET c1 = c1 + ? WHERE id = ? and id_1 = ?")
val batch: BatchStatement = new BatchStatement(BatchStatement.Type.COUNTER);
val id1: Long = 1L
val idText: String = "one"
val c1: Long = 1L
batch.add(p_bc_test_c1.bind(c1, "one", id1))
// which gives the error
scala> batch.add(p_bc_test_c1.bind(c1, "one", id1))
<console>:19: error: implicit numeric widening
batch.add(p_bc_test_c1.bind(c1, "one", id1))
^
I can get around this using type ascription but wanted to find something a bit nicer
that can also be built into a more generic solution. What I came up with was :
import scalaz._
import Scalaz._
import shapeless._
import poly._
import syntax.std.tuple._
trait CassandraLongVal
type CassandraLong = java.lang.Long ## CassandraLongVal
def cassandraLongCol(c : Long): CassandraLong = Tag(c)
trait CassandraIntVal
type CassandraInt = java.lang.Integer ## CassandraIntVal
def cassandraIntCol(c : Int): CassandraInt = Tag(c)
object ToCassandraType extends Poly1 {
implicit def caseInt = at[Int](cassandraIntCol(_))
implicit def caseLong = at[Long](cassandraLongCol(_))
implicit def caseString = at[String](identity)
}
case class Update1(cval: Long, id: String, id1: Long)
val update1Gen = Generic[Update1]
val bUpdate1 = Update1(125L, "two", 1L)
val update1AsArray = update1Gen.to(bUpdate1).map(ToCassandraType).toArray
batch.add(p_bc_test_c1.bind(update1AsArray:_*))
batch.add(p_bc_test_c1.bind(update1AsArray:_*))
sess.execute(batch)
This seems to work quite happily and I can use the basic pattern for dozens of table with different columns.
Is this a reasonable approach, or is there a simpler way to get the same outcome whilst trying to keep a handle on the types
and am I abusing scalaz/ shapeless through my (considerable) ignorance.

Devise confirmation tokens in Java

Can't seem to create a functional way to insert a user from Java for Devise. Currently there are these fields:
"_id",
"access_level",
"confirmation_sent_at",
"confirmation_token",
"confirmed_at",
"email",
"encrypted_password",
"sign_in_count"
I am able to insert a document that counts as a user. The problem is that when I go to:
http://www.mysite.com:3000/users/confirmation?confirmation_token=TOKENHERE
I get a message saying that it's invalid.
EDIT 1:
When I resend confirmation instructions for this user (WHICH GENERATES A NEW TOKEN), the user can be logged into. This confirms my doubts about the token being the problem. How can I port Devise's token generator to Java?
EDIT 2:
When I register on site, it says I should check for a confirmation link. However, if I go into the Mongo shell, manually take out the confirmation token and paste it to site.com/users/confirmation?confirmation_token= then it doesn't work! However, if I actually use the confirmation link I was sent, it works. How can I make a VALID token, all from Java. Please help!

For this quoestion you should refer to this stackoverflow answer and to the Rails API of protect_from_forgery.
The short answer is to disable forgery protection in your controller, but this makes your application vulnerable to CSRF attacks:
skip_before_action :verify_authenticity_token
The better way would be to authenticate with a JSON or XML request as these requests are not protected by CSRF protection. You can find a solution for devise here.
Edit
Monkey patch devise to save unencoded confirmation token. In your config/initializers/devise.rb
module Devise
module Models
module Confirmable
def generate_confirmation_token
raw, enc = Devise.token_generator.generate(self.class, :confirmation_token)
#raw_confirmation_token = raw
self.my_unencoded_column = raw # Patch
self.confirmation_token = enc
self.confirmation_sent_at = Time.now.utc
end
end
end
end

In case anyone else finds themselves trying to get a java or scala app to coexist with a rails app, I hacked up the following. Its in scala but uses java apis so should be easy to read. As far as I can tell it replicates Devise's behavior, and if I hit the confirmation link in the rails app with the raw token rails/devise generates the same encoded string.
import java.security.spec.KeySpec
import javax.crypto.SecretKey
import javax.crypto.SecretKeyFactory
import javax.crypto.spec.PBEKeySpec
import javax.crypto.spec.SecretKeySpec
import javax.crypto.Mac
import javax.xml.bind.DatatypeConverter
import java.util.Base64
// copy functionality from Rails Devise
object TokenGenerator {
// sample values 9exithzwZ8P9meqdVs3K => 54364224169895883e87c8412be5874039b470e26e762cb3ddc37c0bdcf014f5
// 5zNMi6egbyPoDUy2t3NY => 75bd5d53aa36d3fc61ac186b4c6e2be8353e6b39536d3cf846719284e05474ca
private val deviseSecret = sys.env("DEVISE_SECRET")
private val factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1")
val encoder = Base64.getUrlEncoder()
case class TokenInfo(raw: String, encoded: String)
def createConfirmationToken: TokenInfo = {
// copy behavior from rails world. Don't know why it does this
val replacements = Map('l' -> "s", 'I' -> "x", 'O' -> "y", '0' -> "z")
// make a raw key of 20 chars, doesn't seem to matter what they are, just need url valid set
val bytes = new Array[Byte](16)
scala.util.Random.nextBytes(bytes)
val raw = encoder.encodeToString(bytes).take(20).foldLeft(""){(acc, x) => acc ++ replacements.get(x).getOrElse(x.toString)}
TokenInfo(raw, digestForConfirmationToken(raw))
}
private def generateKey(salt: String): Array[Byte] = {
val iter = 65536
val keySize = 512
val spec = new PBEKeySpec(deviseSecret.toCharArray, salt.getBytes("UTF-8"), iter, keySize)
val sk = factory.generateSecret(spec)
val skspec = new SecretKeySpec(sk.getEncoded, "AES")
skspec.getEncoded
}
def sha256HexDigest(s: String, key: Array[Byte]): String = {
val mac = Mac.getInstance("HmacSHA256")
val keySpec = new SecretKeySpec(key, "RAW")
mac.init(keySpec)
val result: Array[Byte] = mac.doFinal(s.getBytes())
DatatypeConverter.printHexBinary(result).toLowerCase
}
private def getDigest(raw: String, salt: String) = sha256HexDigest(raw, generateKey(salt))
// devise uses salt "Devise #{column}", in this case its confirmation_token
def digestForConfirmationToken(raw: String) = getDigest(raw, "Devise confirmation_token")
}