I wrote a small app that pings a remote server by sending ICMP packets using sockets (Android's Os package). I tested it on the API 28 emulator and it worked, but it does not work on API 22 (Android 5.1 Lollipop). I tested it on the VD as well as on my smartphone (same old API) with the same negative result
android.system.ErrnoException: sendto failed: EINVAL (Invalid argument)
I debugged the app on different API versions and the only difference I noticed that ByteBuffer.wrap() produces ByteArrayBuffer instance on API 22 and HeapArrayBuffer instance on newer versions. The payload itself seems to be the same.
Here is the code sample (simplified). Also, there is a test application available: https://github.com/alexeysirenko/android-sockets-icmp-ping-test
You could try to launch it on the different emulators (API 22 and >22) and see the difference
private val timeoutMs = 5000
private val delayMs = 500L
private val ECHO_PORT = 80
private val POLLIN = (if (OsConstants.POLLIN == 0) 1 else OsConstants.POLLIN).toShort()
fun ping(host: String): Unit {
val inetAddress: InetAddress = InetAddress.getByName(host)
if (inetAddress is Inet6Address) throw Exception("IPv6 implementation omitted for simplicity")
val proto = OsConstants.IPPROTO_ICMP
val inet = OsConstants.AF_INET
val type = PacketBuilder.TYPE_ICMP_V4
val socketFileDescriptor = Os.socket(inet, OsConstants.SOCK_DGRAM, proto)
if (!socketFileDescriptor.valid()) throw Exception("Socket descriptor is invalid")
var sequenceNumber: Short = 0
for (i in 0..2) {
sequenceNumber++
val echoPacketBuilder =
PacketBuilder(type, "foobarbazquok".toByteArray())
.withSequenceNumber(sequenceNumber)
val buffer = echoPacketBuilder.build()
/**
* This is the command that throws an exception
*/
val bytesSent = Os.sendto(socketFileDescriptor, buffer,0, buffer.size, 0, inetAddress, ECHO_PORT)
// Response processing code omitted
}
}
class PacketBuilder(val type: Byte, val payload: ByteArray, val sequenceNumber: Short = 0, val identifier: Short = 0xDBB) {
private val MAX_PAYLOAD = 65507
private val CODE: Byte = 0
init {
if (payload.size > MAX_PAYLOAD) throw Exception("Payload limited to $MAX_PAYLOAD")
}
fun build(): ByteArray {
val buffer = ByteArray(8 + payload.size)
val byteBuffer = ByteBuffer.wrap(buffer)
byteBuffer.put(type)
byteBuffer.put(CODE)
val checkPos = byteBuffer.position()
byteBuffer.position(checkPos + 2)
byteBuffer.putShort(identifier)
byteBuffer.putShort(sequenceNumber)
byteBuffer.put(payload)
byteBuffer.putShort(checkPos, checksum(buffer))
byteBuffer.flip()
return buffer
}
fun withSequenceNumber(sequenceNumber: Short): PacketBuilder {
return PacketBuilder(type, payload, sequenceNumber, identifier)
}
/**
* RFC 1071 checksum
*/
private fun checksum(data: ByteArray): Short {
var sum = 0
// High bytes (even indices)
for (i in 0 until data.size step 2) {
sum += data[i].and(0xFF.toByte()).toInt() shl 8
sum = (sum and 0xFFFF) + (sum shr 16)
}
// Low bytes (odd indices)
for (i in 1 until data.size step 2) {
sum += data[i] and 0xFF.toByte()
sum = (sum and 0xFFFF) + (sum shr 16)
}
sum = (sum and 0xFFFF) + (sum shr 16)
return (sum xor 0xFFFF).toShort()
}
companion object {
val TYPE_ICMP_V4: Byte = 8
}
}
If my code is incorrect then I'd expect the same error on all platforms, but as I said it does work on all APIs newer than 22 and I do not know exactly what causes this issue
I am new to scala and java altogether and trying to run a sample producer code. All it does is, takes some raw products and referrers stored in csv files and uses rnd to generate some random log. Following is my code:
object LogProducer extends App {
//WebLog config
val wlc = Settings.WebLogGen
val Products = scala.io.Source.fromInputStream(getClass.getResourceAsStream("/products.csv")).getLines().toArray
val Referrers = scala.io.Source.fromInputStream(getClass.getResourceAsStream("/referrers.csv")).getLines().toArray
val Visitors = (0 to wlc.visitors).map("Visitors-" + _)
val Pages = (0 to wlc.pages).map("Pages-" + _)
val rnd = new Random()
val filePath = wlc.filePath
val fw = new FileWriter(filePath, true)
//adding randomness to time increments for demo
val incrementTimeEvery = rnd.nextInt(wlc.records - 1) + 1
var timestamp = System.currentTimeMillis()
var adjustedTimestamp = timestamp
for (iteration <- 1 to wlc.records) {
adjustedTimestamp = adjustedTimestamp + ((System.currentTimeMillis() - timestamp) * wlc.timeMultiplier)
timestamp = System.currentTimeMillis()
val action = iteration % (rnd.nextInt(200) + 1) match {
case 0 => "purchase"
case 1 => "add_to_cart"
case _ => "page_view"
}
val referrer = Referrers(rnd.nextInt(Referrers.length - 1))
val prevPage = referrer match {
case "Internal" => Pages(rnd.nextInt(Pages.length - 1))
case _ => ""
}
val visitor = Visitors(rnd.nextInt(Visitors.length - 1))
val page = Pages(rnd.nextInt(Pages.length - 1))
val product = Products(rnd.nextInt(Products.length - 1))
val line = s"$adjustedTimestamp\t$referrer\t$action\t$prevPage\t$visitor\t$page\t$product\n"
fw.write(line)
if (iteration % incrementTimeEvery == 0) {
//os.flush()
println(s"Sent $iteration messages!")
val sleeping = rnd.nextInt(incrementTimeEvery * 60)
println(s"Sleeping for $sleeping ms")
}
}
}
It is pretty straightforward where it is basically generating some variables and adding it to the line.
However I am getting a big exception error stack which i am not able to understand:
"C:\Program Files\Java\jdk1.8.0_92\bin\java...
Exception in thread "main" java.nio.charset.MalformedInputException: Input length = 1
at java.nio.charset.CoderResult.throwException(CoderResult.java:281)
at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:339)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.fill(BufferedReader.java:161)
at java.io.BufferedReader.readLine(BufferedReader.java:324)
at java.io.BufferedReader.readLine(BufferedReader.java:389)
at scala.io.BufferedSource$BufferedLineIterator.hasNext(BufferedSource.scala:70)
at scala.collection.Iterator.foreach(Iterator.scala:929)
at scala.collection.Iterator.foreach$(Iterator.scala:929)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1417)
at scala.collection.generic.Growable.$plus$plus$eq(Growable.scala:59)
at scala.collection.generic.Growable.$plus$plus$eq$(Growable.scala:50)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)
at scala.collection.TraversableOnce.to(TraversableOnce.scala:310)
at scala.collection.TraversableOnce.to$(TraversableOnce.scala:308)
at scala.collection.AbstractIterator.to(Iterator.scala:1417)
at scala.collection.TraversableOnce.toBuffer(TraversableOnce.scala:302)
at scala.collection.TraversableOnce.toBuffer$(TraversableOnce.scala:302)
at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1417)
at scala.collection.TraversableOnce.toArray(TraversableOnce.scala:289)
at scala.collection.TraversableOnce.toArray$(TraversableOnce.scala:283)
at scala.collection.AbstractIterator.toArray(Iterator.scala:1417)
at clickstream.LogProducer$.delayedEndpoint$clickstream$LogProducer$1(logProducer.scala:16)
at clickstream.LogProducer$delayedInit$body.apply(logProducer.scala:12)
at scala.Function0.apply$mcV$sp(Function0.scala:34)
at scala.Function0.apply$mcV$sp$(Function0.scala:34)
at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
at scala.App.$anonfun$main$1$adapted(App.scala:76)
at scala.collection.immutable.List.foreach(List.scala:389)
at scala.App.main(App.scala:76)
at scala.App.main$(App.scala:74)
at clickstream.LogProducer$.main(logProducer.scala:12)
at clickstream.LogProducer.main(logProducer.scala)
Process finished with exit code 1
Can someone please help me identify what the exception mean? Thanks all
So it wasnt hard.. it was my amateurish knowledge. It was a simple IO exception where Intellij wasnt able to get the values from my csv file. When i imported it into resources root directory, it gave me a warning message of wrong encoding.
The error was at this point:
val Products = scala.io.Source.fromInputStream(getClass.getResourceAsStream("/products.csv")).getLines().toArray
thanks for efforts though
It was an encoding issue, for Scala a quick fix would be:
replace:
val Products=scala.io.Source.fromInputStream(getClass.getResourceAsStream("/products.csv")).getLines().toArray
val Referrers = scala.io.Source.fromInputStream(getClass.getResourceAsStream("/referrers.csv")).getLines().toArray
using this:
val Products=scala.io.Source.fromInputStream(getClass.getResourceAsStream("/products.csv"))("UTF-8").getLines().toArray
val Referrers = scala.io.Source.fromInputStream(getClass.getResourceAsStream("/referrers.csv"))("UTF-8").getLines().toArray
For java and more details please check out this link: http://biercoff.com/malformedinputexception-input-length-1-exception-solution-for-scala-and-java/
I am trying to write a scala program to generate an output file omega0_Real.txt which contains pre-calculated values of the cosine function for inputs ranging from 0 to pi/2 radians. Each of these calculated values are 72 bit long and are stored in hex format. The code I have written so far is as follows:
import java.io._
import scala.math._
object omega0_Real {
def main (args: Array[String]) {
val arg = (0.0).to(2-pow(2, -10), pow(2, -10))
val cosArg = arg.map (i => cos(i))
val cosBit = cosArg.map (i => List.tabulate(72)(j = (BigDecimal((i*pow(2,j))).toBigInt % 2)))
val cosStr = cosBit.map (i => i mkString)
val cosBig = cosStr.map (i => BigInt(i, 2))
val cosBigStr = cosBig.map (i => i.toString(16))
val cosList = cosBigStr.toList
val file = "omega0_Real.txt"
val writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)))
for (x <- cosList) {
writer.write(x + "\n")
}
writer.close()
}
which gives the error: java.lang.NumberFormatException: Illegal embedded sign character followed by many others. Please help me debug this code.
PS: I ran this code line-by-line on sbt console but it did not give any error, although the values generated were erroneous.
when I calculate the Variance of my data, I have to collect first, Is there any other methods?
my data format:
1 2 3
1 4 5
4 5 6
4 7 8
7 8 9
10 11 12
10 13 14
10 1 2
1 100 100
10 11 2
10 11 2
1 2 5
4 7 6
code:
val conf = new SparkConf().setAppName("hh")
conf.setMaster("local[3]")
val sc = new SparkContext(conf)
val data = sc.textFile("/home/hadoop4/Desktop/i.txt")
.map(_.split("\t")).map(f => f.map(f => f.toDouble))
.map(f => ("k"+f(0),f(1)))
//data:RDD[(String,Double)]
val dataArr = data.map(f=>(f._1,ArrayBuffer(f._2)))
//dataArr RDD[(String,ArrayBuffer[Double])]
dataArr.collect().foreach(println(_))
//output
(k1.0,ArrayBuffer(2.0))
(k1.0,ArrayBuffer(4.0))
(k4.0,ArrayBuffer(5.0))
(k4.0,ArrayBuffer(7.0))
(k7.0,ArrayBuffer(8.0))
(k10.0,ArrayBuffer(11.0))
(k10.0,ArrayBuffer(13.0))
(k10.0,ArrayBuffer(1.0))
(k1.0,ArrayBuffer(100.0))
(k10.0,ArrayBuffer(11.0))
(k10.0,ArrayBuffer(11.0))
(k1.0,ArrayBuffer(2.0))
(k4.0,ArrayBuffer(7.0))
val dataArrRed = dataArr.reduceByKey((x,y)=>x++=y)
//dataArrRed :RDD[(String,ArrayBuffer[Double])]
dataArrRed.collect().foreach(println(_))
//output
(k1.0,ArrayBuffer(2.0, 4.0, 100.0, 2.0))
(k7.0,ArrayBuffer(8.0))
(k10.0,ArrayBuffer(11.0, 13.0, 1.0, 11.0, 11.0))
(k4.0,ArrayBuffer(5.0, 7.0, 7.0))
val dataARM = dataArrRed.collect().map(
f=>(f._1,sc.makeRDD(f._2,2)))
val dataARMM = dataARM.map(
f=>(f._1,(f._2.variance(),f._2.max(),f._2.min())))
.foreach(println(_))
sc.stop()
//output
(k1.0,(1777.0,100.0,2.0))
(k7.0,(0.0,8.0,8.0))
(k10.0,(18.24,13.0,1.0))
(k4.0,(0.8888888888888888,7.0,5.0))
//update ,now I calculate the second column and the third column at the same time and put them into an Array(f(1),f(2)), then turned into an RDD and aggregateByKey with it, the 'zero values' is Array(new StatCounter(),new StatCounter()),it has some problem.
val dataArray2 = dataString.split("\\n")
.map(_.split("\\s+")).map(_.map(_.toDouble))
.map(f => ("k" + f(0), Array(f(1),f(2))))
val data2 = sc.parallelize(dataArray2)
val dataStat2 = data2.aggregateByKey(Array(new StatCounter(),new
StatCounter()))
({
(s,v)=>(
s(0).merge(v(0)),s(1).merge(v(1))
)
},{
(s,t)=>(
s(0).merge(v(0)),s(1).merge(v(1))
)})
it's wrong. Can I use Array(new StatCounter(),new StatCounter())? Thanks.
Worked example. It turns out to be a one-liner, and another line to map it into the OP's format.
Slightly different way of getting the data (more convenient for testing but same result)
val dataString = """1 2 3
1 4 5
4 5 6
4 7 8
7 8 9
10 11 12
10 13 14
10 1 2
1 100 100
10 11 2
10 11 2
1 2 5
4 7 6
""".trim
val dataArray = dataString.split("\\n")
.map(_.split("\\s+")).map(_.map(_.toDouble))
.map(f => ("k" + f(0), f(1)))
val data = sc.parallelize(dataArray)
Build the stats by key
val dataStats = data.aggregateByKey(new StatCounter())
({(s,v)=>s.merge(v)}, {(s,t)=>s.merge(t)})
Or, slightly shorter but perhaps over-tricky:
val dataStats = data.aggregateByKey(new StatCounter())(_ merge _, _ merge _)
Re-format to the OP's format and print
val result = dataStats.map(f=>(f._1,(f._2.variance,f._2.max,f._2.min)))
.foreach(println(_))
Output, same apart from some rounding errors.
(k1.0,(1776.9999999999998,100.0,2.0))
(k7.0,(0.0,8.0,8.0))
(k10.0,(18.240000000000002,13.0,1.0))
(k4.0,(0.888888888888889,7.0,5.0))
EDIT: Version with two columns
val dataArray = dataString.split("\\n")
.map(_.split("\\s+")).map(_.map(_.toDouble))
.map(f => ("k" + f(0), Array(f(1), f(2))))
val data = sc.parallelize(dataArray)
val dataStats = data.aggregateByKey(Array(new StatCounter(), new StatCounter()))({(s, v)=> Array(s(0) merge v(0), s(1) merge v(1))}, {(s, t)=> Array(s(0) merge t(0), s(1) merge t(1))})
val result = dataStats.map(f => (f._1, (f._2(0).variance, f._2(0).max, f._2(0).min), (f._2(1).variance, f._2(1).max, f._2(1).min)))
.foreach(println(_))
Output
(k1.0,(1776.9999999999998,100.0,2.0),(1716.6875,100.0,3.0))
(k7.0,(0.0,8.0,8.0),(0.0,9.0,9.0))
(k10.0,(18.240000000000002,13.0,1.0),(29.439999999999998,14.0,2.0))
(k4.0,(0.888888888888889,7.0,5.0),(0.888888888888889,8.0,6.0))
EDIT2: "n"-column version
val n = 2
val dataStats = data.aggregateByKey(List.fill(n)(new StatCounter()))(
{(s, v)=> (s zip v).map{case (si, vi) => si merge vi}},
{(s, t)=> (s zip t).map{case (si, ti) => si merge ti}})
val result = dataStats.map(f => (f._1, f._2.map(x => (x.variance, x.max, x.min))))
.foreach(println(_))
Output same as above, but if you have more columns, you can change n. It will break if the Arrays in any row has less than n elements.
I would simply use a stats object (class StatCounter). Then, I would :
parse the file, split each line
create the tuples and partition the RDD
use aggregate by key and collect as an RDD of stat objects
What is the Python analog of Time4J's code example:
// duration in seconds normalized to hours, minutes and seconds
Duration<?> dur = Duration.of(337540, ClockUnit.SECONDS).with(Duration.STD_CLOCK_PERIOD);
// custom duration format => hh:mm:ss
String s1 = Duration.Formatter.ofPattern("hh:mm:ss").format(dur);
System.out.println(s1); // output: 93:45:40
// localized duration format for french
String s2 = PrettyTime.of(Locale.FRANCE).print(dur, TextWidth.WIDE);
System.out.println(s2); // output: 93 heures, 45 minutes et 40 secondes
It is easy to get 93:45:40:
#!/usr/bin/env python3
from datetime import timedelta
dur = timedelta(seconds=337540)
print(dur) # -> 3 days, 21:45:40
fields = {}
fields['hours'], seconds = divmod(dur // timedelta(seconds=1), 3600)
fields['minutes'], fields['seconds'] = divmod(seconds, 60)
print("%(hours)02d:%(minutes)02d:%(seconds)02d" % fields) # -> 93:45:40
but how do I emulate PrettyTime.of(Locale.FRANCE).print(dur, TextWidth.WIDE) Java code in Python (without hardcoding the units)?
babel module allows to get close to desired output:
from babel.dates import format_timedelta # $ pip install babel
print(", ".join(format_timedelta(timedelta(**{unit: fields[unit]}),
granularity=unit.rstrip('s'),
threshold=fields[unit] + 1,
locale='fr')
for unit in "hours minutes seconds".split()))
# -> 93 heures, 45 minutes, 40 secondes
It handles locale and plural forms automatically e.g., for dur = timedelta(seconds=1) it produces:
0 heure, 0 minute, 1 seconde
Perhaps a better solution would be to translate the format string manually using standard tools such as gettext.
If you're using Kotlin, I just came across a similar problem with the Kotlin Duration type with localized formatting and because I couldn't find a good solution, I wrote one myself. It is based on APIs provided starting in Android 9 (for localized units), but with a fallback to English units for lower Android versions so it can be used with lower targeting apps.
Here's how it looks like on the usage side (see Kotlin Duration type to understand 1st line):
val duration = 5.days.plus(3.hours).plus(2.minutes).plus(214.milliseconds)
DurationFormat().format(duration) // "5day 3hour 2min"
DurationFormat(Locale.GERMANY).format(duration) // "5T 3Std. 2Min."
DurationFormat(Locale.forLanguageTag("ar").format(duration) // "٥يوم ٣ساعة ٢د"
DurationFormat().format(duration, smallestUnit = DurationFormat.Unit.HOUR) // "5day 3hour"
DurationFormat().format(15.minutes) // "15min"
DurationFormat().format(0.hours) // "0sec"
As you can see, you can specify a custom locale to the DurationFormat type. By default it uses Locale.getDefault(). Languages that have different symbols for number than romanic are also supported (via NumberFormat). Also, you can specify a custom smallestUnit, by default it is set to SECOND, so milliseconds will not be shown. Note that any unit with a value of 0 will be ignored and if the entire number is 0, the smallest unit will be used with the value 0.
This is the full DurationFormat type, feel free to copy (also available as a GitHub gist incl. unit tests):
import android.icu.text.MeasureFormat
import android.icu.text.NumberFormat
import android.icu.util.MeasureUnit
import android.os.Build
import java.util.Locale
import kotlin.time.Duration
import kotlin.time.ExperimentalTime
import kotlin.time.days
import kotlin.time.hours
import kotlin.time.milliseconds
import kotlin.time.minutes
import kotlin.time.seconds
#ExperimentalTime
data class DurationFormat(val locale: Locale = Locale.getDefault()) {
enum class Unit {
DAY, HOUR, MINUTE, SECOND, MILLISECOND
}
fun format(duration: kotlin.time.Duration, smallestUnit: Unit = Unit.SECOND): String {
var formattedStringComponents = mutableListOf<String>()
var remainder = duration
for (unit in Unit.values()) {
val component = calculateComponent(unit, remainder)
remainder = when (unit) {
Unit.DAY -> remainder - component.days
Unit.HOUR -> remainder - component.hours
Unit.MINUTE -> remainder - component.minutes
Unit.SECOND -> remainder - component.seconds
Unit.MILLISECOND -> remainder - component.milliseconds
}
val unitDisplayName = unitDisplayName(unit)
if (component > 0) {
val formattedComponent = NumberFormat.getInstance(locale).format(component)
formattedStringComponents.add("$formattedComponent$unitDisplayName")
}
if (unit == smallestUnit) {
val formattedZero = NumberFormat.getInstance(locale).format(0)
if (formattedStringComponents.isEmpty()) formattedStringComponents.add("$formattedZero$unitDisplayName")
break
}
}
return formattedStringComponents.joinToString(" ")
}
private fun calculateComponent(unit: Unit, remainder: Duration) = when (unit) {
Unit.DAY -> remainder.inDays.toLong()
Unit.HOUR -> remainder.inHours.toLong()
Unit.MINUTE -> remainder.inMinutes.toLong()
Unit.SECOND -> remainder.inSeconds.toLong()
Unit.MILLISECOND -> remainder.inMilliseconds.toLong()
}
private fun unitDisplayName(unit: Unit) = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
val measureFormat = MeasureFormat.getInstance(locale, MeasureFormat.FormatWidth.NARROW)
when (unit) {
DurationFormat.Unit.DAY -> measureFormat.getUnitDisplayName(MeasureUnit.DAY)
DurationFormat.Unit.HOUR -> measureFormat.getUnitDisplayName(MeasureUnit.HOUR)
DurationFormat.Unit.MINUTE -> measureFormat.getUnitDisplayName(MeasureUnit.MINUTE)
DurationFormat.Unit.SECOND -> measureFormat.getUnitDisplayName(MeasureUnit.SECOND)
DurationFormat.Unit.MILLISECOND -> measureFormat.getUnitDisplayName(MeasureUnit.MILLISECOND)
}
} else {
when (unit) {
Unit.DAY -> "day"
Unit.HOUR -> "hour"
Unit.MINUTE -> "min"
Unit.SECOND -> "sec"
Unit.MILLISECOND -> "msec"
}
}
}
This humanize package may help. It has a french localization, or you can add your own. For python 2.7 and 3.3.
Using pendulum module:
>>> import pendulum
>>> it = pendulum.interval(seconds=337540)
>>> it.in_words(locale='fr_FR')
'3 jours 21 heures 45 minutes 40 secondes'