I'm looking to implement a custom hadoop Writable class where one of the fields is a time stamp. I can't seem to find a class in the hadoop libraries (e. g. a Writable for Date or Calendar) which would make this easy. I'm thinking of creating a custom writable using get/setTimeInMillis on Calendar, but I'm wondering if there is a better/built-in solution to this problem.
There is no Writable for a Calendar/Date in Hadoop. Considering that you can get the timeInMillis as a long from the Calendar object you can use the LongWritable to serialiaze a calendar object if and only if your application always uses the default UTC time zone (i.e. it's "agnostic" to time zones, it always assumes that timeInMillis represent an UTC time).
If you use another time zone or if your application needs to be able to interpret a timeInMillis with respect to various time zone, you'll have to write from scratch a default Writable implementation.
Here's a custom writable that I generated for you to illustrate a writable with three properties, one of which is a date. You can see that the data value is persisted as a long and that it's easy to convert a long to and from a Date. If having three properties is too much, I can just generate a writable with a date for you.
package com.lmx.writable;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import com.eaio.uuid.UUID;
import org.apache.hadoop.io.*;
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultDataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
public class MyCustomWritable implements Writable {
public static int PROPERTY_DATE = 0;
public static int PROPERTY_COUNT = 1;
public static int PROPERTY_NAME = 2;
private boolean[] changeFlag = new boolean[3];
private Date _date;
private int _count;
private String _name;
public MyCustomWritable() {
resetChangeFlags();
}
public MyCustomWritable(Date _date, int _count, String _name) {
resetChangeFlags();
setDate(_date);
setCount(_count);
setName(_name);
}
public MyCustomWritable(byte[] bytes) {
ByteArrayInputStream is = new ByteArrayInputStream(bytes);
DataInput in = new DataInputStream(is);
try { readFields(in); } catch (IOException e) { }
resetChangeFlags();
}
public Date getDate() {
return _date;
}
public void setDate(Date value) {
_date = value;
changeFlag[PROPERTY_DATE] = true;
}
public int getCount() {
return _count;
}
public void setCount(int value) {
_count = value;
changeFlag[PROPERTY_COUNT] = true;
}
public String getName() {
return _name;
}
public void setName(String value) {
_name = value;
changeFlag[PROPERTY_NAME] = true;
}
public void readFields(DataInput in) throws IOException {
// Read Date _date
if (in.readBoolean()) {
_date = new Date(in.readLong());
changeFlag[PROPERTY_DATE] = true;
} else {
_date = null;
changeFlag[PROPERTY_DATE] = false;
}
// Read int _count
_count = in.readInt();
changeFlag[PROPERTY_COUNT] = true;
// Read String _name
if (in.readBoolean()) {
_name = Text.readString(in);
changeFlag[PROPERTY_NAME] = true;
} else {
_name = null;
changeFlag[PROPERTY_NAME] = false;
}
}
public void write(DataOutput out) throws IOException {
// Write Date _date
if (_date == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeLong(_date.getTime());
}
// Write int _count
out.writeInt(_count);
// Write String _name
if (_name == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
Text.writeString(out,_name);
}
}
public byte[] getBytes() throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(os);
write(out);
out.flush();
out.close();
return os.toByteArray();
}
public void resetChangeFlags() {
changeFlag[PROPERTY_DATE] = false;
changeFlag[PROPERTY_COUNT] = false;
changeFlag[PROPERTY_NAME] = false;
}
public boolean getChangeFlag(int i) {
return changeFlag[i];
}
public byte[] getDateAsBytes() throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(os);
// Write Date _date
if (_date == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeLong(_date.getTime());
}
out.flush();
out.close();
return os.toByteArray();
}
public byte[] getCountAsBytes() throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(os);
// Write int _count
out.writeInt(_count);
out.flush();
out.close();
return os.toByteArray();
}
public byte[] getNameAsBytes() throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(os);
// Write String _name
if (_name == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
Text.writeString(out,_name);
}
out.flush();
out.close();
return os.toByteArray();
}
public void setDateFromBytes(byte[] b) throws IOException {
ByteArrayInputStream is = new ByteArrayInputStream(b);
DataInput in = new DataInputStream(is);
int len;
// Read Date _date
if (in.readBoolean()) {
_date = new Date(in.readLong());
changeFlag[PROPERTY_DATE] = true;
} else {
_date = null;
changeFlag[PROPERTY_DATE] = false;
}
}
public void setCountFromBytes(byte[] b) throws IOException {
ByteArrayInputStream is = new ByteArrayInputStream(b);
DataInput in = new DataInputStream(is);
int len;
// Read int _count
_count = in.readInt();
changeFlag[PROPERTY_COUNT] = true;
}
public void setNameFromBytes(byte[] b) throws IOException {
ByteArrayInputStream is = new ByteArrayInputStream(b);
DataInput in = new DataInputStream(is);
int len;
// Read String _name
if (in.readBoolean()) {
_name = Text.readString(in);
changeFlag[PROPERTY_NAME] = true;
} else {
_name = null;
changeFlag[PROPERTY_NAME] = false;
}
}
public Tuple asTuple() throws ExecException {
Tuple tuple = TupleFactory.getInstance().newTuple(3);
if (getDate() == null) {
tuple.set(0, (Long) null);
} else {
tuple.set(0, new Long(getDate().getTime()));
}
tuple.set(1, new Integer(getCount()));
if (getName() == null) {
tuple.set(2, (String) null);
} else {
tuple.set(2, getName());
}
return tuple;
}
public static ResourceSchema getPigSchema() throws IOException {
ResourceSchema schema = new ResourceSchema();
ResourceFieldSchema fieldSchema[] = new ResourceFieldSchema[3];
ResourceSchema bagSchema;
ResourceFieldSchema bagField[];
fieldSchema[0] = new ResourceFieldSchema();
fieldSchema[0].setName("date");
fieldSchema[0].setType(DataType.LONG);
fieldSchema[1] = new ResourceFieldSchema();
fieldSchema[1].setName("count");
fieldSchema[1].setType(DataType.INTEGER);
fieldSchema[2] = new ResourceFieldSchema();
fieldSchema[2].setName("name");
fieldSchema[2].setType(DataType.CHARARRAY);
schema.setFields(fieldSchema);
return schema;
}
public static MyCustomWritable fromJson(String source) {
MyCustomWritable obj = null;
try {
JSONObject jsonObj = new JSONObject(source);
obj = fromJson(jsonObj);
} catch (JSONException e) {
System.out.println(e.toString());
}
return obj;
}
public static MyCustomWritable fromJson(JSONObject jsonObj) {
MyCustomWritable obj = new MyCustomWritable();
try {
if (jsonObj.has("date")) {
obj.setDate(new Date(jsonObj.getLong("date")));
}
if (jsonObj.has("count")) {
obj.setCount(jsonObj.getInt("count"));
}
if (jsonObj.has("name")) {
obj.setName(jsonObj.getString("name"));
}
} catch (JSONException e) {
System.out.println(e.toString());
obj = null;
}
return obj;
}
public JSONObject toJson() {
try {
JSONObject jsonObj = new JSONObject();
JSONArray jsonArray;
if (getDate() != null) {
jsonObj.put("date", getDate().getTime());
}
jsonObj.put("count", getCount());
if (getName() != null) {
jsonObj.put("name", getName());
}
return jsonObj;
} catch (JSONException e) { }
return null;
}
public String toJsonString() {
return toJson().toString();
}
}
Related
I am using the following code to create a custom log as per my requirements.
But unable to get the rollback feature as java.util.logging.Logger doesn't support it.
What are the possible options for me to implement?
Is it possible to generate rollback logs automatically using the same library?
Code :
private static class MyCustomFormatterforUpdate extends java.util.logging.Formatter {
#Override
public String format(LogRecord record) {
StringBuffer sb = new StringBuffer();
sb.append("update ApiStatistics set RespDateTime =");
sb.append(record.getMessage());
sb.append(";");
sb.append("\n");
return sb.toString();
}
}
java.util.logging.Logger updatefile = java.util.logging.Logger.getLogger("Update Script");
boolean appendu = false;
FileHandler fhu;
{
try {
fhu = new FileHandler("src/main/resources/updatescript.log",appendu);
updatefile.addHandler(fhu);
fhu.setFormatter(new MyCustomFormatterforUpdate());
} catch (IOException e) {
e.printStackTrace();
}
}
Building upon my previous answer you just need to build out the proxy implementation inferred in that answer. The idea is to install a proxy handler which allows you to open and close the FileHandler object. This enabled the rotation you need. Below is working example that will rotate when the date changes. The test case is included and the output will appear in the home folder by default.
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.Calendar;
import java.util.logging.ErrorManager;
import java.util.logging.FileHandler;
import java.util.logging.Filter;
import java.util.logging.Formatter;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogManager;
import java.util.logging.LogRecord;
import java.util.logging.SimpleFormatter;
import java.util.logging.XMLFormatter;
public class DailyFileHandler extends Handler {
public static void main(String[] args) throws IOException {
DailyFileHandler dfh = new DailyFileHandler();
try {
dfh.setFormatter(new SimpleFormatter());
LogRecord r1 = new LogRecord(Level.SEVERE, "test 1");
LogRecord r2 = new LogRecord(Level.SEVERE, "test 2");
r2.setMillis(r1.getMillis() + (24L * 60L * 60L * 1000L));
dfh.publish(r1);
dfh.publish(r2);
} finally {
dfh.close();
}
}
private Calendar current = Calendar.getInstance();
private FileHandler target;
public DailyFileHandler() throws IOException {
target = new FileHandler(pattern(), limit(), count(), false);
init();
}
public DailyFileHandler(String pattern, int limit, int count)
throws IOException {
target = new FileHandler(pattern, limit, count, false);
init();
}
private void init() {
super.setLevel(level());
super.setFormatter(formatter());
super.setFilter(filter());
try {
super.setEncoding(encoding());
} catch (UnsupportedEncodingException impossible) {
throw new AssertionError(impossible);
}
initTarget();
}
private void initTarget() {
target.setErrorManager(super.getErrorManager());
target.setLevel(super.getLevel());
target.setFormatter(super.getFormatter());
target.setFilter(super.getFilter());
try {
target.setEncoding(super.getEncoding());
} catch (UnsupportedEncodingException impossible) {
throw new AssertionError(impossible);
}
}
private void rotate() {
String pattern = pattern();
int count = count();
int limit = limit();
try {
super.setErrorManager(target.getErrorManager());
target.close();
FileHandler rotate = new FileHandler(pattern, 0, count, false);
rotate.setFormatter(new SimpleFormatter()); //empty tail.
rotate.close();
current = Calendar.getInstance();
target = new FileHandler(pattern, limit, count, true);
initTarget();
} catch (RuntimeException | IOException e) {
this.reportError("", e, ErrorManager.OPEN_FAILURE);
}
}
private boolean shouldRotate(long millis) {
Calendar cal = Calendar.getInstance();
cal.setTimeInMillis(millis);
return cal.get(Calendar.DAY_OF_YEAR) != current.get(Calendar.DAY_OF_YEAR);
}
#Override
public synchronized void publish(LogRecord record) {
if (shouldRotate(record.getMillis())) {
rotate();
}
target.publish(record);
}
#Override
public synchronized void close() throws SecurityException {
target.close();
}
#Override
public synchronized void setEncoding(String encoding) throws SecurityException, UnsupportedEncodingException {
target.setEncoding(encoding);
super.setEncoding(encoding);
}
#Override
public synchronized boolean isLoggable(LogRecord record) {
return target.isLoggable(record);
}
#Override
public synchronized void flush() {
target.flush();
}
#Override
public synchronized void setFormatter(Formatter newFormatter) {
target.setFormatter(newFormatter);
super.setFormatter(newFormatter);
}
#Override
public synchronized Formatter getFormatter() {
return target.getFormatter();
}
#Override
public synchronized String getEncoding() {
return target.getEncoding();
}
#Override
public synchronized void setFilter(Filter newFilter) throws SecurityException {
target.setFilter(newFilter);
super.setFilter(newFilter);
}
#Override
public synchronized Filter getFilter() {
return target.getFilter();
}
#Override
public synchronized void setErrorManager(ErrorManager em) {
target.setErrorManager(em);
super.setErrorManager(em);
}
#Override
public synchronized ErrorManager getErrorManager() {
return target.getErrorManager();
}
#Override
public synchronized void setLevel(Level newLevel) throws SecurityException {
target.setLevel(newLevel);
super.setLevel(newLevel);
}
#Override
public synchronized Level getLevel() {
return target.getLevel();
}
private String pattern() {
LogManager m = LogManager.getLogManager();
String p = getClass().getName();
String pattern = m.getProperty(p + ".pattern");
if (pattern == null) {
pattern = "%h/java%u.log";
}
return pattern;
}
private int limit() {
LogManager m = LogManager.getLogManager();
String p = getClass().getName();
String v = m.getProperty(p + ".limit");
int limit = v == null ? Integer.MAX_VALUE : Integer.parseInt(v);
return limit;
}
private int count() {
LogManager m = LogManager.getLogManager();
String p = getClass().getName();
String v = m.getProperty(p + ".count");
int limit = v == null ? 7 : Integer.parseInt(v);
return limit;
}
private Level level() {
LogManager m = LogManager.getLogManager();
String p = getClass().getName();
String v = m.getProperty(p + ".level");
if (v != null) {
try {
return Level.parse(v);
} catch (Exception e) {
this.reportError(v, e, ErrorManager.OPEN_FAILURE);
}
}
return Level.ALL;
}
private Formatter formatter() {
LogManager m = LogManager.getLogManager();
String p = getClass().getName();
String v = m.getProperty(p + ".formatter");
if (v != null) {
try {
return Formatter.class.cast(Class.forName(v).newInstance());
} catch (Exception e) {
this.reportError("", e, ErrorManager.OPEN_FAILURE);
}
}
return new XMLFormatter();
}
private Filter filter() {
LogManager m = LogManager.getLogManager();
String p = getClass().getName();
String v = m.getProperty(p + ".filter");
if (v != null) {
try {
return Filter.class.cast(Class.forName(v).newInstance());
} catch (Exception e) {
this.reportError("", e, ErrorManager.OPEN_FAILURE);
}
}
return null;
}
private String encoding() {
LogManager m = LogManager.getLogManager();
String p = getClass().getName();
String v = m.getProperty(p + ".encoding");
if (v != null) {
try {
return Charset.forName(v).name();
} catch (Exception e) {
this.reportError(v, e, ErrorManager.OPEN_FAILURE);
}
}
return null;
}
}
A suggestion would be please use other logging frameworks which has many features in them
instead of java.util.logging.Logger
Useful links
configure-log4j-for-creating-daily-rolling-log-files
log4j-formatting-examples
a-guide-to-logging-in-java
I have 4 large files (around 1.5 gb each) and I want to process these files, read each line of the file and convert it to a customer object. I have the following implementation.
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UncheckedIOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.function.Consumer;
import java.util.zip.GZIPInputStream;
import static java.nio.charset.StandardCharsets.UTF_8;
public class CustomerDataAccess {
public static void main(String[] args) throws IOException {
CustomerFileItem john = new CustomerFileItem("CustFile1", "http://w.customer1.com");
CustomerFileItem sarah = new CustomerFileItem("CustFile2", "http://w.customer2.com");
CustomerFileItem charles = new CustomerFileItem("CustFile3", "http://w.customer3.com");
List<CustomerFileItem> customers = Arrays.asList(john, sarah, charles);
Iterator<CustomerFileLineItem> custList = new CustIterator(customers);
}
public static class CustIterator implements Iterator<CustomerFileLineItem> {
private static final int HEADER_LINES = 9; // 8 + 1 blank line
BufferedReader bufferedReader;
private int index = 0;
private final List<CustomerFileItem> custFileItems = new ArrayList<>();
public CustIterator(final List<CustomerFileItem> custFileItems) throws IOException {
this.custFileItems.addAll(custFileItems);
processNext();
}
private void processNext() throws IOException {
if (bufferedReader != null) {
bufferedReader.close();
}
if (index < custFileItems.size()) { // only update if there's another file
CustomerFileItem custFileItem = custFileItems.get(index);
GZIPInputStream gis = new GZIPInputStream(new URL(custFileItem.url).openStream());
// default buffer size is 8 KB
bufferedReader = new BufferedReader(new InputStreamReader(gis, UTF_8));
// read the first few lines
for (int i = 0; i < HEADER_LINES; i++) {
bufferedReader.readLine();
}
}
index++;
}
#Override
public boolean hasNext() {
try {
boolean currentReaderStatus = bufferedReader.ready();
if (currentReaderStatus) {
return true;
} else if (index < custFileItems.size()) {
// at end of current file, try to get the next one
processNext();
return hasNext();
} else { // no more files left
return false;
}
} catch (IOException e) {
try {
bufferedReader.close();
} catch (IOException e1) {
throw new UncheckedIOException(e1);
}
throw new UncheckedIOException(e);
}
}
#Override
public CustomerFileLineItem next() {
try {
String line = bufferedReader.readLine();
if (line != null) {
return new CustomerFileLineItem(line);
} else {
return null;
}
} catch (IllegalArgumentException exception) {
return null;
} catch (IOException e) {
try {
bufferedReader.close();
} catch (IOException e1) {
throw new UncheckedIOException(e1);
}
throw new UncheckedIOException(e);
}
}
#Override
public void remove() {
throw new UnsupportedOperationException();
}
#Override
public void forEachRemaining(final Consumer<? super CustomerFileLineItem> action) {
throw new UnsupportedOperationException();
}
}
public static class CustomerFileLineItem {
private static final int NUMBER_OF_FIELDS = 4;
final String id;
final String productNumber;
final String usageType;
final String operation;
public CustomerFileLineItem(final String line) {
String[] strings = line.split(",");
if (strings.length != NUMBER_OF_FIELDS) {
throw new IllegalArgumentException(String.format("Malformed customer file line: %s", line));
}
this.id = strings[0];
this.productNumber = strings[1];
this.usageType = strings[3];
this.operation = strings[4];
}
}
static class CustomerFileItem {
private String fileName;
private String url;
public CustomerFileItem(String fileName, String url) {
this.fileName = fileName;
this.url = url;
}
}
}
In one of use case I want use streams in the output list(custList). But I know I can't use streams with Iterator. How I can convert it to Spliterator? Or how can I implement the same that I implement with Iterator in Spliterator?
TL;DR You don’t need to implement an Iterator or Spliterator, you can simply use a Stream in the first place:
private static final int HEADER_LINES = 9; // 8 + 1 blank line
Stream<CustomerFileLineItem> stream = customers.stream()
.flatMap(custFileItem -> {
try {
GZIPInputStream gis
= new GZIPInputStream(new URL(custFileItem.url).openStream());
BufferedReader br = new BufferedReader(new InputStreamReader(gis, UTF_8));
// read the first few lines
for (int i = 0; i < HEADER_LINES; i++) br.readLine();
return br.lines().onClose(() -> {
try { br.close(); }
catch(IOException ex) { throw new UncheckedIOException(ex); }
});
} catch(IOException ex) {
throw new UncheckedIOException(ex);
}
})
.map(CustomerFileLineItem::new);
But for completeness, addressing the question literally:
First of all, you should not add a method definition like
#Override
public void forEachRemaining(final Consumer<? super CustomerFileLineItem> action) {
throw new UnsupportedOperationException();
}
This method will surely backfire when you use the Stream API, as that’s where most non-short-circuiting operations will end up.
There is not even a reason to add it. When you don’t declare the method, you’ll get a reasonable default method from the Iterator interface.
When you fixed this issue, you can easily convert the Iterator to a Spliterator using Spliterators.pliteratorUnknownSize(Iterator, int).
But there is no reason to do so. Your code becomes simpler when implementing Spliterator in the first place:
public static class CustIterator
extends Spliterators.AbstractSpliterator<CustomerFileLineItem> {
private static final int HEADER_LINES = 9; // 8 + 1 blank line
BufferedReader bufferedReader;
private final ArrayDeque<CustomerFileItem> custFileItems;
public CustIterator(final List<CustomerFileItem> custFileItems) throws IOException {
super(Long.MAX_VALUE, ORDERED|NONNULL);
this.custFileItems = new ArrayDeque<>(custFileItems);
processNext();
}
#Override
public boolean tryAdvance(Consumer<? super CustomerFileLineItem> action) {
if(bufferedReader == null) return false;
try {
String line = bufferedReader.readLine();
while(line == null) {
processNext();
if(bufferedReader == null) return false;
line = bufferedReader.readLine();
}
action.accept(new CustomerFileLineItem(line));
return true;
}
catch(IOException ex) {
if(bufferedReader != null) try {
bufferedReader.close();
bufferedReader = null;
}
catch(IOException ex2) {
ex.addSuppressed(ex2);
}
throw new UncheckedIOException(ex);
}
}
private void processNext() throws IOException {
if (bufferedReader != null) {
bufferedReader.close();
bufferedReader = null;
}
if (!custFileItems.isEmpty()) { // only update if there's another file
CustomerFileItem custFileItem = custFileItems.remove();
GZIPInputStream gis
= new GZIPInputStream(new URL(custFileItem.url).openStream());
// default buffer size is 8 KB
bufferedReader = new BufferedReader(new InputStreamReader(gis, UTF_8));
// read the first few lines
for (int i = 0; i < HEADER_LINES; i++) {
bufferedReader.readLine();
}
}
}
}
But, as said at the beginning, you don’t even need to implement a Spliterator here.
Every Iterable<T> object has the following methods:
Iterator<T> iterator() returning Iterator<T>
default Spliterator<T> spliterator() (default method) returning Spliterator<T>
Therefore, you want to create Iterable<T> back from Iterator<T> which requires to override the only one non-default and abstract method:
Iterable<CustomerFileLineItem> iterable = new Iterable<CustomerFileLineItem>() {
#Override
public Iterator<CustomerFileLineItem> iterator() {
return custList;
}
};
This can be shortened into a lambda expression resulting in:
Iterable<CustomerFileLineItem> iterable = () -> custList;
Spliterator<CustomerFileLineItem> spliterator = iterable.spliterator();
... so the Stream is easily to be created:
Stream<CustomerFileLineItem> stream = StreamSupport.stream(spliterator, false);
Hey so I'm new to using serialization, at least the way that our professor wants us to. I have the issue that for some reason it will not serialize even with java.io.Serializable implemented. I tried the direct path and "Serializable" neither work. Im not sure whats going wrong or why, as far as I understand there should not be any methods that I need to implement, correct?
THE ERROR I GET
java.io.NotSerializableException: Transaction
at java.base/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1185)
at java.base/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
at TransactionManager.serialize(TransactionManager.java:81)
at Test3.main(Test3.java:62)
java.io.WriteAbortedException: writing aborted; java.io.NotSerializableException: Transaction
at java.base/java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1598)
at java.base/java.io.ObjectInputStream.readObject(ObjectInputStream.java:430)
at TransactionManager.deSerialize(TransactionManager.java:96)
at Test3.main(Test3.java:67)
Caused by: java.io.NotSerializableException: Transaction
at java.base/java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1185)
at java.base/java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:349)
at TransactionManager.serialize(TransactionManager.java:81)
at Test3.main(Test3.java:62)
THE CODE
import java.awt.print.Book;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
public class TransactionManager implements Serializable {
private ArrayList<Transaction> col;
public TransactionManager() {
col = new ArrayList<Transaction>();
}
public void add(Transaction obj) {
col.add(obj);
}
public void clear() {
col.clear();
}
public int getCount() {
return col.size();
}
public ArrayList<Transaction> getAll() {
ArrayList<Transaction> temp = col;
Collections.sort(temp, new Comparator<Transaction>() {
public int compare(Transaction s1, Transaction s2) {
return ((Integer) s1.getId()).compareTo(s2.getId());
}
});
return temp;
}
public ArrayList<Transaction> findBuyer(String name) {
ArrayList<Transaction> temp = new ArrayList<>();
for (int i = 0; i < col.size(); i++) {
if (col.get(i).getBuyer().equals(name)) {
temp.add(col.get(i));
}
}
Collections.sort(temp, new Comparator<Transaction>() {
public int compare(Transaction s1, Transaction s2) {
return ((Integer) s1.getId()).compareTo(s2.getId());
}
});
return temp;
}
public ArrayList<Transaction> findSeller(String name) {
ArrayList<Transaction> temp = new ArrayList<>();
for (int i = 0; i < col.size(); i++) {
if (col.get(i).getSeller().equals(name)) {
temp.add(col.get(i));
}
}
Collections.sort(temp, new Comparator<Transaction>() {
public int compare(Transaction s1, Transaction s2) {
return ((Integer) s1.getId()).compareTo(s2.getId());
}
});
return temp;
}
public Transaction findId(int ID) {
for (int i = 0; i < col.size(); i++) {
if (col.get(i).getId() == ID) {
return col.get(i);
}
}
return null;
}
public void serialize() throws IOException {
FileOutputStream output = new FileOutputStream(new File("savePerson.ser"));
ObjectOutputStream outputStream = new ObjectOutputStream(output);
for (int i = 0; i < col.size(); i++) {
outputStream.writeObject(col.get(i));
}
output.close();
outputStream.close();
}
public void deSerialize() throws IOException, ClassNotFoundException {
FileInputStream inputStream = new FileInputStream(new File("savePerson.ser"));
boolean cont = true;
try {
ObjectInputStream ois = new ObjectInputStream(inputStream);
while (cont) {
Transaction obj = null;
try {
obj = (Transaction) ois.readObject();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
if (obj != null)
col.add(obj);
else
cont = false;
}
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
inputStream.close();
}
}
}
I'm developing a radio app with multiple radios, the stream is playing fine. But I'm struggling to show artist and music playing at the moment.
This is the class I'm using to get metadata from shoutcast stream:
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class IcyStreamMeta<Message> {
protected URL streamUrl;
private Map<String, String> metadata;
private boolean isError;
public IcyStreamMeta(URL streamUrl) {
setStreamUrl(streamUrl);
isError = false;
}
/**
* Get artist using stream's title
*
* #return String
* #throws IOException
*/
public String getArtist() throws IOException {
Map<String, String> data = getMetadata();
if (!data.containsKey("StreamTitle"))
return "";
String streamTitle = data.get("StreamTitle");
String title = streamTitle.substring(0, streamTitle.indexOf("-"));
return title.trim();
}
/**
* Get title using stream's title
*
* #return String
* #throws IOException
*/
public String getTitle() throws IOException {
Map<String, String> data = getMetadata();
if (!data.containsKey("StreamTitle"))
return "";
String streamTitle = data.get("StreamTitle");
String artist = streamTitle.substring(streamTitle.indexOf("-")+1);
return artist.trim();
}
public Map<String, String> getMetadata() throws IOException {
if (metadata == null) {
refreshMeta();
}
return metadata;
}
public void refreshMeta() throws IOException {
retreiveMetadata();
}
private void retreiveMetadata() throws IOException {
URLConnection con = streamUrl.openConnection();
con.setRequestProperty("Icy-MetaData", "1");
con.setRequestProperty("Connection", "close");
con.setRequestProperty("Accept", null);
con.connect();
int metaDataOffset = 0;
Map<String, List<String>> headers = con.getHeaderFields();
InputStream stream = con.getInputStream();
if (headers.containsKey("icy-metaint")) {
// Headers are sent via HTTP
metaDataOffset = Integer.parseInt(headers.get("icy-metaint").get(0));
} else {
// Headers are sent within a stream
StringBuilder strHeaders = new StringBuilder();
char c;
while ((c = (char)stream.read()) != -1) {
strHeaders.append(c);
if (strHeaders.length() > 5 && (strHeaders.substring((strHeaders.length() - 4), strHeaders.length()).equals("\r\n\r\n"))) {
// end of headers
break;
}
}
// Match headers to get metadata offset within a stream
Pattern p = Pattern.compile("\\r\\n(icy-metaint):\\s*(.*)\\r\\n");
Matcher m = p.matcher(strHeaders.toString());
if (m.find()) {
metaDataOffset = Integer.parseInt(m.group(2));
}
}
// In case no data was sent
if (metaDataOffset == 0) {
isError = true;
return;
}
// Read metadata
int b;
int count = 0;
int metaDataLength = 4080; // 4080 is the max length
boolean inData = false;
StringBuilder metaData = new StringBuilder();
// Stream position should be either at the beginning or right after headers
while ((b = stream.read()) != -1) {
count++;
// Length of the metadata
if (count == metaDataOffset + 1) {
metaDataLength = b * 16;
}
if (count > metaDataOffset + 1 && count < (metaDataOffset + metaDataLength)) {
inData = true;
} else {
inData = false;
}
if (inData) {
if (b != 0) {
metaData.append((char)b);
}
}
if (count > (metaDataOffset + metaDataLength)) {
break;
}
}
// Set the data
metadata = IcyStreamMeta.parseMetadata(metaData.toString());
// Close
stream.close();
}
public boolean isError() {
return isError;
}
public URL getStreamUrl() {
return streamUrl;
}
public void setStreamUrl(URL streamUrl) {
this.metadata = null;
this.streamUrl = streamUrl;
this.isError = false;
}
public static Map<String, String> parseMetadata(String metaString) {
Map<String, String> metadata = new HashMap();
String[] metaParts = metaString.split(";");
Pattern p = Pattern.compile("^([a-zA-Z]+)=\\'([^\\']*)\\'$");
Matcher m;
for (int i = 0; i < metaParts.length; i++) {
m = p.matcher(metaParts[i]);
if (m.find()) {
metadata.put(m.group(1), m.group(2));
}
}
return metadata;
}
}
And the method on MainActivity to get the metadata every 10 seconds
private void getMeta()
{
Timer timer = new Timer();
timer.schedule(new TimerTask() {
public void run() {
try {
IcyStreamMeta icy = new IcyStreamMeta(new URL(RadiophonyService.getRadioURL()));
final String data = icy.getArtist() + " - " + icy.getTitle();
final TextView meta = (TextView) findViewById(R.id.now_playing);
runOnUiThread(new Runnable() {
public void run() {
meta.setText(data);
}
});
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}, 0, 10000);
}
Initially, when I select one station, it plays but does not show metadata and when I select another station the app crashes with this runtime error:
E/AndroidRuntime: FATAL EXCEPTION: Timer-0
Process: com.example.app, PID: 23597
java.lang.StringIndexOutOfBoundsException: length=0; regionStart=0; regionLength=-1
at java.lang.String.startEndAndLength(String.java:504)
at java.lang.String.substring(String.java:1333)
at com.example.app.utilities.IcyStreamMeta.getArtist(IcyStreamMeta.java:41)
at com.example.app.activities.MainActivity$8.run(MainActivity.java:306)
at java.util.Timer$TimerImpl.run(Timer.java:284)
I'm stuck at this for days, and I tried other solutions but nothing works!
Looks like this line
String title = streamTitle.substring(0, streamTitle.indexOf("-"));
is the culprit.
If streamTitle does not have a dash in its title, indexOf() will return a -1, and substring() is choking because you can't have an end index less than the start index.
Maybe you need something like this:
int pos = streamTitle.indexOf("-");
String title = (pos == -1) ? streamTitle : streamTitle.substring(0, pos);
I used telegram api from this source :
https://github.com/voleon/telegram-trivia-bot
But my problem is , how keep the user signed in.
Because after the App stop need user renter Mobile phone and get activation code from SMS message.
I have implemented Serializable for saving ApiState object. but this method not solved my problem.
this is the code for my ApiState :
package engine;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import org.telegram.api.TLConfig;
import org.telegram.api.TLDcOption;
import org.telegram.api.engine.storage.AbsApiState;
import org.telegram.mtproto.state.AbsMTProtoState;
import org.telegram.mtproto.state.ConnectionInfo;
import org.telegram.mtproto.state.KnownSalt;
/**
* Created by ex3ndr on 13.01.14.
*/
public class MemoryApiState implements AbsApiState,java.io.Serializable {
public HashMap<Integer, ConnectionInfo[]> connections = new HashMap<Integer, ConnectionInfo[]>();
private HashMap<Integer, byte[]> keys = new HashMap<Integer, byte[]>();
private HashMap<Integer, Boolean> isAuth = new HashMap<Integer, Boolean>();
private int primaryDc = 1;
public MemoryApiState(boolean isTest) {
connections.put(1, new ConnectionInfo[]{
new ConnectionInfo(1, 0, isTest ? "149.154.167.40" : "149.154.167.50", 443)
});
}
#Override
public synchronized int getPrimaryDc() {
return primaryDc;
}
#Override
public synchronized void setPrimaryDc(int dc) {
primaryDc = dc;
}
#Override
public synchronized boolean isAuthenticated(int dcId) {
if (isAuth.containsKey(dcId)) {
return isAuth.get(dcId);
}
return false;
}
#Override
public synchronized void setAuthenticated(int dcId, boolean auth) {
isAuth.put(dcId, auth);
}
#Override
public synchronized void updateSettings(TLConfig config) {
connections.clear();
HashMap<Integer, ArrayList<ConnectionInfo>> tConnections = new HashMap<Integer, ArrayList<ConnectionInfo>>();
int id = 0;
for (TLDcOption option : config.getDcOptions()) {
if (!tConnections.containsKey(option.getId())) {
tConnections.put(option.getId(), new ArrayList<ConnectionInfo>());
}
tConnections.get(option.getId()).add(new ConnectionInfo(id++, 0, option.getIpAddress(), option.getPort()));
}
for (Integer dc : tConnections.keySet()) {
connections.put(dc, tConnections.get(dc).toArray(new ConnectionInfo[0]));
}
}
#Override
public synchronized byte[] getAuthKey(int dcId) {
return keys.get(dcId);
}
#Override
public synchronized void putAuthKey(int dcId, byte[] key) {
keys.put(dcId, key);
}
#Override
public synchronized ConnectionInfo[] getAvailableConnections(int dcId) {
if (!connections.containsKey(dcId)) {
return new ConnectionInfo[0];
}
return connections.get(dcId);
}
#Override
public synchronized AbsMTProtoState getMtProtoState(final int dcId) {
return new AbsMTProtoState() {
private KnownSalt[] knownSalts = new KnownSalt[0];
#Override
public byte[] getAuthKey() {
return MemoryApiState.this.getAuthKey(dcId);
}
#Override
public ConnectionInfo[] getAvailableConnections() {
return MemoryApiState.this.getAvailableConnections(dcId);
}
#Override
public KnownSalt[] readKnownSalts() {
return knownSalts;
}
#Override
protected void writeKnownSalts(KnownSalt[] salts) {
knownSalts = salts;
}
};
}
#Override
public synchronized void resetAuth() {
isAuth.clear();
}
#Override
public synchronized void reset() {
isAuth.clear();
keys.clear();
}
public void saveObject()
{
try
{
FileOutputStream fileOut =
new FileOutputStream("apistate3.tmp");
ObjectOutputStream out = new ObjectOutputStream(fileOut);
out.writeObject(this);
out.close();
fileOut.close();
System.out.printf("Serialized data is saved");
}catch(IOException i)
{
i.printStackTrace();
}
}
public MemoryApiState readObject()
{
try
{
FileInputStream fileIn = new FileInputStream("apistate3.tmp");
ObjectInputStream in = new ObjectInputStream(fileIn);
MemoryApiState obj = (MemoryApiState) in.readObject();
in.close();
fileIn.close();
return obj;
}catch(IOException i)
{
i.printStackTrace();
return null;
}catch(ClassNotFoundException c)
{
System.out.println("Employee class not found");
c.printStackTrace();
return null;
}
}
}
hi : you can save state in file with any extension as follow:
private void SaveState(String fileName, MemoryApiState mas)
{
try
{
FileOutputStream fileOut = new FileOutputStream(fileName + ".sta");
ObjectOutputStream out = new ObjectOutputStream(fileOut);
out.writeObject(mas);
out.close();
fileOut.close();
}
catch (IOException i)
{
i.printStackTrace();
}
}
and than load saved state from file as follow :
private MemoryApiState LoadState(String fileName)
{
try
{
FileInputStream fileIn = new FileInputStream(fileName + ".sta");
ObjectInputStream in = new ObjectInputStream(fileIn);
MemoryApiState mas = (MemoryApiState)in.readObject();
in.close();
fileIn.close();
return mas;
}
catch (IOException i)
{
return null;
}
catch (ClassNotFoundException c)
{
c.printStackTrace();
}
return null;
}