Develop Reference

Line reader fails depending on buffer size

I need to read an input stream line by line. A line is considered to be terminated only by CRLF, but not by a single CR or LF. This rules out BufferedReader's readLine() and had me implement my own solution:
final class LineReader
{
private final Reader reader;
private final char[] buffer;
private final Queue<String> lines = new LinkedList<>();
private StringBuilder line = new StringBuilder();
private boolean cr = false;
LineReader(final Reader reader, final int bufferSize)
{
this.reader = reader;
buffer = new char[bufferSize];
}
String readLine() throws IOException
{
while (lines.peek() == null)
{
final int read = reader.read(buffer);
if (read == - 1)
{
if (line == null)
{
return null;
}
// Reached EOF. Return the last line.
lines.add(line.toString());
line = null;
continue;
}
// Split the buffer by line.
int offset = 0;
for (int i = 0; i < read; i++)
{
final char ch = buffer[i];
if (cr)
{
// Last character was CR.
switch (ch)
{
case '\n':
// Found a CRLF.
if (i != 0)
{
line.append(buffer, offset, i - 1 - offset);
}
// Next line starts at the next character.
offset = i + 1;
lines.add(line.toString());
line = new StringBuilder();
cr = false;
break;
case '\r':
break;
default:
cr = false;
break;
}
}
else if (ch == '\r')
{
cr = true;
}
}
// Append remaining characters to the next line.
line.append(buffer, offset, read - offset);
}
return lines.poll();
}
}
Initially, the reader passed some naive tests. However, once I started altering the buffer size, I noticed that some tests failed.
#Test
void readLine() throws IOException
{
final String[] lines = new String[]{"foo bar", "baz", ""};
final String str = Stream.of(lines).collect(joining("\r\n"));
final Collection<Executable> assertions = new LinkedList<>();
for (int bufferSize = 1; bufferSize <= 10; bufferSize++)
{
final LineReader reader = new LineReader(new StringReader(str),
bufferSize);
assertions.add(() ->
{
for (int i = 0; i < lines.length; i++)
{
assertEquals(lines[i], reader.readLine());
}
assertNull(reader.readLine());
});
}
assertAll(assertions);
}
More specifically, the equality assertion only fails when the buffer size is set to 1, 2, 4 or 8. And even stranger, the error messages are all blank.
Multiple Failures (4 failures)
>
>
>
>
org.opentest4j.MultipleFailuresError: Multiple Failures (4 failures)
>
>
>
>
at org.junit.jupiter.api.AssertAll.assertAll(AssertAll.java:80)
at org.junit.jupiter.api.AssertAll.assertAll(AssertAll.java:54)
...
I can't wrap my head around this.

So it turned out that I've been wrongly appending a CR when it's the last character in the buffer, even though it is followed by a LF. The redundant CR also caused the error messages to be truncated weirdly in my console output. Below is the working method:
String readLine() throws IOException
{
while (lines.peek() == null)
{
if (line == null)
{
break;
}
final int read = reader.read(buffer);
if (read == - 1)
{
lines.add(line.toString());
line = null;
continue;
}
int offset = 0;
for (int i = 0; i < read; i++)
{
final char ch = buffer[i];
if (cr)
{
if (ch == '\n')
{
if (i != 0)
{
line.append(buffer, offset, i - 1 - offset);
}
offset = i + 1;
lines.add(line.toString());
line = new StringBuilder();
cr = false;
}
else
{
if (i == 0)
{
line.append('\r');
}
if (ch != '\r')
{
cr = false;
}
}
}
else if (ch == '\r')
{
cr = true;
}
}
line.append(buffer, offset, read - offset - (cr ? 1 : 0));
}
return lines.poll();
}

Java - Writing a Method to Count Lines In a Text File Without Throwing Exceptions

Below is a solution from Number of lines in a file in Java
to quickly count the number of lines in a text file.
However, I am trying to write a method that will perform the same task without throwing an 'IOException'.
Under the original solution is my attempt to do this with a nested try-catch block <-- (Is this usually done/frowned upon/ or easily avoidable??) which returns 0 no matter how many lines are in the given file (obviously a fail).
Just to be clear, I am not looking for advice on how to better use the original method that does contain the exception and, therefore, the context within which I am using it is irrelevant to this question.
Can somebody please help me write a method that counts the number of lines in a text file and does not throw any exceptions? (In other words, deals with potential errors with a try-catch.)
Original line counter by martinus:
public static int countLines(String filename) throws IOException {
InputStream is = new BufferedInputStream(new FileInputStream(filename));
try {
byte[] c = new byte[1024];
int count = 0;
int readChars = 0;
boolean empty = true;
while ((readChars = is.read(c)) != -1) {
empty = false;
for (int i = 0; i < readChars; ++i) {
if (c[i] == '\n') {
++count;
}
}
}
return (count == 0 && !empty) ? 1 : count;
} finally {
is.close();
}
}
My Attempt:
public int countLines(String fileName ) {
InputStream input = null;
try{
try{
input = new BufferedInputStream(new FileInputStream(fileName));
byte[] count = new byte[1024];
int lines = 0;
int forChar;
boolean empty = true;
while((forChar = input.read(count)) != -1){
empty = false;
for(int x = 0; x < forChar; x++){
if(count[x] == '\n'){
lines++;
}
}
}
return (!empty && lines == 0) ? 1 : lines + 1;
}
finally{
if(input != null)
input.close();
}
}
catch(IOException f){
int lines = 0;
return lines;
}
}

It is more robust to use char instead of byte for '\n' and return -1 in case of any errors, for example if the filename does not exist:
public static int countLines(String filename) {
BufferedReader br = null;
try {
br = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
char[] c = new char[1024];
int count = 0;
int readChars = 0;
boolean emptyLine = true;
while ((readChars = br.read(c)) != -1) {
for (int i = 0; i < readChars; ++i) {
emptyLine = false;
if (c[i] == '\n') {
++count;
emptyLine = true;
}
}
}
return count + (!emptyLine ? 1 : 0);
} catch (IOException ex) {
return -1;
} finally {
if (br != null)
try {
br.close();
} catch (IOException e) {
// Ignore intentionally
}
}
}

Sharing my attempt.
public static int countLines(String filename) {
InputStream is = new BufferedInputStream(new FileInputStream(filename));
int numLines = 0;
try {
byte[] c = new byte[1024];
int count = 0;
int readChars = 0;
boolean empty = true;
while ((readChars = is.read(c)) != -1) {
empty = false;
for (int i = 0; i < readChars; ++i) {
if (c[i] == '\n') {
++count;
}
}
}
numLines = (count == 0 && !empty) ? 1 : count;
} catch (IOException ex) {
numLines = 0;
} catch (FileNotFoundException ex) {
System.out.println("File not found.");
numLines = 0;
} finally {
is.close();
}
return numLines;
}

External authentication in ejabberd using java

I am working on task to implement external authentication in ejabberd using java.
I searched for the examples on internet and found examples in PHP, Perl, Python but could not find any example in java.
I know the configuration that is required to be made in 'ejabberd.cfg' file.
Any code sample in java will be very helpful.

Try this:
public static void main(String[] args) {
try {
outerloop: while (true) {
byte[] lB = new byte[2];
int startPos = 0;
while (startPos < lB.length) {
int ret = System.in.read(lB, startPos,
(lB.length - startPos));
if (ret < 0) {
break outerloop;
}
startPos += ret;
}
int streamLen = System.in.available();
byte[] rd = new byte[streamLen];
startPos = 0;
while (startPos < streamLen) {
int ret = System.in.read(rd, startPos,
(streamLen - startPos));
if (ret < 0) {
break outerloop;
}
startPos += ret;
}
String inputArgs = new String(rd, "ASCII");
String[] arguments = inputArgs.split(":");
String userName = arguments[1];
String password = arguments[3];
//
// Here do the authentication
//
boolean resultOfAuthentication = // Result of Authentication;
byte[] res = new byte[4];
res[0] = 0;
res[1] = 2;
res[2] = 0;
if (resultOfAuthentication) {
res[3] = 1;
} else {
res[3] = 0;
}
System.out.write(res, 0, res.length);
System.out.flush();
}
} catch (Exception e) {
System.out.println("ERROR");
}
}

Xuggle combine audio with generated audio

I have an mp3 file, and an image. I need to create a video combining them, in java.
I'm trying to do it with xuggle, but there are still no results.
Can anybody give me any suggestions ?

Finally, I found a solution.
I used pieces of code from Xuggle's examples.
I also solved a problem with audio transcoding.
I'll write my code here, because I cannot explain why it works, but it just works.
public String make() throws IOException, InterruptedException {
BufferedImage s1 = genImage();
writer = ToolFactory.makeWriter("temp/" + sermon.getFile().getName() + ".flv");
String filename = sermon.getFile().getAbsolutePath();
IContainer container = IContainer.make();
if (container.open(filename, IContainer.Type.READ, null) < 0) {
throw new IllegalArgumentException("could not open file: " + filename);
}
int numStreams = container.getNumStreams();
int audioStreamId = -1;
IStreamCoder audioCoder = null;
for (int i = 0; i < numStreams; i++) {
IStream stream = container.getStream(i);
IStreamCoder coder = stream.getStreamCoder();
if (coder.getCodecType() == ICodec.Type.CODEC_TYPE_AUDIO) {
audioStreamId = i;
audioCoder = coder;
break;
}
}
if (audioStreamId == -1) {
throw new RuntimeException("could not find audio stream in container: " + filename);
}
if (audioCoder.open() < 0) {
throw new RuntimeException("could not open audio decoder for container: " + filename);
}
writer.addAudioStream(0, 0, audioCoder.getChannels(), audioCoder.getSampleRate());
writer.addVideoStream(1, 1, width, height);
IPacket packet = IPacket.make();
int n = 0;
while (container.readNextPacket(packet) >= 0) {
n++;
if (packet.getStreamIndex() == audioStreamId) {
IAudioSamples samples = IAudioSamples.make(2048, audioCoder.getChannels());
int offset = 0;
while (offset < packet.getSize()) {
try {
int bytesDecoded = audioCoder.decodeAudio(samples, packet, offset);
if (bytesDecoded < 0) {
//throw new RuntimeException("got error decoding audio in: " + filename);
break;
}
offset += bytesDecoded;
if (samples.isComplete()) {
if (n % 1000 == 0) {
writer.flush();
System.out.println(n);
System.gc();
}
writer.encodeAudio(0, samples);
}
} catch (Exception e) {
System.out.println(e);
}
}
} else {
do {
} while (false);
}
}
for (int i = 0; i < container.getDuration() / 1000000; i++) {
writer.encodeVideo(1, s1, i, TimeUnit.SECONDS);
}
writer.close();
if (audioCoder != null) {
audioCoder.close();
audioCoder = null;
}
if (container != null) {
container.close();
container = null;
}
return "temp/" + sermon.getFile().getName() + ".flv";
}
Thanks, good luck.

How to read file from end to start (in reverse order) in Java?

I want to read file in opposite direction from end to the start my file,
[1322110800] LOG ROTATION: DAILY
[1322110800] LOG VERSION: 2.0
[1322110800] CURRENT HOST STATE:arsalan.hussain;DOWN;HARD;1;CRITICAL - Host Unreachable (192.168.1.107)
[1322110800] CURRENT HOST STATE: localhost;UP;HARD;1;PING OK - Packet loss = 0%, RTA = 0.06 ms
[1322110800] CURRENT HOST STATE: musewerx-72c7b0;UP;HARD;1;PING OK - Packet loss = 0%, RTA = 0.27 ms
i use code to read it in this way,
String strpath="/var/nagios.log";
FileReader fr = new FileReader(strpath);
BufferedReader br = new BufferedReader(fr);
String ch;
int time=0;
String Conversion="";
do {
ch = br.readLine();
out.print(ch+"<br/>");
} while (ch != null);
fr.close();
I would prefer to read in reverse order using buffer reader

I had the same problem as described here. I want to look at lines in file in reverse order, from the end back to the start (The unix tac command will do it).
However my input files are fairly large so reading the whole file into memory, as in the other examples was not really a workable option for me.
Below is the class I came up with, it does use RandomAccessFile, but does not need any buffers, since it just retains pointers to the file itself, and works with the standard InputStream methods.
It works for my cases, and empty files and a few other things I've tried. Now I don't have Unicode characters or anything fancy, but as long as the lines are delimited by LF, and even if they have a LF + CR it should work.
Basic Usage is :
in = new BufferedReader (new InputStreamReader (new ReverseLineInputStream(file)));
while(true) {
String line = in.readLine();
if (line == null) {
break;
}
System.out.println("X:" + line);
}
Here is the main source:
package www.kosoft.util;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
public class ReverseLineInputStream extends InputStream {
RandomAccessFile in;
long currentLineStart = -1;
long currentLineEnd = -1;
long currentPos = -1;
long lastPosInFile = -1;
public ReverseLineInputStream(File file) throws FileNotFoundException {
in = new RandomAccessFile(file, "r");
currentLineStart = file.length();
currentLineEnd = file.length();
lastPosInFile = file.length() -1;
currentPos = currentLineEnd;
}
public void findPrevLine() throws IOException {
currentLineEnd = currentLineStart;
// There are no more lines, since we are at the beginning of the file and no lines.
if (currentLineEnd == 0) {
currentLineEnd = -1;
currentLineStart = -1;
currentPos = -1;
return;
}
long filePointer = currentLineStart -1;
while ( true) {
filePointer--;
// we are at start of file so this is the first line in the file.
if (filePointer < 0) {
break;
}
in.seek(filePointer);
int readByte = in.readByte();
// We ignore last LF in file. search back to find the previous LF.
if (readByte == 0xA && filePointer != lastPosInFile ) {
break;
}
}
// we want to start at pointer +1 so we are after the LF we found or at 0 the start of the file.
currentLineStart = filePointer + 1;
currentPos = currentLineStart;
}
public int read() throws IOException {
if (currentPos < currentLineEnd ) {
in.seek(currentPos++);
int readByte = in.readByte();
return readByte;
}
else if (currentPos < 0) {
return -1;
}
else {
findPrevLine();
return read();
}
}
}

Apache Commons IO has the ReversedLinesFileReader class for this now (well, since version 2.2).
So your code could be:
String strpath="/var/nagios.log";
ReversedLinesFileReader fr = new ReversedLinesFileReader(new File(strpath));
String ch;
int time=0;
String Conversion="";
do {
ch = fr.readLine();
out.print(ch+"<br/>");
} while (ch != null);
fr.close();

The ReverseLineInputStream posted above is exactly what I was looking for. The files I am reading are large and cannot be buffered.
There are a couple of bugs:
File is not closed
if the last line is not terminated the last 2 lines are returned on the first read.
Here is the corrected code:
package www.kosoft.util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
public class ReverseLineInputStream extends InputStream {
RandomAccessFile in;
long currentLineStart = -1;
long currentLineEnd = -1;
long currentPos = -1;
long lastPosInFile = -1;
int lastChar = -1;
public ReverseLineInputStream(File file) throws FileNotFoundException {
in = new RandomAccessFile(file, "r");
currentLineStart = file.length();
currentLineEnd = file.length();
lastPosInFile = file.length() -1;
currentPos = currentLineEnd;
}
private void findPrevLine() throws IOException {
if (lastChar == -1) {
in.seek(lastPosInFile);
lastChar = in.readByte();
}
currentLineEnd = currentLineStart;
// There are no more lines, since we are at the beginning of the file and no lines.
if (currentLineEnd == 0) {
currentLineEnd = -1;
currentLineStart = -1;
currentPos = -1;
return;
}
long filePointer = currentLineStart -1;
while ( true) {
filePointer--;
// we are at start of file so this is the first line in the file.
if (filePointer < 0) {
break;
}
in.seek(filePointer);
int readByte = in.readByte();
// We ignore last LF in file. search back to find the previous LF.
if (readByte == 0xA && filePointer != lastPosInFile ) {
break;
}
}
// we want to start at pointer +1 so we are after the LF we found or at 0 the start of the file.
currentLineStart = filePointer + 1;
currentPos = currentLineStart;
}
public int read() throws IOException {
if (currentPos < currentLineEnd ) {
in.seek(currentPos++);
int readByte = in.readByte();
return readByte;
} else if (currentPos > lastPosInFile && currentLineStart < currentLineEnd) {
// last line in file (first returned)
findPrevLine();
if (lastChar != '\n' && lastChar != '\r') {
// last line is not terminated
return '\n';
} else {
return read();
}
} else if (currentPos < 0) {
return -1;
} else {
findPrevLine();
return read();
}
}
#Override
public void close() throws IOException {
if (in != null) {
in.close();
in = null;
}
}
}

The proposed ReverseLineInputStream works really slow when you try to read thousands of lines. At my PC Intel Core i7 on SSD drive it was about 60k lines in 80 seconds. Here is the inspired optimized version with buffered reading (opposed to one-byte-at-a-time reading in ReverseLineInputStream). 60k lines log file is read in 400 milliseconds:
public class FastReverseLineInputStream extends InputStream {
private static final int MAX_LINE_BYTES = 1024 * 1024;
private static final int DEFAULT_BUFFER_SIZE = 1024 * 1024;
private RandomAccessFile in;
private long currentFilePos;
private int bufferSize;
private byte[] buffer;
private int currentBufferPos;
private int maxLineBytes;
private byte[] currentLine;
private int currentLineWritePos = 0;
private int currentLineReadPos = 0;
private boolean lineBuffered = false;
public ReverseLineInputStream(File file) throws IOException {
this(file, DEFAULT_BUFFER_SIZE, MAX_LINE_BYTES);
}
public ReverseLineInputStream(File file, int bufferSize, int maxLineBytes) throws IOException {
this.maxLineBytes = maxLineBytes;
in = new RandomAccessFile(file, "r");
currentFilePos = file.length() - 1;
in.seek(currentFilePos);
if (in.readByte() == 0xA) {
currentFilePos--;
}
currentLine = new byte[maxLineBytes];
currentLine[0] = 0xA;
this.bufferSize = bufferSize;
buffer = new byte[bufferSize];
fillBuffer();
fillLineBuffer();
}
#Override
public int read() throws IOException {
if (currentFilePos <= 0 && currentBufferPos < 0 && currentLineReadPos < 0) {
return -1;
}
if (!lineBuffered) {
fillLineBuffer();
}
if (lineBuffered) {
if (currentLineReadPos == 0) {
lineBuffered = false;
}
return currentLine[currentLineReadPos--];
}
return 0;
}
private void fillBuffer() throws IOException {
if (currentFilePos < 0) {
return;
}
if (currentFilePos < bufferSize) {
in.seek(0);
in.read(buffer);
currentBufferPos = (int) currentFilePos;
currentFilePos = -1;
} else {
in.seek(currentFilePos);
in.read(buffer);
currentBufferPos = bufferSize - 1;
currentFilePos = currentFilePos - bufferSize;
}
}
private void fillLineBuffer() throws IOException {
currentLineWritePos = 1;
while (true) {
// we've read all the buffer - need to fill it again
if (currentBufferPos < 0) {
fillBuffer();
// nothing was buffered - we reached the beginning of a file
if (currentBufferPos < 0) {
currentLineReadPos = currentLineWritePos - 1;
lineBuffered = true;
return;
}
}
byte b = buffer[currentBufferPos--];
// \n is found - line fully buffered
if (b == 0xA) {
currentLineReadPos = currentLineWritePos - 1;
lineBuffered = true;
break;
// just ignore \r for now
} else if (b == 0xD) {
continue;
} else {
if (currentLineWritePos == maxLineBytes) {
throw new IOException("file has a line exceeding " + maxLineBytes
+ " bytes; use constructor to pickup bigger line buffer");
}
// write the current line bytes in reverse order - reading from
// the end will produce the correct line
currentLine[currentLineWritePos++] = b;
}
}
}}

#Test
public void readAndPrintInReverseOrder() throws IOException {
String path = "src/misctests/test.txt";
BufferedReader br = null;
try {
br = new BufferedReader(new FileReader(path));
Stack<String> lines = new Stack<String>();
String line = br.readLine();
while(line != null) {
lines.push(line);
line = br.readLine();
}
while(! lines.empty()) {
System.out.println(lines.pop());
}
} finally {
if(br != null) {
try {
br.close();
} catch(IOException e) {
// can't help it
}
}
}
}
Note that this code reads the hole file into memory and then starts printing it. This is the only way you can do it with a buffered reader or anry other reader that does not support seeking. You have to keep this in mind, in your case you want to read a log file, log files can be very big!
If you want to read line by line and print on the fly then you have no other alternative than using a reader that support seeking such as java.io.RandomAccessFile and this anything but trivial.

As far as I understand, you try to read backwards line by line.
Suppose this is the file you try to read:
line1
line2
line3
And you want to write it to the output stream of the servlet as follows:
line3
line2
line1
Following code might be helpful in this case:
List<String> tmp = new ArrayList<String>();
do {
ch = br.readLine();
tmp.add(ch);
out.print(ch+"<br/>");
} while (ch != null);
for(int i=tmp.size()-1;i>=0;i--) {
out.print(tmp.get(i)+"<br/>");
}

I had a problem with your solution #dpetruha because of this:
Does RandomAccessFile.read() from local file guarantee that exact number of bytes will be read?
Here is my solution: (changed only fillBuffer)
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
public class ReverseLineInputStream extends InputStream {
private static final int MAX_LINE_BYTES = 1024 * 1024;
private static final int DEFAULT_BUFFER_SIZE = 1024 * 1024;
private RandomAccessFile in;
private long currentFilePos;
private int bufferSize;
private byte[] buffer;
private int currentBufferPos;
private int maxLineBytes;
private byte[] currentLine;
private int currentLineWritePos = 0;
private int currentLineReadPos = 0;
private boolean lineBuffered = false;
public ReverseLineInputStream(File file) throws IOException {
this(file, DEFAULT_BUFFER_SIZE, MAX_LINE_BYTES);
}
public ReverseLineInputStream(File file, int bufferSize, int maxLineBytes) throws IOException {
this.maxLineBytes = maxLineBytes;
in = new RandomAccessFile(file, "r");
currentFilePos = file.length() - 1;
in.seek(currentFilePos);
if (in.readByte() == 0xA) {
currentFilePos--;
}
currentLine = new byte[maxLineBytes];
currentLine[0] = 0xA;
this.bufferSize = bufferSize;
buffer = new byte[bufferSize];
fillBuffer();
fillLineBuffer();
}
#Override
public int read() throws IOException {
if (currentFilePos <= 0 && currentBufferPos < 0 && currentLineReadPos < 0) {
return -1;
}
if (!lineBuffered) {
fillLineBuffer();
}
if (lineBuffered) {
if (currentLineReadPos == 0) {
lineBuffered = false;
}
return currentLine[currentLineReadPos--];
}
return 0;
}
private void fillBuffer() throws IOException {
if (currentFilePos < 0) {
return;
}
if (currentFilePos < bufferSize) {
in.seek(0);
buffer = new byte[(int) currentFilePos + 1];
in.readFully(buffer);
currentBufferPos = (int) currentFilePos;
currentFilePos = -1;
} else {
in.seek(currentFilePos - buffer.length);
in.readFully(buffer);
currentBufferPos = bufferSize - 1;
currentFilePos = currentFilePos - bufferSize;
}
}
private void fillLineBuffer() throws IOException {
currentLineWritePos = 1;
while (true) {
// we've read all the buffer - need to fill it again
if (currentBufferPos < 0) {
fillBuffer();
// nothing was buffered - we reached the beginning of a file
if (currentBufferPos < 0) {
currentLineReadPos = currentLineWritePos - 1;
lineBuffered = true;
return;
}
}
byte b = buffer[currentBufferPos--];
// \n is found - line fully buffered
if (b == 0xA) {
currentLineReadPos = currentLineWritePos - 1;
lineBuffered = true;
break;
// just ignore \r for now
} else if (b == 0xD) {
continue;
} else {
if (currentLineWritePos == maxLineBytes) {
throw new IOException("file has a line exceeding " + maxLineBytes
+ " bytes; use constructor to pickup bigger line buffer");
}
// write the current line bytes in reverse order - reading from
// the end will produce the correct line
currentLine[currentLineWritePos++] = b;
}
}
}
}