Develop Reference

How to update a byte array in a method, without running it again?

I have a class(an AsyncTask) which does image processing and generates yuv bytes continously, at around ~200ms interval.
Now I send these yuv bytes to another method where the they are recorded using FFmpeg frame recorder:
public void recordYuvData() {
byte[] yuv = getNV21();
System.out.println(yuv.length + " returned yuv bytes ");
if (audioRecord == null || audioRecord.getRecordingState() != AudioRecord.RECORDSTATE_RECORDING) {
startTime = System.currentTimeMillis();
return;
}
if (RECORD_LENGTH > 0) {
int i = imagesIndex++ % images.length;
yuvimage = images[i];
timestamps[i] = 1000 * (System.currentTimeMillis() - startTime);
}
/* get video data */
if (yuvimage != null && recording) {
((ByteBuffer) yuvimage.image[0].position(0)).put(yuv);
if (RECORD_LENGTH <= 0) {
try {
long t = 1000 * (System.currentTimeMillis() - startTime);
if (t > recorder.getTimestamp()) {
recorder.setTimestamp(t);
}
recorder.record(yuvimage);
} catch (FFmpegFrameRecorder.Exception e) {
e.printStackTrace();
}
}
}
}
This method; recordYuvData() is initiated on button click.
If I initiate it only once , then only the initial image gets recorded, rest are not.
If I initiate this each time after the end of the image processing it records but leads to 'weird' fps count of the video; and finally this leads to application crash after sometime.
For above what I feel is, at the end of image processing a new instance of recordYuvData() is created without ending the previous one, accumulating many instances of recordYuvData(). [correct me if I am wrong]
So, how do I update 'ONLY' yuv bytes in the method without running it again?
Thanks....!
Edit:
On Click:
record.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View v) {
recordYuvdata();
startRecording();
getNV21()
byte[] getNV21(Bitmap bitmap) {
int inputWidth = 1024;
int inputHeight = 640;
int[] argb = new int[inputWidth * inputHeight];
bitmap.getPixels(argb, 0, inputWidth, 0, 0, inputWidth, inputHeight);
System.out.println(argb.length + "#getpixels ");
byte[] yuv = new byte[inputWidth * inputHeight * 3 / 2];
encodeYUV420SP(yuv, argb, inputWidth, inputHeight);
return yuv;
}
void encodeYUV420SP(byte[] yuv420sp, int[] argb, int width, int height) {
final int frameSize = width * height;
int yIndex = 0;
int uvIndex = frameSize;
System.out.println(yuv420sp.length + " #encoding " + frameSize);
int a, R, G, B, Y, U, V;
int index = 0;
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
a = (argb[index] & 0xff000000) >> 24; // a is not used obviously
R = (argb[index] & 0xff0000) >> 16;
G = (argb[index] & 0xff00) >> 8;
B = (argb[index] & 0xff) >> 0;
// well known RGB to YUV algorithm
Y = ((66 * R + 129 * G + 25 * B + 128) >> 8) + 16;
U = ((-38 * R - 74 * G + 112 * B + 128) >> 8) + 128;
V = ((112 * R - 94 * G - 18 * B + 128) >> 8) + 128;
// NV21 has a plane of Y and interleaved planes of VU each sampled by a factor of 2
// meaning for every 4 Y pixels there are 1 V and 1 U. Note the sampling is every other
// pixel AND every other scanline.
yuv420sp[yIndex++] = (byte) ((Y < 0) ? 0 : ((Y > 255) ? 255 : Y));
if (j % 2 == 0 && index % 2 == 0) {
yuv420sp[uvIndex++] = (byte) ((V < 0) ? 0 : ((V > 255) ? 255 : V));
yuv420sp[uvIndex++] = (byte) ((U < 0) ? 0 : ((U > 255) ? 255 : U));
}
index++;
}
}
}

Convert JPEG byte[] to NV21 byte[]

I am trying to convert JPEG byte[] data from Camera.PictureCallback to NV21 byte[] format but it didn't work.
I try to do this:
byte [] getNV21(int inputWidth, int inputHeight, Bitmap scaled) {
int [] argb = new int[inputWidth * inputHeight];
scaled.getPixels(argb, 0, inputWidth, 0, 0, inputWidth, inputHeight);
byte [] yuv = new byte[inputWidth*inputHeight*3/2];
encodeYUV420SP(yuv, argb, inputWidth, inputHeight);
scaled.recycle();
return yuv;
}
void encodeYUV420SP(byte[] yuv420sp, int[] argb, int width, int height) {
final int frameSize = width * height;
int yIndex = 0;
int uvIndex = frameSize;
int a, R, G, B, Y, U, V;
int index = 0;
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
a = (argb[index] & 0xff000000) >> 24; // a is not used obviously
R = (argb[index] & 0xff0000) >> 16;
G = (argb[index] & 0xff00) >> 8;
B = (argb[index] & 0xff) >> 0;
// well known RGB to YUV algorithm
Y = ( ( 66 * R + 129 * G + 25 * B + 128) >> 8) + 16;
U = ( ( -38 * R - 74 * G + 112 * B + 128) >> 8) + 128;
V = ( ( 112 * R - 94 * G - 18 * B + 128) >> 8) + 128;
// NV21 has a plane of Y and interleaved planes of VU each sampled by a factor of 2
// meaning for every 4 Y pixels there are 1 V and 1 U. Note the sampling is every other
// pixel AND every other scanline.
yuv420sp[yIndex++] = (byte) ((Y < 0) ? 0 : ((Y > 255) ? 255 : Y));
if (j % 2 == 0 && index % 2 == 0) {
yuv420sp[uvIndex++] = (byte)((V<0) ? 0 : ((V > 255) ? 255 : V));
yuv420sp[uvIndex++] = (byte)((U<0) ? 0 : ((U > 255) ? 255 : U));
}
index ++;
}
}
}

Conversion of Java code to C -> call by JNI -> program stops running

I tried to convert this Java code:
// http://www.stanford.edu/class/ee368/Android/index.html
// Source: http://www.stanford.edu/class/ee368/Android/HelloViewfinder/Project.zip
private void decodeYUV420RGB(int[] rgb, byte[] yuv420sp, int width, int height) {
Convert YUV to RGB
final int frameSize = width * height;
for (int j = 0, yp = 0; j < height; j++) {
int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
for (int i = 0; i < width; i++, yp++) {
int y = (0xff & ((int) yuv420sp[yp])) - 16;
if (y < 0) y = 0;
if ((i & 1) == 0) {
v = (0xff & yuv420sp[uvp++]) - 128;
u = (0xff & yuv420sp[uvp++]) - 128;
}
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0) r = 0; else if (r > 262143) r = 262143;
if (g < 0) g = 0; else if (g > 262143) g = 262143;
if (b < 0) b = 0; else if (b > 262143) b = 262143;
rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);
}
}
}
which is called this way:
//byte[] mYUVData; int[] mRGBData;
decodeYUV420RGB(mRGBData, mYUVData, mImageWidth, mImageHeight);
to this C code:
#include <string.h>
#include <jni.h>
jint*
Java_com_camera_DrawOnTop_decodeYUV420RGB565(JNIEnv* env, jobject thiz, jintArray rgb, jbyteArray yuv420sp, jint width, jint height)
{
jbyte* yuv420spc = (*env)->GetByteArrayElements(env, yuv420sp, NULL);
jint* rgbc = (*env)->GetIntArrayElements(env, rgb, NULL);
int frameSize = width * height;
int j;
int i;
int yp;
for (j = 0, yp = 0; j < height; j++) {
int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
for (i = 0; i < width; i++, yp++) {
int y = (0xff & ((int) yuv420spc[yp])) - 16;
if (y < 0) y = 0;
if ((i & 1) == 0) {
v = (0xff & yuv420spc[uvp++]) - 128;
u = (0xff & yuv420spc[uvp++]) - 128;
}
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0) r = 0; else if (r > 262143) r = 262143;
if (g < 0) g = 0; else if (g > 262143) g = 262143;
if (b < 0) b = 0; else if (b > 262143) b = 262143;
rgbc[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);
}
}
(*env)->ReleaseByteArrayElements(env, yuv420sp, yuv420spc, 0 );
(*env)->ReleaseIntArrayElements(env, rgb, rgbc, 0 );
return rgbc;
}
and call it via JNI:
//int[] mRGBData; int [] tmpData = {1,2,3};
mRGBData = decodeYUV420RGB565(tmpData, mYUVData, mImageWidth, mImageHeight);
But the program breaks running after the above call.
I don't now how to do call by reference with JNI so I used tmpData
only to have data but return real data to mRGBData via equals sign.
What's wrong with my C code so it breaks at running time?
And what have I to change so that it works with reference like
the original code (without equals sign)?

You should provide to your JNI function an RGB array of correct size (that is, w*h), and an YUV array of correct size and structure (w*h*3/2, with w*h luma bytes (Y), followed by (w/2)*(h/2) pairs of chroma (U and V) bytes. The call will crash if you provide rgb array of size 3, as in your snippet.
Also note that you are building an rgb565 array. Its elements are probably expected to be of type short (16 bit) and not int (32 bit).

GetByteArrayElements() creates an array on the C side, which you fill, then release, then return to Java, which gets a pointer to released memory. If you want to actually get the data over to the Java side, you need to create a new Java array object and return that. Or, create the new object on the Java side and pass it to the C code to modify. I find the latter is usually easier. I usually create a DirectByteBuffer on the Java side and pass it into the native function, let the native function call GetDirectBufferAddress and write into that.
For an example, see my PD ojrandlib JNI code: https://github.com/lcrocker/ojrandlib/tree/master/source/java/com/onejoker/randlib

I changed code to this:
Java:
native void decodeYUV420RGB565(ByteBuffer rgb, byte[] yuv420sp, int width, int height);
//There's no allocateDirect for IntBuffer so I have to use ByteBuffer
//although mRGBData is an int array
//allocateDirect(mRGBData.length * 4) because mRGBData is an int array
ByteBuffer mTempData = ByteBuffer.allocateDirect(mRGBData.length*4);
decodeYUV420RGB565(mTempData, mYUVData, mImageWidth, mImageHeight);
mRGBData = mTempData.asIntBuffer().array();
mTempData.clear();
C:
void Java_com_camera_DrawOnTop_decodeYUV420RGB565(JNIEnv* env, jobject thiz, jintArray rgb, jbyteArray yuv420sp, jint width, jint height)
{
jbyte* yuv420spc = (*env)->GetByteArrayElements(env, yuv420sp, NULL);
jint* rgbc = (*env)->GetDirectBufferAddress(env, thiz);
//...conversion code as above...
(*env)->ReleaseByteArrayElements(env, yuv420sp, yuv420spc, 0 );
(*env)->ReleaseIntArrayElements(env, rgb, rgbc, 0 );
But I have the same problem: Code crashes when I call decodeYUV420RGB565(mTempData, mYUVData, mImageWidth, mImageHeight). What have I to change in the code?

NV21 to RGB conversion result in black&white

I'm converting a single pixel of NV21 to RGB, it is what I did:
public int getYUVvalue(byte[] yuv,int width,int height,int x,int y){
int total=width*height;
int Y=(0xff&yuv[y*width+x])-16;
int U=(0xff&yuv[(y/2)*(width/2)+(x/2)+total])-128;
int V=(0xff&yuv[(y/2)*(width/2)+(x/2)+total+1])-128;
return this.convertYUVtoRGB(Y, U, V);
}
private static int convertYUVtoRGB(int y, int u, int v) {
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0) r = 0; else if (r > 262143) r = 262143;
if (g < 0) g = 0; else if (g > 262143) g = 262143;
if (b < 0) b = 0; else if (b > 262143) b = 262143;
return 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);
}
but the result is just black and white, looks like the U and V value are not correct, can someone help me with this issue?
thank you!
update: I confirm my Y value is right, so I think the problem is at the U and V value, which I have no clue where the problem is.
update#2: My code is very similar to decodeYUV420sp(), which is a widely used function to convert the whole image, it works perfectly, mine is just to convert a single pixel, still trying to figure out the problem by comparing it with my code. Here is the decodeyuv420sp function:
static public void decodeYUV420SP(int[] rgb, byte[] yuv420sp, int width, int height) {
final int frameSize = width * height;
for (int j = 0, yp = 0; j < height; j++) {
int uvp = frameSize + (j >> 1) * width, u = 0, v = 0;
for (int i = 0; i < width; i++, yp++) {
int y = (0xff & ((int) yuv420sp[yp])) - 16;
if (y < 0) y = 0;
if ((i & 1) == 0) {
v = (0xff & yuv420sp[uvp++]) - 128;
u = (0xff & yuv420sp[uvp++]) - 128;
}
int y1192 = 1192 * y;
int r = (y1192 + 1634 * v);
int g = (y1192 - 833 * v - 400 * u);
int b = (y1192 + 2066 * u);
if (r < 0) r = 0; else if (r > 262143) r = 262143;
if (g < 0) g = 0; else if (g > 262143) g = 262143;
if (b < 0) b = 0; else if (b > 262143) b = 262143;
rgb[yp] = 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);
}
}
}
I still can't see where my mistake is.
update#3: problem solved, thanks for your help everyone!

The logic for the array indexes of U & V in the getYUVvalue function does not match the logic in decodeYUV420SP. (I count 3 differences.) Without a sample byte array in this format to test with, I can't be sure, but I think it should be:
int U=(0xff&yuv[(y/2)*width+(x&~1)+total+1])-128;
int V=(0xff&yuv[(y/2)*width+(x&~1)+total])-128;
It's missing the if (y < 0) y = 0; check also. Maybe that matters.

Create / Use YUV Image buffer from Camera Preview class

In our application, we need to use capture frame from camera # 33 fps and pass it to compression before sending it to server,
My compression module can take YUV Image, to compress the image, my camera configuration as follows,
Width = 500 px,
height = 300 px ,
Image format : YV12
On the preview callback
camera.setPreviewCallback(new PreviewCallback() {
public void onPreviewFrame(byte[] data, Camera arg1) {
}
}
data size is coming out to be 230400, but i suppose it would be around
500*300(Y) + 500*300/4(U) + 500*300/4(V) i.e. 2250000
i.e. 5400 bytes more, does that mean, i can ignore the reamining one ?
Also i need to create YUVImage object , but stride info is not coming, so how we can create YUVImage from above data.
Sincere Thanks for reading and i really appreciate if anyone can help me out on this.

Cant help with the data size question but to get YUV from Camera preview you have 2 choices. If running Android 2.2 or later your can use the android.graphics.YuvImage class and just pass it's constructor your bytearray from PreviewCallback.
If you need to support pre 2.2 then you need to do something like:
/**
* Decodes YUV frame to a buffer which can be use to create a bitmap.
* use this for OS < FROYO which has a native YUV decoder
* decode Y, U, and V values on the YUV 420 buffer described as YCbCr_422_SP by Android
* #param rgb the outgoing array of RGB bytes
* #param fg the incoming frame bytes
* #param width of source frame
* #param height of source frame
* #throws NullPointerException
* #throws IllegalArgumentException
*/
private static void decodeYUV_impl(int[] rgb, byte[] fg, int width, int height) throws NullPointerException, IllegalArgumentException
{
int sz = width * height;
if (rgb == null)
throw new NullPointerException("buffer out is null");
if (rgb.length < sz)
throw new IllegalArgumentException("buffer out size " + rgb.length
+ " < minimum " + sz);
if (fg == null)
throw new NullPointerException("buffer 'fg' is null");
if (fg.length < sz)
throw new IllegalArgumentException("buffer fg size " + fg.length
+ " < minimum " + sz * 3 / 2);
int i, j;
int Y, Cr = 0, Cb = 0;
for (j = 0; j < height; j++) {
int pixPtr = j * width;
final int jDiv2 = j >> 1;
for (i = 0; i < width; i++) {
Y = fg[pixPtr];
if (Y < 0)
Y += 255;
if ((i & 0x1) != 1) {
final int cOff = sz + jDiv2 * width + (i >> 1) * 2;
Cb = fg[cOff];
if (Cb < 0)
Cb += 127;
else
Cb -= 128;
Cr = fg[cOff + 1];
if (Cr < 0)
Cr += 127;
else
Cr -= 128;
}
int R = Y + Cr + (Cr >> 2) + (Cr >> 3) + (Cr >> 5);
if (R < 0)
R = 0;
else if (R > 255)
R = 255;
int G = Y - (Cb >> 2) + (Cb >> 4) + (Cb >> 5) - (Cr >> 1)
+ (Cr >> 3) + (Cr >> 4) + (Cr >> 5);
if (G < 0)
G = 0;
else if (G > 255)
G = 255;
int B = Y + Cb + (Cb >> 1) + (Cb >> 2) + (Cb >> 6);
if (B < 0)
B = 0;
else if (B > 255)
B = 255;
rgb[pixPtr++] = (0xff000000 + (B << 16) + (G << 8) + R);
}
}
}