Develop Reference

Android Camera2 conjunction between requests

I am currently trying to implement computer vision algorithms on my android device. My idea is to have a camera preview running and draw on top of the preview (in a separate view)
edges and other geometric figures. To command the camera, I use the camera2 API.
I start a capture session with two requests with the following targets:
Request number 1: target/surface -> an ImageReader for image processing
Request number 2: target/surface -> a SurfaceView for camera preview
I burst both requests with setRepeatingBurst. Now a problem occurs when I process the image in the
ImageReader callback (onImageAvailable). When I process for let's say 500 ms then the image preview drops to 1 fps. I think the problem is because the image.close() gets delayed and that the second request is somehow connected to that image.
I thought that if I have two requests then there are sent two individual images to each surface. But apparently that's not the case. Well, I could make a quick copy in the ImageReader callback and close the image afterwards. And then start processing the copy. But the thing is that I don't
want to copy.
Here is the code:
HandlerThread imageProcessingThread = new HandlerThread("imageProcessingThread");
imageProcessingThread.start();
Handler imageProcessingHandler = new Handler(imageProcessingThread.getLooper());
ImageReader imageReader = ImageReader.newInstance(width, height, format, 1);
imageReader.setOnImageAvailableListener(new ImageReader.OnImageAvailableListener() {
#Override
public void onImageAvailable(ImageReader reader) {
Image image = reader.acquireNextImage();
if(image == null) return;
// simulating processing
try {
Thread.sleep(500);
} catch (InterruptedException e) {
e.printStackTrace();
}
// processing end
image.close();
}
}, imageProcessingHandler);
try {
HandlerThread imageProcessingThread3 = new HandlerThread("imageProcessingThread3");
imageProcessingThread3.start();
Handler imageProcessingHandler3 = new Handler(imageProcessingThread3.getLooper());
camManager.openCamera(cam.second, new CameraDevice.StateCallback() {
#Override
public void onOpened(#NonNull CameraDevice cameraDevice) {
try {
List<Surface> surfaces = new ArrayList<>();
surfaces.add(binding.svPreview.getHolder().getSurface());
surfaces.add(imageReader.getSurface());
cameraDevice.createCaptureSession(surfaces, new CameraCaptureSession.StateCallback() {
#Override
public void onConfigured(#NonNull CameraCaptureSession session) {
try {
CaptureRequest.Builder requestBuilder1 = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
requestBuilder1.addTarget(imageReader.getSurface());
requestBuilder1.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, new Range<>(30, 30));
CaptureRequest.Builder requestBuilder2 = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
requestBuilder2.addTarget(binding.svPreview.getHolder().getSurface());
requestBuilder2.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, new Range<>(30, 30));
HandlerThread imageProcessingThread2 = new HandlerThread("imageProcessingThread2");
imageProcessingThread2.start();
Handler imageProcessingHandler2 = new Handler(imageProcessingThread2.getLooper());
List<CaptureRequest> requests = new ArrayList<>();
requests.add(requestBuilder1.build());
requests.add(requestBuilder2.build());
session.setRepeatingBurst(requests, new CameraCaptureSession.CaptureCallback() {
#Override
public void onCaptureProgressed(#NonNull CameraCaptureSession session, #NonNull CaptureRequest request, #NonNull CaptureResult partialResult) {
}
#Override
public void onCaptureCompleted(#NonNull CameraCaptureSession session, #NonNull CaptureRequest request, #NonNull TotalCaptureResult result) {
}
}, imageProcessingHandler2);
} catch (Exception ex) {
fatalError(ex.getMessage());
return;
}
}
#Override
public void onConfigureFailed(#NonNull CameraCaptureSession cameraCaptureSession) {
fatalError("onConfigureFailed");
return;
}
}, null);
} catch (Exception ex) {
fatalError(ex.getMessage());
return;
}
}
#Override
public void onDisconnected(#NonNull CameraDevice cameraDevice) {
}
#Override
public void onError(#NonNull CameraDevice cameraDevice, int i) {
}
}, imageProcessingHandler3);
} catch (Exception ex) {
fatalError(ex.getMessage());
return;
}

video recording using camera 2

I am trying to record video using media recorder and camera2 but the app crashes as soon as mediarecorder.start() function is encountered. In the oncreate first prepareCamera is called and then trigger is called. I am a bit new to camera2. Can anyone help me find out why it is happening so?
public void prepareCamera() throws CameraAccessException {
manager = (CameraManager) getSystemService(CAMERA_SERVICE);
String[] cameras = manager.getCameraIdList();
if (ActivityCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
Log.v("mycontroller","permission not granted");
return;
}
Log.v("mycontroller","permission granted "+cameras[0]);
manager.openCamera(cameras[0], new CameraDevice.StateCallback(){
#Override
public void onOpened(CameraDevice camera) {
Log.v("mycontroller","camera opened");
mCamera2 = camera;
mediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
mediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4);
mediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.MPEG_4_SP);
try {
mediaRecorder.setOutputFile(createFile().getAbsolutePath());
mediaRecorder.prepare();
Log.v("mycontroller","recorder prepared");
List<Surface> list = new ArrayList<>();
list.add(mediaRecorder.getSurface());
final CaptureRequest.Builder captureRequest = mCamera2.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
captureRequest.addTarget(mediaRecorder.getSurface());
mCaptureRequest = captureRequest.build();
mCamera2.createCaptureSession(list, new CameraCaptureSession.StateCallback(){
#Override
public void onConfigured(CameraCaptureSession session) {
mSession = session;
}
#Override
public void onConfigureFailed(CameraCaptureSession session) {
mSession = session;
}
}, null);
} catch (Exception e) {
e.printStackTrace();
}
}
#Override
public void onDisconnected(CameraDevice camera) {}
#Override
public void onError(CameraDevice camera, int error) {}
}, null);
}
public void trigger() {
try {
mediaRecorder.start();
mSession.setRepeatingRequest(mCaptureRequest,
new CameraCaptureSession.CaptureCallback() {
#Override
public void onCaptureStarted(CameraCaptureSession session, CaptureRequest request, long timestamp, long frameNumber) {
Log.v("mycontroller","camera started capturing");
super.onCaptureStarted(session, request, timestamp, frameNumber);
}
#Override
public void onCaptureCompleted(CameraCaptureSession session, CaptureRequest request, TotalCaptureResult result) {
Log.v("mycontroller","camera stoped capturing");
super.onCaptureCompleted(session, request, result);
}
}, null);
} catch (CameraAccessException e) {
Log.v("mycontroller",e.getMessage());
e.printStackTrace();
}
}
private void releaseMediaRecorder() throws CameraAccessException {
mSession.stopRepeating();
try {
mediaRecorder.stop();
mediaRecorder.reset();
mediaRecorder.release();
}
catch (Exception e){}
mediaRecorder= null;
mCamera2=null;
}

Take a look at Google's Camera2Video sample to see if you can find any critical differences between your code and the sample:
https://github.com/googlearchive/android-Camera2Video/tree/master/Application/src/main/java/com/example/android/camera2video
You may need to set more MediaRecorder settings like resolution; often the CamcorderProfile class is used to do this easily.

Why turning on flash change the focus mode of the camera?

I have 2 AF Focus Distance can be control by button and an On Off switch for flash.
Why IF ELSE for flashligh in updatePreview() will affect focus mode to auto? When I comment out that part (IF ELSE for flashligh), AF mode and Focus distance work fine with the btnFocus.
How to solve this while still can allow user to turn on and off the flashlight?
private float focusDistance = 0;
private int flashSwitch = 0;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
textureView = (TextureView)findViewById(R.id.textureView);
//From Java 1.4 , you can use keyword 'assert' to check expression true or false
assert textureView != null;
textureView.setSurfaceTextureListener(textureListener);
btnFocus = (Button)findViewById(R.id.btnFocus);
btnFlash = (Button)findViewById(R.id.btnFlash);
btnFocus.setOnClickListener(new View.OnClickListener() {
#Override
public void onClick(View view) {
changeFocus();
}
});
btnFlash.setOnClickListener(new View.OnClickListener(){
#Override
public void onClick(View view) {
Flash();
}
});
}
private void Flash() {
if(cameraDevice == null)
return;
if(flashSwitch == 0 ) {
Toast.makeText(MainActivity.this, "Flash Turn On ", Toast.LENGTH_SHORT).show();
flashSwitch = 1;
}
else{
Toast.makeText(MainActivity.this, "Flash Turn Off ", Toast.LENGTH_SHORT).show();
flashSwitch = 0;
}
createCameraPreview();
}
private void changeFocus() {
if(cameraDevice == null)
return;
if(focusDistance == 0 )
focusDistance = 10;
else
focusDistance = 0;
Toast.makeText(MainActivity.this, "Focus Change "+ focusDistance, Toast.LENGTH_SHORT).show();
createCameraPreview();
}
private void createCameraPreview() {
try{
SurfaceTexture texture = textureView.getSurfaceTexture();
assert texture != null;
texture.setDefaultBufferSize(imageDimension.getWidth(),imageDimension.getHeight());
Surface surface = new Surface(texture);
captureRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
captureRequestBuilder.addTarget(surface);
cameraDevice.createCaptureSession(Arrays.asList(surface), new CameraCaptureSession.StateCallback() {
#Override
public void onConfigured(#NonNull CameraCaptureSession cameraCaptureSession) {
if(cameraDevice == null)
return;
cameraCaptureSessions = cameraCaptureSession;
updatePreview();
}
#Override
public void onConfigureFailed(#NonNull CameraCaptureSession cameraCaptureSession) {
Toast.makeText(MainActivity.this, "Changed", Toast.LENGTH_SHORT).show();
}
},null);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
If you uncomment part the problem will come out when turn on flash and change focus.
private void updatePreview() {
if(cameraDevice == null)
Toast.makeText(this, "Error", Toast.LENGTH_SHORT).show();
try{
// if you uncomment this part the problem will come out
// if(flashSwitch == 0){
// captureRequestBuilder.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_OFF);
// }
// else{
// captureRequestBuilder.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_TORCH);
// }
captureRequestBuilder.set(CaptureRequest.CONTROL_AF_MODE,CaptureRequest.CONTROL_AF_MODE_OFF);
captureRequestBuilder.set(CaptureRequest.LENS_FOCUS_DISTANCE, focusDistance);
cameraCaptureSessions.setRepeatingRequest(captureRequestBuilder.build(),null,mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
Expected Output
Can manually turn on and off flash without affecting focus mode.

I think that your problem might be on the way you set the requests.
You should create the preview first and only once, the way you have it, you are creating a preview everytime you switch flash and/or focus.
I suggest to create the preview in onStart with some defaults like:
focusDistance = 0;
flashSwitch = CaptureRequest.FLASH_MODE_OFF;
And then in Flash() and changeFocus() instead of calling createCameraPreview() you call updatePreview(), your update preview would look something like this:
private void updatePreview() {
if(cameraDevice == null)
Toast.makeText(this, "Error", Toast.LENGTH_SHORT).show();
try{
captureRequestBuilder.set(CaptureRequest.FLASH_MODE, flashSwitch);
captureRequestBuilder.set(CaptureRequest.LENS_FOCUS_DISTANCE, focusDistance);
cameraCaptureSessions.setRepeatingRequest(captureRequestBuilder.build(),null,mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}

In updatePreview(), you're sending multiple capture requests to cameraCaptureSessions before settling all keys in captureRequestBuilder. Trying setting all keys first and then sending one repeating capture request.
private void updatePreview() {
if(cameraDevice == null)
Toast.makeText(this, "Error", Toast.LENGTH_SHORT).show();
try{
if(flashSwitch == 0){
captureRequestBuilder.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_OFF);
}
else{
captureRequestBuilder.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_TORCH);
}
captureRequestBuilder.set(CaptureRequest.CONTROL_AF_MODE,CaptureRequest.CONTROL_AF_MODE_OFF);
captureRequestBuilder.set(CaptureRequest.LENS_FOCUS_DISTANCE, focusDistance);
cameraCaptureSessions.setRepeatingRequest(captureRequestBuilder.build(),null,mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}

setPreviewTexture failed when taking second photo with Camera2

I'm trying to configure a camera compatibility without preview but after taking second photo the app crashes with exception:
2018-12-27 14:36:20.392 12389-12977/com.example.android.braillefeeder E/RequestThread-0: Received device exception during capture call:
java.io.IOException: setPreviewTexture failed
at android.hardware.Camera.setPreviewTexture(Native Method)
at android.hardware.camera2.legacy.RequestThreadManager.doJpegCapturePrepare(RequestThreadManager.java:298)
at android.hardware.camera2.legacy.RequestThreadManager.-wrap1(Unknown Source:0)
at android.hardware.camera2.legacy.RequestThreadManager$5.handleMessage(RequestThreadManager.java:830)
at android.os.Handler.dispatchMessage(Handler.java:101)
at android.os.Looper.loop(Looper.java:164)
at android.os.HandlerThread.run(HandlerThread.java:65)
I have this code:
I'm targeting for 28 SDK version. I thought the problem may have something with thread so I tried to initialize thread in main but with no success.
public class CameraService {
// Size of taken photo
private static final int IMAGE_WIDTH = 1280;
private static final int IMAGE_HEIGHT = 960;
private CameraDevice mCameraDevice;
private CameraCaptureSession mCameraCaptureSession;
private HandlerThread backgroundThread;
private Handler backgroundHandler;
private ImageReader mImageReader;
private CameraService() {
}
private static class InstanceHolder {
private static CameraService sCameraService = new CameraService();
}
public static CameraService getInstance() {
return InstanceHolder.sCameraService;
}
public void initializeCamera(Context context,
ImageReader.OnImageAvailableListener onImageAvailableListener) {
CameraManager cameraManager = (CameraManager) context.getSystemService(CAMERA_SERVICE);
String[] camIds = {};
try {
camIds = cameraManager.getCameraIdList();
} catch (CameraAccessException e) {
e.printStackTrace();
}
if( camIds.length < 1) {
Log.e("CameraService", "Camera not available.");
return;
}
startBackgroundThread();
mImageReader = ImageReader.newInstance(IMAGE_WIDTH, IMAGE_HEIGHT, ImageFormat.JPEG, 2);
mImageReader.setOnImageAvailableListener(onImageAvailableListener, backgroundHandler);
try {
cameraManager.openCamera(camIds[0], mStateCallback, backgroundHandler);
} catch (SecurityException e) {
e.printStackTrace();
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
private void startBackgroundThread() {
backgroundThread = new HandlerThread("CameraBackground");
backgroundThread.start();
backgroundHandler = new Handler(backgroundThread.getLooper());
}
private void stopBackgroundThread() {
backgroundThread.quitSafely();
try {
backgroundThread.join();
backgroundThread = null;
backgroundHandler = null;
} catch (InterruptedException e) {
e.printStackTrace();
}
}
private final CameraDevice.StateCallback mStateCallback = new CameraDevice.StateCallback() {
#Override
public void onOpened(#NonNull CameraDevice cameraDevice) {
mCameraDevice = cameraDevice;
}
#Override
public void onDisconnected(#NonNull CameraDevice cameraDevice) {
cameraDevice.close();
mCameraDevice = null;
}
#Override
public void onError(#NonNull CameraDevice cameraDevice, int i) {
cameraDevice.close();
mCameraDevice = null;
}
#Override
public void onClosed(#NonNull CameraDevice camera) {
mCameraDevice = null;
}
};
public void takePicture() {
Log.d("CameraService", "takePicture()");
if( mCameraDevice == null) {
Log.d("CameraService", "Cannot take picture. Camera device is null.");
return;
}
try {
mCameraDevice.createCaptureSession(Collections.singletonList(mImageReader.getSurface()),
new CameraCaptureSession.StateCallback() {
#Override
public void onConfigured(CameraCaptureSession cameraCaptureSession) {
if( mCameraDevice == null) {
Log.e("mStateCallback", " mStateCallbackCaptureSession configured");
return;
}
Log.d("CameraService", "imageCapture()");
mCameraCaptureSession = cameraCaptureSession;
imageCapture();
}
#Override
public void onConfigureFailed(CameraCaptureSession cameraCaptureSession) {
Log.e("mStateCallback", "Configure failed");
}
}, null);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
private void imageCapture() {
Log.d("CameraService", "imageCapture()");
try {
final CaptureRequest.Builder builder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_STILL_CAPTURE);
builder.addTarget(mImageReader.getSurface());
builder.set(CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON);
mCameraCaptureSession.stopRepeating();
mCameraCaptureSession.capture(builder.build(), mCaptureCallback, null);
} catch (CameraAccessException e) {
Log.e("imagecapture()", "KOKOTKO");
e.printStackTrace();
}
}
private final CameraCaptureSession.CaptureCallback mCaptureCallback =
new CameraCaptureSession.CaptureCallback() {
#Override
public void onCaptureStarted(#NonNull CameraCaptureSession session, #NonNull CaptureRequest request, long timestamp, long frameNumber) {
super.onCaptureStarted(session, request, timestamp, frameNumber);
}
#Override
public void onCaptureProgressed(#NonNull CameraCaptureSession session, #NonNull CaptureRequest request, #NonNull CaptureResult partialResult) {
super.onCaptureProgressed(session, request, partialResult);
}
#Override
public void onCaptureCompleted(#NonNull CameraCaptureSession session, #NonNull CaptureRequest request, #NonNull TotalCaptureResult result) {
if( session != null) {
session.close();
}
}
};
public void shutdown() {
Log.d("CameraService", "shutdown()");
if( mCameraDevice != null) {
mCameraDevice.close();
}
if( mCameraCaptureSession != null) {
mCameraCaptureSession.close();
}
stopBackgroundThread();
}
}
Thanks

I have same issue with camera2 Api in second time capture image. That problem cause you not close image image.close in image reader listener after get bytes;
I hope this help

getCameraInstance(0) returns null

The basic idea is if no face detected,it should return to Main(reset Activity) but after returning back to main ,getCameraInstance(0) returns null.
If i want to back to Main ,it will call the onPause() and release the cam and after main re-created the backCamera is already relased and it should problem-free create a new instance of back camera.Am i wrong ?
Thanks
ps :i get java.lang.RuntimeException: Fail to connect to camera service and that means i didnt release the camera correctly but i cant find my error.
Here is my code
public class MainActivity extends Activity{
....
private Camera mCameraBack=null;
private CameraPreviewBack mPreviewBack;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
if(mCameraBack==null){
mCameraBack=getCameraInstance(0);
}
try{
mPreviewBack=new CameraPreviewBack(this,mCameraBack );
previewBack=(FrameLayout)findViewById(R.id.camera_preview_back);
previewBack.addView(mPreviewBack);
}catch(Exception ex){
Log.d(TAG,"surfaceCreate");
ex.printStackTrace();
}
public void capture(View view){
mPreviewBack.takePicture();
...
}
#Override
protected void onPause() {
Log.d(TAG, "onPause");
super.onPause();
if (mCameraBack != null) {
mCameraBack.stopPreview();
mCameraBack.release();
mCameraBack = null;
}
if (mPreviewBack != null) {
previewBack.removeView(mPreviewBack);
mPreviewBack = null;
}
}
#Override
public void onResume() {
Log.d(TAG, "onResume");
super.onResume();
if (mCameraBack == null) {
mCameraBack = getCameraInstance(0);
}
if (mPreviewBack == null) {
mPreviewBack=new CameraPreviewBack(this,mCameraBack );
previewBack.addView(mPreviewBack);
}
}
public static Camera getCameraInstance(int cameraId){
Camera c = null;
try {
c = Camera.open(cameraId);
}
catch (Exception e){
}
return c; // returns null if camera is unavailable
}
}
and
public class CameraPreviewBack extends SurfaceView implements SurfaceHolder.Callback {
...
public CameraPreviewBack(Context context,Camera camera) {
super(context);
mCamera=camera;
this.context=context;
mHolder=getHolder();
mHolder.addCallback(this);
}
#Override
public void surfaceCreated(SurfaceHolder holder) {
Log.d(TAG, "Surface created");
try {
if(mCamera!=null){
mCamera.setPreviewDisplay(holder);
mCamera.startPreview();
}
} catch (IOException e) {
Log.d(TAG, "mHolder failure");
e.printStackTrace();
}
}
#Override
public void surfaceChanged(SurfaceHolder holder, int format, int width,
int height) {
Log.d(TAG, "Surface changed");
configureCamRotation();
}
#Override
public void surfaceDestroyed(SurfaceHolder holder) {
}
public void takePicture(){
..
task.execute();
try {
task.get();
} catch (InterruptedException e) {
} catch (ExecutionException e) {
}
task.cancel(isFinished);
}
private PictureCallback getPictureCallback() {
PictureCallback mPicture = new PictureCallback() {
#Override
public void onPictureTaken(byte[] data, Camera camera){
....
if(!detected){
Intent intentMain=new Intent(context, MainActivity.class);
intentMain.addFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP);
context.startActivity(intentMain);
}
...
}
private class TakePictureTask extends AsyncTask<Void, Void, Void> {
#Override
protected Void doInBackground(Void... params) {
mCamera.takePicture(null, null, getPictureCallback());
try {
Thread.sleep(2000); // 3 second preview
} catch (InterruptedException e) {
}
return null;
}
}

newer swallow exceptions
like here:
catch (Exception e){
}
Print this exception Log.d(TAG, "Error when init camera", e); and maybe you will get:
Runtime Exception http://developer.android.com/reference/android/hardware/Camera.html#open(int)

releasing camera onDestroy and changing Intents in onPostExecute method of Asynctask solved my problem