Develop Reference

Android canny, hough transform. I want to go back to original image

Android, I used canny and hough transform to make line detection on phone screen. However, I can't find a way to get back to original image(I mean HSV to BGR). I want to see keep lines well with BGR image. Thank you!
My code, please let me know which code I have to put in.
public class CameraActivity extends Activity implements CameraBridgeViewBase.CvCameraViewListener2{
private static final String TAG="MainActivity";
private Mat mRgba;
private Mat mGray;
Scalar scalarLow,scalarHigh;///
Mat mat1,mat2;////
Rect rect;////
Rect roi_rect;///
private Mat m_matRoi;////
Point rect1, rect2;///////
public void onCameraViewStarted(int width ,int height){
mRgba=new Mat(height,width, CvType.CV_8UC4);
mGray =new Mat(height,width,CvType.CV_8UC1);
mat1 = new Mat(height,width,CvType.CV_8UC4);
mat2 = new Mat(height,width,CvType.CV_8UC4);
}
public void onCameraViewStopped(){
mRgba.release();
}
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame){
mRgba=inputFrame.rgba();/////mRgba = image, matInput 과 같은 역할
mGray=inputFrame.gray();
double m_dWscale = (double) 1/3;
double m_dHscale = (double) 1/4;
int mRoiWidth = (int)(mRgba.size().width * m_dWscale);
int mRoiHeight = (int)(mRgba.size().height * m_dHscale);
int mRoiX = (int) (mRgba.size().width - mRoiWidth) ;
int mRoiY = (int) (mRgba.size().height - mRoiHeight) ;
//rect = new Rect(mRoiX,mRoiY,mRoiWidth,mRoiHeight);
rect1 =new Point(mRoiX, mRoiY);
rect2 = new Point(mRoiWidth,mRoiHeight);
Imgproc.rectangle(mRgba,rect1,rect2,new Scalar(0, 255, 0, 255),5);
roi_rect = new Rect(mRoiX+4,mRoiY+4,mRoiWidth-8,mRoiHeight-8);
m_matRoi = mRgba.submat(roi_rect);
Mat temp = new Mat();
Imgproc.cvtColor(m_matRoi, temp, Imgproc.COLOR_BGRA2GRAY,0);
Mat temp_rgba = new Mat();
Imgproc.cvtColor(temp, temp_rgba, Imgproc.COLOR_GRAY2BGRA,0);
temp_rgba.copyTo(m_matRoi);
scalarLow=new Scalar(0,0,200);
scalarHigh=new Scalar(180,255,255);
Imgproc.cvtColor(mRgba,mat1,Imgproc.COLOR_BGR2HSV);
Core.inRange(mat1,scalarLow,scalarHigh,mat2);
Core.bitwise_and(mRgba,mRgba,mat1,mat2);
mRgba=mat1;
Imgproc.dilate(mRgba,mRgba,new Mat(),new Point(1,2),2);//
Mat edges=new Mat();
Imgproc.Canny(mRgba,edges,90,150);
Mat lines=new Mat();
Point p1=new Point();
Point p2=new Point();
double a,b;
double x0,y0;
Imgproc.HoughLinesP(edges,lines,1.0,Math.PI/180.0,50,100.0,10.0);//
for(int i=0;i<lines.rows();i++) {
double[] l = lines.get(i, 0);
Imgproc.line(mRgba, new Point(l[0], l[1]), new Point(l[2], l[3]), new Scalar(0, 0, 255.0), 3);
}
return mRgba;
}
}
I divided it to two parts b/c there were not necessary codes between them.
Thank you.
Edit1)
To #Jeru Luke, I changed the code you said. I put many slashes where I changed the code.
public void onCameraViewStarted(int width ,int height){
mRgba=new Mat(height,width, CvType.CV_8UC4);
mGray =new Mat(height,width,CvType.CV_8UC1);
mat1 = new Mat(height,width,CvType.CV_8UC4);
mat2 = new Mat(height,width,CvType.CV_8UC4);//////////
}
public void onCameraViewStopped(){
mRgba.release();
}
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame){
mRgba=inputFrame.rgba();/////mRgba = image, matInput 과 같은 역할
mGray=inputFrame.gray();
double m_dWscale = (double) 1/3;
double m_dHscale = (double) 1/4;
int mRoiWidth = (int)(mRgba.size().width * m_dWscale);
int mRoiHeight = (int)(mRgba.size().height * m_dHscale);
int mRoiX = (int) (mRgba.size().width - mRoiWidth) ;
int mRoiY = (int) (mRgba.size().height - mRoiHeight) ;
//rect = new Rect(mRoiX,mRoiY,mRoiWidth,mRoiHeight);
rect1 =new Point(mRoiX, mRoiY);
rect2 = new Point(mRoiWidth,mRoiHeight);
Imgproc.rectangle(mRgba,rect1,rect2,new Scalar(0, 255, 0, 255),5);
roi_rect = new Rect(mRoiX+4,mRoiY+4,mRoiWidth-8,mRoiHeight-8);
m_matRoi = mRgba.submat(roi_rect);
Mat temp = new Mat();
Imgproc.cvtColor(m_matRoi, temp, Imgproc.COLOR_BGRA2GRAY,0);
Mat temp_rgba = new Mat();
Imgproc.cvtColor(temp, temp_rgba, Imgproc.COLOR_GRAY2BGRA,0);
temp_rgba.copyTo(m_matRoi);
scalarLow=new Scalar(0,0,200);
scalarHigh=new Scalar(180,255,255);
Imgproc.cvtColor(mRgba,mat1,Imgproc.COLOR_BGR2HSV);
Core.inRange(mat1,scalarLow,scalarHigh,mat2);
Core.bitwise_and(mRgba,mRgba,mat1,mat2);
mRgba=mat2;////////////////////////////////////
Imgproc.dilate(mat1,mRgba,new Mat(),new Point(1,2),2);/////////////
Mat edges=new Mat();
Imgproc.Canny(mat1,edges,90,150);//////////////////////////
Mat lines=new Mat();
Point p1=new Point();
Point p2=new Point();
double a,b;
double x0,y0;
Imgproc.HoughLinesP(edges,lines,1.0,Math.PI/180.0,50,100.0,10.0);
for(int i=0;i<lines.rows();i++) {
double[] l = lines.get(i, 0);
Imgproc.line(mRgba, new Point(l[0], l[1]), new Point(l[2], l[3]), new Scalar(0, 0, 255.0), 3);
Scalar(255.0,255.0,255.0),1,Imgproc.LINE_AA,0);
}
return mRgba;
}
}
Edit2) My final objective of this project is draw lines only in ROI(Region of Interest) area. Below code is about ROI. But the problem is it only shows rectangular box and lines are everywhere in my phone screen.
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame){
mRgba=inputFrame.rgba();
mGray=inputFrame.gray();
mat3=inputFrame.rgba();
double m_dWscale = (double) 1/3;
double m_dHscale = (double) 1/4;
int mRoiWidth = (int)(mRgba.size().width * m_dWscale);
int mRoiHeight = (int)(mRgba.size().height * m_dHscale);
int mRoiX = (int) (mRgba.size().width - mRoiWidth) ;
int mRoiY = (int) (mRgba.size().height - mRoiHeight) ;
//rect = new Rect(mRoiX,mRoiY,mRoiWidth,mRoiHeight);
rect1 =new Point(mRoiX, mRoiY);
rect2 = new Point(mRoiWidth,mRoiHeight);
Imgproc.rectangle(mRgba,rect1,rect2,new Scalar(0, 255, 0, 255),5);
roi_rect = new Rect(mRoiX+4,mRoiY+4,mRoiWidth-8,mRoiHeight-8);
m_matRoi = mRgba.submat(roi_rect);
//Mat temp = new Mat();
//Imgproc.cvtColor(m_matRoi, temp, Imgproc.COLOR_BGRA2GRAY,0);
//Mat temp_rgba = new Mat();
//Imgproc.cvtColor(temp, temp_rgba, Imgproc.COLOR_GRAY2BGRA,0);
//temp_rgba.copyTo(m_matRoi);
this part is about ROI.
Edit3) below code is about making mask I guess..
roi_rect = new Rect(mRoiX,mRoiY,mRoiWidth,mRoiHeight);
mat5 = mRgba.submat(roi_rect);
//Mat temp = new Mat();
//Mat temp_rgba = new Mat();
Imgproc.cvtColor(mRgba,mRgba,Imgproc.COLOR_BGR2GRAY);
Imgproc.cvtColor(mat5, mat5, Imgproc.COLOR_GRAY2BGR,0); //
//Imgproc.cvtColor(temp, temp_rgba, Imgproc.COLOR_GRAY2BGRA,0);
mat5.copyTo(mRgba);
roi_rect 's parameters are all (int) and mat5 is
mat5 = new Mat(height,width,CvType.CV_8UC4);
I made this code firstly to make all region dark. And using roi_rect only that part, which means mat5 is converted GRAY2BGR and then use mat5.copyTo(mRgba) to paste mat5 on mRgba. But it shuts down when I turn on the app.

How to crop the words in order as they appear in the text image

I have a text image and I have found the contours for every word. Now I want to crop all the contours in the order they appear in the text. I have tried to sort the contours from left to right but did not succeed. How do I do this?
What I am getting in result is the word images in random order but I want them in the same order as in the image.
Mat finalMat = mats[1];
Mat mat2 = mats[0];
Mat gray = new Mat(finalMat.rows(),finalMat.height(),CvType.CV_8UC1);
Imgproc.cvtColor(finalMat,gray,Imgproc.COLOR_BGR2GRAY);
Mat mat = new Mat();
Imgproc.dilate(mat2, mat, Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(12, 5)));
Mat hierarchy = new Mat();
List<MatOfPoint> contours = new ArrayList<>();
Imgproc.findContours(mat,contours,hierarchy,Imgproc.RETR_EXTERNAL,
Imgproc.CHAIN_APPROX_SIMPLE);
Collections.sort(contours, new Comparator<MatOfPoint>() {
#Override
public int compare(MatOfPoint o1, MatOfPoint o2) {
Rect rect1 = Imgproc.boundingRect(o1);
Rect rect2 = Imgproc.boundingRect(o2);
int result = 0;
double total = rect1.tl().y/rect2.tl().y;
if (total>=0.9 && total<=1.4 ){
result = Double.compare(rect1.tl().x, rect2.tl().x);
}
return result;
}
});
for (int contourIdx = 0; contourIdx < contours.size(); contourIdx++) {
if (Imgproc.contourArea(contours.get(contourIdx)) > 50 ){
Rect rect = Imgproc.boundingRect(contours.get(contourIdx));
if (rect.height > 28){
Mat crop;
if((rect.y + rect.height+15)<gray.rows() && rect.y-3>1)
{
Imgproc.rectangle(finalMat, new Point(rect.x,rect.y-3), new Point(rect.x+rect.width,rect.y-3+rect.height+15),new Scalar(255,0,0));
crop = gray.submat(rect.y-3, rect.y-3 + rect.height+15, rect.x, rect.x + rect.width);
}
else {
Imgproc.rectangle(finalMat, new Point(rect.x,rect.y), new Point(rect.x+rect.width,rect.y+rect.height),new Scalar(255,0,0));
crop = gray.submat(rect.y, rect.y + rect.height, rect.x, rect.x + rect.width);
}
Bitmap bmp = Bitmap.createBitmap(crop.width(), crop.height(), Bitmap.Config.ARGB_8888);
Utils.matToBitmap(crop, bmp);
MediaStore.Images.Media.insertImage(getContentResolver(), bmp, contourIdx+".jpg" , null);
}
}
}

Detect Biggest Rectangle in the Image using Java Opencv [SOLVED]

How can I detect the four corner points of the biggest square (at center of the image) using opencv in java
I have solved this using findContours.
Original Image
Output Image
Please find the code below. I don't now how to detect the end points of center square. I tried to detect lines using HoughLinesP but it is returning only 1 verticle line instead of giving all the 4 lines.
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
String path = "/Users/saurabhsaluja/Desktop/cimg.jpg";
Mat img = Imgcodecs.imread(path);
Mat destination = new Mat(img.rows(),img.cols(),img.type());
Core.addWeighted(img, 1.3, destination, -0.7, 0, destination);
Mat cannyOutput = new Mat();
int threshold = 15;
Mat srcGray = new Mat();
Imgproc.cvtColor(destination, srcGray, Imgproc.COLOR_BGR2GRAY);
Imgproc.Canny(srcGray, cannyOutput, threshold, threshold* 4);
Mat element = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(10,10));
Mat element2 = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(10,10));
Imgproc.dilate(cannyOutput, cannyOutput, element);
Imgproc.dilate(cannyOutput, cannyOutput, element2);
element = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(9,9));
element2 = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(9,9));
Imgproc.erode(cannyOutput, cannyOutput, element);
Imgproc.erode(cannyOutput, cannyOutput, element2);
Imgcodecs.imwrite("/Users/saurabhsaluja/Desktop/cannyOutput.jpg", cannyOutput); //THE IMAGE YOU ARE LOOKING AT
Mat lines = new Mat();
Imgproc.HoughLinesP(cannyOutput, lines, 1, Math.PI / 180, 50, 20, 20);
for(int i = 0; i < lines.cols(); i++) {
double[] val = lines.get(0, i);
Imgproc.line(img, new Point(val[0], val[1]), new Point(val[2], val[3]), new Scalar(0, 0, 255), 2);
}
Imgcodecs.imwrite("/Users/saurabhsaluja/Desktop/finalimg.jpg", img);
Solution:
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
Imgproc.findContours(cannyOutput, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
double inf = 0;
Rect max_rect = null;
for(int i=0; i< contours.size();i++){
Rect rect = Imgproc.boundingRect(contours.get(i));
double area = rect.area();
if(inf < area) {
max_rect = rect;
inf = area;
//Imgcodecs.imwrite("/Users/saurabhsaluja/Desktop/input"+i+".jpg", img);
}
if(area > 50000) {
System.out.println(area);
Imgproc.rectangle(img, new Point(rect.x,rect.y), new Point(rect.x+rect.width,rect.y+rect.height),new Scalar(0,0,0),5);
}
}
Now just get the biggest by looking area of each counter.
Thanks.
Solution Image:

List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
Imgproc.findContours(cannyOutput, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
double inf = 0;
Rect max_rect = null;
for(int i=0; i< contours.size();i++){
Rect rect = Imgproc.boundingRect(contours.get(i));
double area = rect.area();
if(inf < area) {
max_rect = rect;
inf = area;
//Imgcodecs.imwrite("/Users/saurabhsaluja/Desktop/input"+i+".jpg", img);
}
if(area > 50000) {
System.out.println(area);
Imgproc.rectangle(img, new Point(rect.x,rect.y), new Point(rect.x+rect.width,rect.y+rect.height),new Scalar(0,0,0),5);
}
}
Output:

OpenCV detect rectangle with the largest width

I'm trying to get this rectangle from that image:
Found this solution using OpenCV:
private Bitmap findRectangle(Bitmap src) throws Exception {
Mat imageMat = new Mat();
Utils.bitmapToMat(src, imageMat);
Mat imgSource=imageMat.clone();
Imgproc.cvtColor(imgSource, imageMat, Imgproc.COLOR_BGR2GRAY);
//find the contours
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
Imgproc.findContours(imageMat, contours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_NONE);
Imgproc.Canny(imageMat,imageMat,0,255);
Bitmap canny=Bitmap.createBitmap(imageMat.cols(),imageMat.rows(),Bitmap.Config.ARGB_8888);
Utils.matToBitmap(imageMat,canny);
Imgproc.GaussianBlur(imageMat, imageMat, new org.opencv.core.Size(1, 1), 2, 2);
Bitmap blur=Bitmap.createBitmap(imageMat.cols(),imageMat.rows(),Bitmap.Config.ARGB_8888);
Utils.matToBitmap(imageMat,blur);
MatOfPoint temp_contour = contours.get(0); //the largest is at the index 0 for starting point
for (int idx = 0; idx < contours.size(); idx++) {
temp_contour = contours.get(idx);
//check if this contour is a square
MatOfPoint2f new_mat = new MatOfPoint2f( temp_contour.toArray() );
int contourSize = (int)temp_contour.total();
MatOfPoint2f approxCurve_temp = new MatOfPoint2f();
Imgproc.approxPolyDP(new_mat, approxCurve_temp, contourSize*0.05, true);
if (approxCurve_temp.total() == 4) {
MatOfPoint points = new MatOfPoint( approxCurve_temp.toArray() );
Rect rect = Imgproc.boundingRect(points);
Imgproc.rectangle(imgSource, new Point(rect.x,rect.y), new Point(rect.x+rect.width,rect.y+rect.height), new Scalar(255, 0, 0, 255), 3);
}
}
Bitmap analyzed=Bitmap.createBitmap(imgSource.cols(),imgSource.rows(),Bitmap.Config.ARGB_8888);
Utils.matToBitmap(imgSource,analyzed);
return analyzed;
}
The best i got was this:
The problem is that the rectangle isn't perfect, maybe find the white numbers inside of that can be a best option, but i don't know too much of OpenCV.
Original image:

This is a very simple C++ implementation which tries to search for the text box. The accuracy of the detection depends on three parameters:
The thresh value provided to cv::threshold function to convert gray image to binary.
The height/width ratio, since the height of the text box is relatively smaller than the width, and the area of the text box.
Mat img = imread("image.jpg",-1), gray, binary;
/*pre-processing steps*/
uchar thresh = 80;
cvtColor(img, gray, cv::COLOR_BGR2GRAY);
GaussianBlur(gray, gray, Size(7,7), 0);
// change the thresh value to fine tune this program for your images
threshold(gray, binary, thresh, 255, cv::THRESH_BINARY_INV);
/*contour searching*/
std::vector<std::vector<Point>> contours;
std::vector<Vec4i> hierarchy;
findContours(binary, contours, hierarchy, cv::RETR_LIST, cv::CHAIN_APPROX_SIMPLE);
/*Filtering contours based on height/width ratio and bounding box area*/
std::vector<Rect> boxes;
double box_ratio = 0.3;
int box_area = 20000;
for(auto& cnt : contours)
{
auto box = minAreaRect(cnt).boundingRect();
// we are searching for a rectangle which a has relatively large area,
// and the height is smaller than the width, so the
// height/width ratio should be small. Change the these two values for fine tuning
if((min(box.width,box.height)/double(max(box.width,box.height)) < box_ratio) && box.area() > box_area )
{
boxes.push_back(box);
}
}
Mat txt_box = img(boxes.at(0));

Here is the almost same solution on java:
private Bitmap findRoi(Bitmap sourceBitmap) {
Mat sourceMat = new Mat(sourceBitmap.getWidth(), sourceBitmap.getHeight(), CV_8UC3);
Utils.bitmapToMat(sourceBitmap, sourceMat);
Mat grayMat = new Mat(sourceBitmap.getWidth(), sourceBitmap.getHeight(), CV_8UC3);
Imgproc.cvtColor(sourceMat, grayMat, Imgproc.COLOR_BGR2GRAY);
Imgproc.threshold(grayMat, grayMat, 125, 200, Imgproc.THRESH_BINARY);
// find contours
List<MatOfPoint> whiteContours = new ArrayList<>();
Rect largestRect = null;
Imgproc.findContours(grayMat, whiteContours, new Mat(), Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
// find appropriate bounding rectangles
for (MatOfPoint contour : whiteContours) {
RotatedRect boundingRect = Imgproc.minAreaRect(new MatOfPoint2f(contour.toArray()));
double rectangleArea = boundingRect.size.area();
// test min ROI area in pixels
if (rectangleArea > 10000) {
Point rotated_rect_points[] = new Point[4];
boundingRect.points(rotated_rect_points);
Rect rect = Imgproc.boundingRect(new MatOfPoint(rotated_rect_points));
// test horizontal ROI orientation and aspect ratio
if (rect.width > 3 * rect.height) {
if (largestRect == null) {
largestRect = rect;
} else {
if (rect.width > largestRect.width) {
largestRect = rect;
}
}
}
}
}
Mat roiMat = new Mat(sourceMat, largestRect);
Bitmap bitmap = Bitmap.createBitmap(roiMat.cols(), roiMat.rows(), Bitmap.Config.ARGB_8888);
Utils.matToBitmap(roiMat, bitmap);
return bitmap;
}
Also, you can use additional information: red number places on the right.

Image Detection Open CV

So I'm trying to draw a rect around an image. I'm finding a sub image from a larger image using Open CV. How I calculated it was to convert both the template image (target) and the target image (scene) to HSV and get the back projection of the object and compare it with the scene image which is saturated. Works a bit. (Would be glad for any improvements). Basically i want to draw a rect around the image and extract the found rect from the scene unto a Mat. I've tried a doing it a couple of ways but doesn't seem to be working. Here's my code. My question is how do I get the sub image from the target image?
public List<DMatch> subListGoodMatches(List<DMatch> good_matches) {
Collections.sort(good_matches, (DMatch o1, DMatch o2) -> {
if (o1.distance < o2.distance) {
return -1;
}
if (o1.distance > o2.distance) {
return 1;
}
return 0;
});
if (good_matches.size() > 10) {
good_matches = good_matches.subList(0, 10);
}
return good_matches;
}
public List<Mat> calculateHistograms(Mat image) {
Imgproc.cvtColor(image, image, Imgproc.COLOR_BGR2HSV);
List<Mat> hsv_planes = new ArrayList<Mat>();
Core.split(image, hsv_planes);
MatOfInt histSize = new MatOfInt(256);
final MatOfFloat histRange = new MatOfFloat(0f, 256f);
boolean accumulate = true;
Mat h_hist = new Mat();
Mat s_hist = new Mat();
Mat v_hist = new Mat();
//Break channels
List<Mat> h_plane = new ArrayList<Mat>();
List<Mat> s_plane = new ArrayList<Mat>();
List<Mat> v_plane = new ArrayList<Mat>();
h_plane.add(hsv_planes.get(0));
s_plane.add(hsv_planes.get(1));
v_plane.add(hsv_planes.get(2));
Imgproc.calcHist(h_plane, new MatOfInt(0), new Mat(), h_hist, histSize, histRange, accumulate);
Imgproc.calcHist(s_plane, new MatOfInt(0), new Mat(), s_hist, histSize, histRange, accumulate);
Imgproc.calcHist(v_plane, new MatOfInt(0), new Mat(), v_hist, histSize, histRange, accumulate);
//Draw combined histograms
int hist_w = 512;
int hist_h = 600;
long bin_w = Math.round((double) hist_w / 256);
Mat histImage = new Mat(hist_h, hist_w, CvType.CV_8UC3, new Scalar(0, 0, 0));
Core.normalize(h_hist, h_hist, 3, histImage.rows(), Core.NORM_MINMAX, -1, new Mat());
Core.normalize(s_hist, s_hist, 3, histImage.rows(), Core.NORM_MINMAX, -1, new Mat());
Core.normalize(v_hist, v_hist, 3, histImage.rows(), Core.NORM_MINMAX, -1, new Mat());
for (int i = 1; i < 256; i++) {
Point p1 = new Point(bin_w * (i - 1), hist_h - Math.round(h_hist.get(i - 1, 0)[0]));
Point p2 = new Point(bin_w * (i), hist_h - Math.round(h_hist.get(i, 0)[0]));
Core.line(histImage, p1, p2, RED, 2, 8, 0);
Point p3 = new Point(bin_w * (i - 1), hist_h - Math.round(s_hist.get(i - 1, 0)[0]));
Point p4 = new Point(bin_w * (i), hist_h - Math.round(s_hist.get(i, 0)[0]));
Core.line(histImage, p3, p4, GREEN, 2, 8, 0);
Point p5 = new Point(bin_w * (i - 1), hist_h - Math.round(v_hist.get(i - 1, 0)[0]));
Point p6 = new Point(bin_w * (i), hist_h - Math.round(v_hist.get(i, 0)[0]));
Core.line(histImage, p5, p6, BLUE, 2, 8, 0);
}
Highgui.imwrite("img-histogram.jpg", histImage);
System.out.println("Hist size is: " + hsv_planes.size());
List<Mat> histograms = new ArrayList<Mat>();
histograms.add(h_hist);
histograms.add(s_hist);
histograms.add(v_hist);
return histograms;
}
public Mat identifyLowSat(Mat image) {
Mat hsvTargetImage = new Mat();
Imgproc.cvtColor(image, hsvTargetImage, Imgproc.COLOR_BGR2HSV);
List<Mat> hsv_planes = new ArrayList<Mat>();
Core.split(hsvTargetImage, hsv_planes);
//Get saturation channel
Mat s_hist = hsv_planes.get(1);
Imgproc.threshold(s_hist, s_hist, 65, 255, Imgproc.THRESH_BINARY);
Highgui.imwrite("img-saturation.png", s_hist);
return s_hist;
}
public Mat getBackProjOfHueTemplate(Mat image, Mat hue_histogram) {
Mat hsvTargetImage = new Mat();
Imgproc.cvtColor(image, hsvTargetImage, Imgproc.COLOR_BGR2HSV);
List<Mat> hsv_planes = new ArrayList<Mat>();
Core.split(hsvTargetImage, hsv_planes);
Mat backProj = new Mat();
final MatOfFloat range = new MatOfFloat(0f, 256f);
Imgproc.calcBackProject(hsv_planes, new MatOfInt(0), hue_histogram, backProj, range, 4);
Highgui.imwrite("img-backProj.png", backProj);
return backProj;
}
public Mat meanShift(Mat image) {
Mat map = new Mat();
Rect rect = new Rect();
TermCriteria term = new TermCriteria();
term.maxCount = 100;
term.type = TermCriteria.EPS;
term.epsilon = 0.1;
Imgproc.pyrMeanShiftFiltering(image, map, 0.5, 0.5, 5, term);
Highgui.imwrite("img-meanshift.png", map);
return map;
}
public MatOfDMatch filterMatches(Mat img1, Mat img2) {
FeatureDetector detector = FeatureDetector.create(FeatureDetector.SIFT);
DescriptorExtractor descriptor = DescriptorExtractor.create(DescriptorExtractor.BRISK);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
// First photo
//Imgproc.cvtColor(img1, img1, Imgproc.COLOR_RGB2GRAY);
Mat descriptors1 = new Mat();
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
detector.detect(img1, keypoints1);
descriptor.compute(img1, keypoints1, descriptors1);
// Second photo
//Imgproc.cvtColor(img2, img2, Imgproc.COLOR_RGB2GRAY);
Mat descriptors2 = new Mat();
MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
detector.detect(img2, keypoints2);
descriptor.compute(img2, keypoints2, descriptors2);
// Matching
MatOfDMatch matches = new MatOfDMatch();
MatOfDMatch filteredMatches = new MatOfDMatch();
matcher.match(descriptors1, descriptors2, matches);
List<DMatch> matchesList = matches.toList();
Double max_dist = Double.MIN_VALUE;
Double min_dist = Double.POSITIVE_INFINITY;
for (DMatch matchesList1 : matchesList) {
Double dist = (double) matchesList1.distance;
if (dist < min_dist) {
min_dist = dist;
}
if (dist > max_dist) {
max_dist = dist;
}
}
LinkedList<DMatch> good_matches = new LinkedList<DMatch>();
for (DMatch matchesList1 : matchesList) {
if (matchesList1.distance <= (1.5 * min_dist)) {
good_matches.addLast(matchesList1);
}
}
MatOfDMatch goodMatches = new MatOfDMatch();
//goodMatches.fromList(good_matches);
List<DMatch> newGood_Matches = subListGoodMatches(good_matches);
goodMatches.fromList(newGood_Matches);
//put keypoints mats into lists
List<KeyPoint> keypoints1_List = keypoints1.toList();
List<KeyPoint> keypoints2_List = keypoints2.toList();
//put keypoints into point2f mats so calib3d can use them to find homography
LinkedList<Point> objList = new LinkedList<Point>();
LinkedList<Point> sceneList = new LinkedList<Point>();
for (int i = 0; i < newGood_Matches.size(); i++) {
objList.addLast(keypoints2_List.get(newGood_Matches.get(i).trainIdx).pt);
sceneList.addLast(keypoints1_List.get(newGood_Matches.get(i).queryIdx).pt);
}
MatOfPoint2f obj = new MatOfPoint2f();
MatOfPoint2f scene = new MatOfPoint2f();
obj.fromList(objList);
scene.fromList(sceneList);
System.out.println(matches.size() + " " + goodMatches.size());
//output image
Mat outputImg = new Mat();
MatOfByte drawnMatches = new MatOfByte();
Features2d.drawMatches(img1, keypoints1, img2, keypoints2, goodMatches, outputImg, Scalar.all(-1), Scalar.all(-1), drawnMatches, Features2d.NOT_DRAW_SINGLE_POINTS);
Highgui.imwrite("img-matches.png", outputImg);
drawWithRect(img1, img2, obj, scene, outputImg);
keypointers1 = keypoints1;
keypointers2 = keypoints2;
return goodMatches;
}
public MatOfDMatch filterMatchesByHomography(MatOfDMatch matches) {
MatOfKeyPoint keypoints1 = keypointers1;
MatOfKeyPoint keypoints2 = keypointers2;
List<Point> lp1 = new ArrayList<Point>();
List<Point> lp2 = new ArrayList<Point>();
KeyPoint[] k1 = keypoints1.toArray();
KeyPoint[] k2 = keypoints2.toArray();
List<DMatch> matches_original = matches.toList();
if (matches_original.size() < 4) {
MatOfDMatch mat = new MatOfDMatch();
return mat;
}
// Add matches keypoints to new list to apply homography
for (DMatch match : matches_original) {
Point kk1 = k1[match.queryIdx].pt;
Point kk2 = k2[match.trainIdx].pt;
lp1.add(kk1);
lp2.add(kk2);
}
//srcPoints = new MatOfPoint2f(lp1.toArray(new Point[0]));
//dstPoints = new MatOfPoint2f(lp2.toArray(new Point[0]));
Mat mask = new Mat();
//Mat homography = Calib3d.findHomography(srcPoints, dstPoints, Calib3d.LMEDS, 0.2, mask);
List<DMatch> matches_homo = new ArrayList<DMatch>();
int size = (int) mask.size().height;
for (int i = 0; i < size; i++) {
if (mask.get(i, 0)[0] == 1) {
DMatch d = matches_original.get(i);
matches_homo.add(d);
}
}
MatOfDMatch mat = new MatOfDMatch();
mat.fromList(matches_homo);
//Highgui.imwrite("img-matchesWithRect.png", mat);
return mat;
}
public void drawMatches(Mat img1, Mat img2, MatOfDMatch matches, boolean imageOnly) {
Mat out = new Mat();
MatOfKeyPoint key2 = keypointers2;
MatOfKeyPoint key1 = keypointers1;
//Imgproc.cvtColor(img1, im1, Imgproc.COLOR_BGR2RGB);
//Imgproc.cvtColor(img2, im2, Imgproc.COLOR_BGR2RGB);
if (imageOnly) {
MatOfDMatch emptyMatch = new MatOfDMatch();
MatOfKeyPoint emptyKey1 = new MatOfKeyPoint();
MatOfKeyPoint emptyKey2 = new MatOfKeyPoint();
Features2d.drawMatches(img1, emptyKey1, img2, emptyKey2, emptyMatch, out);
} else {
Features2d.drawMatches(img1, key1, img2, key2, matches, out);
}
//Imgproc.cvtColor(out, out, Imgproc.COLOR_BGR2RGB);
Core.putText(out, "FRAME", new Point(img1.width() / 2, 30), Core.FONT_HERSHEY_PLAIN, 2, new Scalar(0, 255, 255), 3);
Core.putText(out, "MATCHED", new Point(img1.width() + img2.width() / 2, 30), Core.FONT_HERSHEY_PLAIN, 2, new Scalar(255, 0, 0), 3);
Highgui.imwrite("img-drawnMatches.png", out);
}
public void drawWithRect(Mat img1, Mat img2, MatOfPoint2f obj, MatOfPoint2f scene, Mat outputImg){
//run homography on object and scene points
Mat H = Calib3d.findHomography(obj, scene, Calib3d.RANSAC, 5);
Mat tmp_corners = new Mat(4, 1, CvType.CV_32FC2);
Mat scene_corners = new Mat(4, 1, CvType.CV_32FC2);
//get corners from object
tmp_corners.put(0, 0, new double[]{0, 0});
tmp_corners.put(1, 0, new double[]{img2.cols(), 0});
tmp_corners.put(2, 0, new double[]{img2.cols(), img2.rows()});
tmp_corners.put(3, 0, new double[]{0, img2.rows()});
Core.perspectiveTransform(tmp_corners, scene_corners, H);
Core.line(outputImg, new Point(scene_corners.get(0, 0)), new Point(scene_corners.get(1, 0)), new Scalar(0, 255, 0), 4);
Core.line(outputImg, new Point(scene_corners.get(1, 0)), new Point(scene_corners.get(2, 0)), new Scalar(0, 255, 0), 4);
Core.line(outputImg, new Point(scene_corners.get(2, 0)), new Point(scene_corners.get(3, 0)), new Scalar(0, 255, 0), 4);
Core.line(outputImg, new Point(scene_corners.get(3, 0)), new Point(scene_corners.get(0, 0)), new Scalar(0, 255, 0), 4);
Highgui.imwrite("img-matchesWithRect.png", outputImg);
}
Main method:
public static void main(String args[]) {
System.load(new File("/usr/local/Cellar/opencv/2.4.9/share/OpenCV/java/libopencv_java249.dylib").getAbsolutePath());
Mat img1 = Highgui.imread(scenesD);
Mat img2 = Highgui.imread(objectD);
MeanShift Tester = new MeanShift();
List<Mat> histogramsList;
Mat hue_histogram;
Mat saturationChannel;
Mat getBackProjOfHueTemp;
//Calulate Histogram of Object
histogramsList = Tester.calculateHistograms(img2);
//Get saturation channel of scene
saturationChannel = Tester.identifyLowSat(img1);
//Get hue of calculated object histogram
hue_histogram = histogramsList.get(0);
//Get back projection of object from calculated hue histogram template
getBackProjOfHueTemp = Tester.getBackProjOfHueTemplate(img2, hue_histogram);
//Filtering matches
MatOfDMatch matches = Tester.filterMatches(saturationChannel, getBackProjOfHueTemp);
MatOfDMatch homo_matches = Tester.filterMatchesByHomography(matches);
//Draw img unto screen;
Tester.drawMatches(saturationChannel, getBackProjOfHueTemp, homo_matches, false);
}
And so far here's the final image i get (img-matches.png)
And trying to get the rect from the image gives me this (img-matchesWithRect.png)

Ok, so you have the Points in your original image that you found have matches in your target image. These points all have x and y coordinates - to find the "subimage" (or a rectangle in the image), all you need to do is take the minimum of all the x's to get your top left x coordinate, the minimum of the ys to get your top left y coordinate, the max of the xs to get your bottom right x coordinate, and the max of the ys to get your bottom right y coordinate.
Then with these you can create a Rect (http://docs.opencv.org/java/org/opencv/core/Rect.html) and use it to access the subimage: image(Rect) (assuming you have a previous variable image that is a Mat):
Rect sub_rect = new Rect(min_x, min_y, max_x - min_x, max_y - min_y);
Mat sub_region = image(sub_rect);
Now sub_region will have your subregion in it.