Develop Reference

Java OCR Not reading clearly obvious numbers

picture of processed image + original
I'm working on a project where I'm taking Smash Bros. game output, taking a screenshot, processing it in order to detect the percents which the characters are sitting at.
The program I wrote is detecting the 57 as a 55 and the 11 (which I let settle to it's normal position) as a 51. And while the gameplay is running, the numbers will jump around.
The program I wrote uses Tess4J, and I've configured everything right. I have trained Tesseract with my own custom font that I made using the games percentage numbers. I've tried multiple different fonts as well. What will make it more accurate!?
I've thought about instead of calculating percents, instead just detecting when they're damaged, but I'm also figuring that out.
This is the code I use to process images:
public static Mat blur(Mat input, int numberOfTimes){
Mat sourceImage = new Mat();
Mat destImage = input.clone();
for(int i=0;i<numberOfTimes;i++){
sourceImage = destImage.clone();
Imgproc.blur(sourceImage, destImage, new Size(3.0, 3.0));
}
return destImage;
}
public static BufferedImage purify(BufferedImage image) {
BufferedImage image2 = ImageHelper.convertImageToGrayscale(image);
Mat mat = BufferedImage2Mat(image2, -1);
Mat resizedMat = new Mat();
double width = mat.cols();
double height = mat.rows();
double aspect = width / height;
Size sz = new Size(width * aspect * 1.4, height * aspect * 1.4);
Imgproc.resize(mat, resizedMat, sz);
double thresh = Imgproc.threshold(resizedMat, resizedMat, 23, 255, Imgproc.THRESH_BINARY_INV);
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));
Imgproc.dilate(resizedMat, resizedMat, kernel, new Point(0, 0), 9);
return toBufferedImage(HighGui.toBufferedImage(blur(resizedMat, 0)));
}
public static BufferedImage toBufferedImage(Image img)
{
if (img instanceof BufferedImage)
{
return (BufferedImage) img;
}
BufferedImage bimage = new BufferedImage(img.getWidth(null), img.getHeight(null), BufferedImage.TYPE_INT_ARGB);
Graphics2D bGr = bimage.createGraphics();
bGr.drawImage(img, 0, 0, null);
bGr.dispose();
return bimage;
}
public static Image denoise(BufferedImage img) {
Mat image = BufferedImage2Mat(img, 0);
Mat out = new Mat();
Mat tmp = new Mat();
Mat kernel = new Mat(new Size(3, 3), CvType.CV_8UC1, new Scalar(255));
Imgproc.morphologyEx(image, tmp, Imgproc.MORPH_OPEN, kernel);
Imgproc.morphologyEx(tmp, out, Imgproc.MORPH_CLOSE, kernel);
return HighGui.toBufferedImage(out);
}
public static Mat BufferedImage2Mat(BufferedImage image, int filter) {
try {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
ImageIO.write(image, "jpg", byteArrayOutputStream);
byteArrayOutputStream.flush();
return Imgcodecs.imdecode(new MatOfByte(byteArrayOutputStream.toByteArray()), filter);
} catch (IOException e) {
return null;
}
}
public static Image clean(BufferedImage image) {
Mat og = BufferedImage2Mat(image, Imgcodecs.IMREAD_UNCHANGED);
Mat im = BufferedImage2Mat(image, 0);
Mat bw = new Mat(im.size(), CvType.CV_8U);
Imgproc.threshold(im, bw, 0, 255, Imgproc.THRESH_BINARY_INV | Imgproc.THRESH_OTSU);
Mat dist = new Mat(im.size(), CvType.CV_32F);
Imgproc.distanceTransform(bw, dist, Imgproc.CV_DIST_L2, Imgproc.CV_DIST_MASK_PRECISE);
Mat dibw32f = new Mat(im.size(), CvType.CV_32F);
final double SWTHRESH = 8.0; // stroke width threshold
Imgproc.threshold(dist, dibw32f, SWTHRESH/2.0, 255, Imgproc.THRESH_BINARY);
Mat dibw8u = new Mat(im.size(), CvType.CV_8U);
dibw32f.convertTo(dibw8u, CvType.CV_8U);
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));
Mat cont = new Mat(im.size(), CvType.CV_8U);
Imgproc.morphologyEx(dibw8u, cont, Imgproc.MORPH_OPEN, kernel);
final double HTHRESH = im.rows() * 0.5;
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
List<Point> digits = new ArrayList<Point>();
Mat hierchy = new Mat();
Imgproc.findContours(cont, contours, hierchy, Imgproc.RETR_CCOMP, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0));
List<Mat>cleanedMatList = new ArrayList<Mat>();
int c = 0;
for (int i = 0; i >= hierchy.cols(); i++) {
Rect rect = Imgproc.boundingRect(contours.get(i));
if (rect.height > HTHRESH) {
Mat binary = new Mat();
Imgproc.rectangle(binary, new Point(rect.x, rect.y), new Point(rect.x + rect.width - 1, rect.y + rect.height - 1), new Scalar(0, 0, 255), 3);
cleanedMatList.add(c, binary);
c++;
}
}
List<MatOfInt> digitsHull = new ArrayList<MatOfInt>();
for(int i=0; i < contours.size(); i++){
digitsHull.add(new MatOfInt());
}
for(int i=0; i < contours.size(); i++){
Imgproc.convexHull(contours.get(i), digitsHull.get(i));
}
List<MatOfPoint> digitRegions = new ArrayList<MatOfPoint>();
for (int i = 0; i< digitRegions.size(); i++) {
MatOfPoint dr = digitRegions.get(i);
dr.push_back(digitsHull.get(i));
}
Mat digitsMask = new Mat(og.rows(),og.cols(), CvType.CV_8U);
Imgproc.drawContours(digitsMask, digitRegions, 0, new Scalar(255, 255, 255), -1);
Imgproc.morphologyEx(digitsMask, digitsMask, Imgproc.MORPH_DILATE, kernel);
Mat cleaned = new Mat(og.rows(), og.cols(), CvType.CV_8U);
dibw8u.copyTo(cleaned, digitsMask);
return HighGui.toBufferedImage(dibw8u);
}

How to find area in image using OpenCV findContours?

I'm using OpenCV in Android Studio.
I have an image with a white background and a black area. In the black area, I have some contours. I want to detect the contours which are inside of the black area like this:
But my code is not working; instead it returns the sides of the entire image as contours:
What am I doing wrong?
public Mat[] Smudge_detection(Mat color_mat)
{
Mat lab_mat=new Mat();
Imgproc.cvtColor(color_mat, lab_mat, Imgproc.COLOR_RGB2Lab);
List<Mat> lab_list = new ArrayList(3);
Core.split(lab_mat, lab_list);
Mat eqHist=new Mat();
Imgproc.equalizeHist(lab_list.get(0),eqHist);
lab_list.set(0,eqHist);
Core.merge(lab_list,lab_mat);
Mat rgb_mat=new Mat();
Mat gray_mat=new Mat();
Imgproc.cvtColor(lab_mat,rgb_mat,Imgproc.COLOR_Lab2RGB);
Imgproc.cvtColor(rgb_mat,gray_mat,Imgproc.COLOR_RGB2GRAY);
MatOfInt histSize = new MatOfInt(256);
MatOfInt channels=new MatOfInt(0);
Mat hist_ = new Mat();
MatOfFloat histRange = new MatOfFloat(0, 220);
List<Mat>gray_lst= new ArrayList<Mat>(1);
Core.split(gray_mat,gray_lst);
Imgproc.calcHist(gray_lst,channels,new Mat(),hist_,histSize,histRange,false);
Core.MinMaxLocResult mml= Core.minMaxLoc(hist_);
double tresh=mml.maxLoc.y;
// for remove white area around of black circle
Mat thresh_mat=new Mat();
Imgproc.threshold(gray_mat,thresh_mat,tresh,220,THRESH_BINARY);
Mat notmat=new Mat();
Core.bitwise_not(thresh_mat,notmat);
Mat morph_mat=new Mat();
Imgproc.morphologyEx(notmat,morph_mat,Imgproc.MORPH_CLOSE, Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new Size(2,2)));
List<MatOfPoint> contours=new ArrayList<>();
final Mat hierarchy = new Mat();
Imgproc.findContours(morph_mat,contours,hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
Mat drawmat=color_mat.clone();
Imgproc.drawContours(drawmat,contours,0,new Scalar(255,50,180),15);
Mat[] out=new Mat[2];
out[0]=contours.get(0);
out[1]=drawmat.clone();
return out;
}

My result in python, you may take for reference:
The library is closing, i'll go...
Here is my python code:
#!/usr/bin/python3
# 2017.10.08 21:48:13 CST
import cv2
import numpy as np
img = cv2.imread("t.png")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
retval, threshed = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)
_,contours,h = cv2.findContours(threshed, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
dst1 = img.copy()
dst2 = img.copy()
cv2.drawContours(dst1, contours, -1, (0,255,0), 3)
cnts = []
for cnt in contours:
rect = cv2.boundingRect(cnt)
x,y,w,h = rect
if w < 10 or h < 10 >>w > 100 or h > 100:
continue
cnts.append(cnt)
print(len(cnts))
cv2.drawContours(dst2, cnts, -1, (0,255,0), 3)
res = np.hstack((dst1, dst2))
cv2.imwrite("res.png", res)
cv2.imshow("res", res)
cv2.waitKey()
cv2.destroyAllWindows()

OpenCV change RGB parameters in rectangle

I have 4 points. For example...
A(1;1)
B(2;5)
C(4;4)
D(3;2)
How I can change RGB parameters in this rectangle (for all pixels)?
Like this:
double[] data = mat.get(x, y);
data[0] = data[0]+30;
data[1] = data[1]+20;
data[2] = data[2]+10;
mat.put(x, y, data);

Try something like that for implementing approach described in Dan Mašek comment:
...
Bitmap sourceBitmap = BitmapFactory.decodeResource(getResources(), R.drawable.<your_image>);
Mat sourceMat = new Mat(sourceBitmap.getWidth(), sourceBitmap.getHeight(), CvType.CV_8UC3);
Utils.bitmapToMat(sourceBitmap, sourceMat);
Mat maskMat = new Mat(sourceBitmap.getWidth(), sourceBitmap.getHeight(), CvType.CV_8UC4);
Mat resultMat = new Mat(sourceBitmap.getWidth(), sourceBitmap.getHeight(), CvType.CV_8UC4);
// create color, which added to sourceMat region (+100 - for red channel)
Scalar color = new Scalar(100, 0, 0, 255);
// or you can try Scalar color = new Scalar(10, 20, 30); as in your question
Point[] region = new Point[4];
// your coords multiplied by 50 for visualization convenience
region[0] = new Point(50, 50);
region[1] = new Point(100, 250);
region[2] = new Point(200, 200);
region[3] = new Point(150, 100);
List<MatOfPoint> contours = new ArrayList();
MatOfPoint regionMat = new MatOfPoint(region);
contours.add(regionMat);
// create mask
Imgproc.drawContours(maskMat, contours, 0, color, -1);
// apply mask to source
Core.add(maskMat, sourceMat, resultMat);
// just for visualisation
Bitmap bitmap = Bitmap.createBitmap(sourceMat.cols(), sourceMat.rows(), Bitmap.Config.ARGB_8888);
Utils.matToBitmap(resultMat, bitmap);
<your_ImageView>.setImageBitmap(bitmap);
...
NB! This is just example of masking, not optimized.

Edited answer due to comment below:
This is OpenCV C++ code but you can easily port it to JAVA. Also my code assumes the points always represent a rectangle.
// read image
Mat image=imread("image.jpg",-1);
// region of interest, shape=rectangle
Point p1(50,50), p2(100,80);
Rect roi(p1.x,p1.y,p2.x,p2.y);
// vector hold channels
std::vector<Mat> channels(3);
// split original image to bgr channels
cv::split(image, channels);
// Mat to hold ROI
Mat extractedRoi;
//For channel B
extractedRoi = channels.at(0)(roi);
extractedRoi += 30;
//For channel G
extractedRoi = channels.at(1)(roi);
extractedRoi += 20;
//For channel R
extractedRoi = channels.at(2)(roi);
extractedRoi += 10;
// merge channels back together
cv::merge(channels, image);
Edit2: A faster approach.
Mat image=imread("/home/haseebullah/Pictures/S1.jpg",-1);
Point p1(50,50), p2(100,80);
Rect roi(p1.x,p1.y,p2.x,p2.y);
Mat extractedRoi;
extractedRoi = image(roi);
Scalar constants(30,20,10);
extractedRoi += constants

Align detected face with open cv or dlib c++ Android

I working with open cv and dlib c++ library for android application.
To align detected face I am using code from this post:
public static Bitmap warp(Bitmap originPhoto, List<Point> topCorners) {
int resultWidth = 600;
int resultHeight = 600;
Mat inputMat = new Mat(originPhoto.getHeight(), originPhoto.getHeight(), CvType.CV_8UC1);
Utils.bitmapToMat(originPhoto, inputMat);
Mat outputMat = new Mat(resultWidth, resultHeight, CvType.CV_8UC1);
Mat startM = Converters.vector_Point2f_to_Mat(topCorners);
Point ocvPOut1 = new Point(0, 0);
Point ocvPOut2 = new Point(resultWidth, 0);
Point ocvPOut3 = new Point(0, resultHeight);
Point ocvPOut4 = new Point(resultWidth, resultHeight);
List<Point> dest = new ArrayList<>();
dest.add(ocvPOut1);
dest.add(ocvPOut2);
dest.add(ocvPOut4);
dest.add(ocvPOut3);
Mat endM = Converters.vector_Point2f_to_Mat(dest);
MatOfPoint2f matOfPoint2fStart = new MatOfPoint2f(startM);
MatOfPoint2f matOfPoint2fEnd = new MatOfPoint2f(endM);
Mat perspectiveTransform = Calib3d.findHomography(matOfPoint2fStart, matOfPoint2fEnd);
Imgproc.warpPerspective(inputMat, outputMat, perspectiveTransform, new Size(resultWidth, resultHeight));
Bitmap output = Bitmap.createBitmap(resultWidth, resultHeight, Bitmap.Config.ARGB_8888);
Utils.matToBitmap(outputMat, output);
return output;
}
This code successfully cuts out the face but not align. I know I can rotate bitmap something like this:
Matrix matrix = new Matrix();
matrix.postRotate(angle);
Bitmap rotatedBitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
But it's not cool. How can I align detected face using this libraries?

Opencv javacpp-presets findContours gives error: Unrecognized or unsupported array type in cvGetMat

I am trying to extract text from image(binarize) using opencv javacpp-presets (version 3.0.0-1.0) and using the code-snippit below. The snippit is a translate from this python version of the code.
The input image is from file and is loaded through imread but the code fails at findContours with the following error message:
OpenCV Error: Bad flag (parameter or structure field) (Unrecognized or unsupported array type) in cvGetMat, file src\array.cpp, line 2494
Suggested solutions from here don't work for me. Any help greatly appreciated!
// Load the image
Mat image_orig = imread(inputFile);
if ( image_orig.empty() ) { LOGGER.error("Empty image!");}
this.image = new Mat();
//Add a border to the image for processing sake
copyMakeBorder(image_orig, this.image, 50, 50, 50, 50, BORDER_CONSTANT);
//# Calculate the width and height of the image
this.img_y = this.image.arrayHeight();
this.img_x = this.image.arrayWidth();
if (DEBUG)
LOGGER.info("Image is " + this.img_x + "x" + this.img_x);
//Split out each channel
Mat red = new Mat();
Mat green = new Mat();
Mat blue = new Mat();
MatVector v = new MatVector(blue, green, red);
split(image, v);
//Run canny edge detection on each channel
Mat blue_edges = new Mat();
Canny(blue, blue_edges, 200, 250);
Mat green_edges = new Mat();
Canny(green, green_edges, 200, 250);
Mat red_edges = new Mat();
Canny(red, red_edges, 200, 250);
//Join edges back into image
Mat edges = new Mat();
MatVector vEdges = new MatVector(red_edges, green_edges, blue_edges);
merge(vEdges, edges);
//Find the contours
Mat edgesCopy = new Mat();
edges.copyTo(edgesCopy);
Mat hierarchy = new Mat();
MatVector contours = new MatVector();
findContours(edgesCopy, contours, hierarchy, RETR_TREE, CHAIN_APPROX_NONE);

Gosh, silly me. I should have used the input from the MatVector i.s.o. the empty Mat's for the Canny detection. Then using the grayscale as #user3510227 suggests i get the following working code:
// Load the image
Mat image_orig = imread(inputFile);
if ( image_orig.empty() ) { LOGGER.error("Empty image!");}
this.image = new Mat();
//Add a border to the image for processing sake
copyMakeBorder(image_orig, this.image, 50, 50, 50, 50, BORDER_CONSTANT);
//# Calculate the width and height of the image
this.img_y = this.image.arrayHeight();
this.img_x = this.image.arrayWidth();
if (DEBUG)
LOGGER.info("Image is " + this.img_x + "x" + this.img_x);
//Split out each channel
Mat red = new Mat();
Mat green = new Mat();
Mat blue = new Mat();
MatVector v = new MatVector(red, green, blue);
split(image, v);
//Run canny edge detection on each channel
Mat blue_edges = new Mat();
Canny(v.get(0), blue_edges, 200, 250);
Mat green_edges = new Mat();
Canny(v.get(1), green_edges, 200, 250);
Mat red_edges = new Mat();
Canny(v.get(2), red_edges, 200, 250);
//Join edges back into image
Mat edges = new Mat();
MatVector vEdges = new MatVector(red_edges, green_edges, blue_edges);
merge(vEdges, edges);
LOGGER.info("Type: " + edges.type());
//convert to grayscale
Mat gray = new Mat();
cvtColor(edges, gray, COLOR_BGR2GRAY);
Mat blur = new Mat();
GaussianBlur(gray, blur, new Size(5,5), 0);
Mat edgesCopy = new Mat();
adaptiveThreshold(blur, edgesCopy, 255,1,1,11,2);
//Find the contours
Mat hierarchy = new Mat();
MatVector contours = new MatVector();
findContours(edgesCopy, contours, hierarchy, RETR_TREE, CHAIN_APPROX_NONE);