UPDATE
You can find all the images I have for testing on my GitHub here:
GitHub repository with sources
There are also 2 videos, where the detection should work on as well
ORIGINAL QUESTION
I tried to use OpenCV 4.x.x to find the edges of a blackboard (image following), but somehow I cannot succeed. My code at the moment looks like this: (Android with OpenCV and live camera feed), where imgMat is a Mat from the camera feed:
Mat gray = new Mat();
Imgproc.cvtColor(imgMat, gray, Imgproc.COLOR_RGB2BGR);
Mat blurred = new Mat();
Imgproc.blur(gray, blurred, new org.opencv.core.Size(3, 3));
Mat canny = new Mat();
Imgproc.Canny(blurred, canny, 80, 230);
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new org.opencv.core.Size(2, 2));
Mat dilated = new Mat();
Imgproc.morphologyEx(canny, dilated, Imgproc.MORPH_DILATE, kernel, new Point(0, 0), 10);
Mat rectImage = new Mat();
Imgproc.morphologyEx(dilated, rectImage, Imgproc.MORPH_CLOSE, kernel, new Point(0, 0), 5);
Mat endproduct = new Mat();
Imgproc.Canny(rectImage, endproduct, 120, 230);
List<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
Imgproc.findContours(endproduct, contours, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
double maxArea = 0;
boolean hasContour = false;
MatOfPoint2f biggestContour = new MatOfPoint2f();
Iterator<MatOfPoint> each = contours.iterator();
while (each.hasNext()) {
MatOfPoint wrapper = each.next();
double area = Imgproc.contourArea(wrapper);
if (area > maxArea) {
maxArea = area;
biggestContour = new MatOfPoint2f(wrapper.toArray());
hasContour = true;
}
}
if (hasContour) {
Mat output = imgMat.clone();
MatOfPoint2f approx = new MatOfPoint2f();
MatOfPoint poly = new MatOfPoint();
Imgproc.approxPolyDP(biggestContour, approx, Imgproc.arcLength(biggestContour, true) * .02, true);
approx.convertTo(poly, CvType.CV_32S);
Rect rect = Imgproc.boundingRect(poly);
}
Somehow I am not able to get it working, although the same code(written in python) worked on my computer with a video. I take the output from the rectangle and display it on my mobile screen, where it flickers around a lot and does not work properly.
These are my images I tried the python program on, and they worked:
What am I doing wrong? I am not able to constantly detect the edges of the blackboard.
Additional information about the blackboard:
always rectangular
may have different lighting
the text should be ignored, only the main board should be detected
the outer blackboard should be ignored as well
only the contour for the main board should be shown/returned
Thanks for any advice or code!
I used HSV because that's the easiest way to detect specific colors. I used an abundancy test to automatically select the color threshold (so this will work for green or blue boards). However, this test will fail on white or black boards since white and black count as all colors according to hue. Instead, in HSV, white and black are easiest to detect as very low saturation (white) or as very low value (black).
I did a 3-way check for each and selected the mask that had the most pixels in it (I assume that the boards are the majority of the image). I'm not sure how this will work on other images since we only have one here, so this may or may not work for other boards.
I used approxPolyDP to cut down on the number of points in the contour until I had 4 points and used that to draw the shape.
import cv2
import numpy as np
# get unique colors (to speed up search) and return the most abundant mask
def getAbundantColor(channel, margin):
# get uniques
unique_colors, counts = np.unique(channel, return_counts=True);
# check for the most abundant color
most = None;
biggest_count = -1;
for col in unique_colors:
# count number of white pixels
mask = cv2.inRange(channel, int(col - margin), int(col + margin));
count = np.count_nonzero(mask);
# if bigger, set new "most"
if count > biggest_count:
biggest_count = count;
most = mask;
return most, biggest_count;
# load image
img = cv2.imread("blackboard.jpg");
# it's huge, scale down so that we can see the whole thing
h, w = img.shape[:2];
scale = 0.25;
h = int(scale*h);
w = int(scale*w);
img = cv2.resize(img, (w,h));
# hsv
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV);
h,s,v = cv2.split(hsv);
# median blur to get rid of most of the text
h = cv2.medianBlur(h, 5);
s = cv2.medianBlur(s, 5);
v = cv2.medianBlur(v, 5);
# get most abundant color
color_margin = 30;
hmask, hcount = getAbundantColor(h, color_margin);
# detect white and black separately
light_margin = 30;
# white
wmask = cv2.inRange(s, 0, light_margin);
wcount = np.count_nonzero(wmask);
# black
bmask = cv2.inRange(v, 0, light_margin);
bcount = np.count_nonzero(bmask);
# check which is biggest
sorter = [[hcount, hmask], [wcount, wmask], [bcount, bmask]];
sorter.sort();
mask = sorter[-1][1];
# dilate and erode to close holes
kernel = np.ones((3,3), np.uint8);
mask = cv2.dilate(mask, kernel, iterations = 2);
mask = cv2.erode(mask, kernel, iterations = 4);
mask = cv2.dilate(mask, kernel, iterations = 2);
# get contours # OpenCV 3.4, in OpenCV 2* or 4* it returns (contours, _)
_, contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE);
# for each contour, approximate a simpler shape until we have 4 points
simplified = [];
for con in contours:
# go until we have 4 points
num_points = 999999;
step_size = 0.01;
percent = step_size;
while num_points >= 4:
# get number of points
epsilon = percent * cv2.arcLength(con, True);
approx = cv2.approxPolyDP(con, epsilon, True);
num_points = len(approx);
# increment
percent += step_size;
# step back and get the points
# there could be more than 4 points if our step size misses it
percent -= step_size * 2;
epsilon = percent * cv2.arcLength(con, True);
approx = cv2.approxPolyDP(con, epsilon, True);
simplified.append(approx);
cv2.drawContours(img, simplified, -1, (0,0,200), 2);
# print out the number of points
for points in simplified:
print("Num Points: " + str(len(points)));
# show image
cv2.imshow("Image", img);
cv2.imshow("Hue", h);
cv2.imshow("Mask", mask);
cv2.waitKey(0);
Edit: In order to accommodate the uncertainty in the board's color and appearance I run the assumption that the board itself will be the majority of the picture. The lines involving the sorter are looking for the most abundant color in the image. If the white wall behind the board takes up more space in the image then that'll be the color that gets selected for the mask.
There are other ways to try and select just the board, but it's really difficult to come up with a catch-all solution. The rest of the code should do its job the same if you can come up with some way of masking the board. If you're willing to budge on the unknown color assumption and provide the original pictures of the failing cases then I can probably come up with an appropriate mask.
Related
I am developing an image segmentation application that will use watersheds. For that, I found a code that I will need to use in python. However, I'm having a hard time converting to kotlin, as the Mat Mat () doesn't have the zero_likes function, just the zero function. I am using opencv 3.31. How do I do this check in kotlin:
marked[marked == 1] = 0
marked[marked > 1] = 255
Code python:
import cv2
import numpy as np
import matplotlib.pyplot as plt
# Load the image
img = cv2.imread("/path/to/image.png", 3)
# Create a blank image of zeros (same dimension as img)
# It should be grayscale (1 color channel)
marker = np.zeros_like(img[:,:,0]).astype(np.int32)
# This step is manual. The goal is to find the points
# which create the result we want. I suggest using a
# tool to get the pixel coordinates.
# Dictate the background and set the markers to 1
marker[204][95] = 1
marker[240][137] = 1
marker[245][444] = 1
marker[260][427] = 1
marker[257][378] = 1
marker[217][466] = 1
# Dictate the area of interest
# I used different values for each part of the car (for visibility)
marker[235][370] = 255 # car body
marker[135][294] = 64 # rooftop
marker[190][454] = 64 # rear light
marker[167][458] = 64 # rear wing
marker[205][103] = 128 # front bumper
# rear bumper
marker[225][456] = 128
marker[224][461] = 128
marker[216][461] = 128
# front wheel
marker[225][189] = 192
marker[240][147] = 192
# rear wheel
marker[258][409] = 192
marker[257][391] = 192
marker[254][421] = 192
# Now we have set the markers, we use the watershed
# algorithm to generate a marked image
marked = cv2.watershed(img, marker)
# Plot this one. If it does what we want, proceed;
# otherwise edit your markers and repeat
plt.imshow(marked, cmap='gray')
plt.show()
# Make the background black, and what we want to keep white
marked[marked == 1] = 0
marked[marked > 1] = 255
# Use a kernel to dilate the image, to not lose any detail on the outline
# I used a kernel of 3x3 pixels
kernel = np.ones((3,3),np.uint8)
dilation = cv2.dilate(marked.astype(np.float32), kernel, iterations = 1)
# Plot again to check whether the dilation is according to our needs
# If not, repeat by using a smaller/bigger kernel, or more/less iterations
plt.imshow(dilation, cmap='gray')
plt.show()
# Now apply the mask we created on the initial image
final_img = cv2.bitwise_and(img, img, mask=dilation.astype(np.uint8))
# cv2.imread reads the image as BGR, but matplotlib uses RGB
# BGR to RGB so we can plot the image with accurate colors
b, g, r = cv2.split(final_img)
final_img = cv2.merge([r, g, b])
# Plot the final result
plt.imshow(final_img)
plt.show()
code kotlin:
// Load the image
val srcOriginal = Imgcodecs.imread(currentPhotoPath)
// Create a blank image of zeros (same dimension as img)
// It should be grayscale (1 color channel)
val markers = Mat.zeros(srcOriginal.rows(), srcOriginal.cols(), CvType.CV_32S)
// This step is manual. The goal is to find the points
// which create the result we want. I suggest using a
// tool to get the pixel coordinates.
// Dictate the area of interest
for(x in my_canvas.pointsToDrawX.indices) {
for(y in my_canvas.pointsToDrawY.indices) {
markers.put(
my_canvas.pointsToDrawX.get(x).toInt(),
my_canvas.pointsToDrawY.get(y).toInt(),
255.0
)
}
}
//Now we have set the markers, we use the watershed
//algorithm to generate a marked image
Imgproc.watershed(srcOriginal, markers)
val marker_tempo = Mat()
markers.convertTo(marker_tempo, CvType.CV_8U)
// Plot this one. If it does what we want, proceed;
// otherwise edit your markers and repeat
//Create Bitmap
val bmpOut = Bitmap.createBitmap(srcOriginal.cols(), srcOriginal.rows(), Bitmap.Config.RGB_565)
Utils.matToBitmap(marker_tempo, bmpOut)
val mPath = Environment.getExternalStorageDirectory().toString() + "/gray.png"
Imgcodecs.imwrite(mPath,marker_tempo)
//Make the background black, and what we want to keep white
//Use a kernel to dilate the image, to not lose any detail on the outline
//I used a kernel of 3x3 pixels
val kernel = Mat(3, 3, CvType.CV_8U)
val dilatation = Imgproc.dilate(marker_tempo, marker_tempo, kernel)
val mPath1 = Environment.getExternalStorageDirectory().toString() + "/dilation.png"
Imgcodecs.imwrite(mPath1,marker_tempo)
//Now apply the mask we created on the initial image
val final_image = Core.bitwise_and(srcOriginal, srcOriginal, dilatation)
//cv2.imread reads the image as BGR, but matplotlib uses RGB
//BGR to RGB so we can plot the image with accurate colors
In pointsToDrawX and pointsToDrawY I'm saving all the x, y coordinates of the user's touch event on the screen. It is from these coordinates that I will pass to the watershed algorithm to perform the segmentation and remove the background from the image. Can someone help me convert this code?
//Load the image
srcOriginal = Imgcodecs.imread(currentPhotoPath)
//Create a blank image of zeros (same dimension as img)
//It should be grayscale (1 color channel)
markers = Mat.zeros(srcOriginal.rows(), srcOriginal.cols(), CvType.CV_32S)
//This step is manual. The goal is to find the points
//which create the result we want. I suggest using a
//tool to get the pixel coordinates.
//Dictate the background and set the markers to 1
for (value in 0..my_canvas.pointsToDrawY.size - 1) {
markers.put(
my_canvas.pointsToDrawX[value].toInt(),
my_canvas.pointsToDrawY[value].toInt(),
1.0
)
}
//Dictate the area of interest
//I used different values for each part of the car (for visibility)
for (value in 0..my_canvas.pointsToDrawYStepTwo.size - 1) {
markers.put(
my_canvas.pointsToDrawXStepTwo[value].toInt(),
my_canvas.pointsToDrawYStepTwo[value].toInt(),
255.0
)
}
//Now we have set the markers, we use the watershed
//algorithm to generate a marked image
watershed(srcOriginal, markers)
//Plot this one. If it does what we want, proceed;
//otherwise edit your markers and repeat
val mPath1 = Environment.getExternalStorageDirectory().toString() + "/watershed.png"
Imgcodecs.imwrite(mPath1,markers)
//Make the background black, and what we want to keep white
for (x in 0 until srcOriginal.rows()-1) {
for (y in 0 until srcOriginal.cols()-1) {
if(markers.get(x,y).get(0).equals(1.0)){
markers.put(
x,
y,
0.0
)
}
if((markers[x, y].get(0) == 255.0)){
markers.put(
x,
y,
255.0
)
}
}
}
//Use a kernel to dilate the image, to not lose any detail on the outline
//I used a kernel of 3x3 pixels
val marker_tempo = Mat()
val dilatation = Mat()
markers.convertTo(marker_tempo, CvType.CV_8U)
val kernel = Mat(3, 3, CvType.CV_8U)
Imgproc.dilate(marker_tempo, dilatation, kernel)
//Plot again to check whether the dilation is according to our needs
//If not, repeat by using a smaller/bigger kernel, or more/less iterations
val mPath2 = Environment.getExternalStorageDirectory().toString() + "/dilatation.png"
Imgcodecs.imwrite(mPath2,dilatation)
//Now apply the mask we created on the initial image
val final = Mat()
Core.bitwise_and(srcOriginal, srcOriginal, final, dilatation)
//Plot the final result
val mPath = Environment.getExternalStorageDirectory().toString() + "/final.png"
Imgcodecs.imwrite(mPath,final)
numpy.zeros_like basically creates an array the same shape as the input with all zeros in it.
https://numpy.org/doc/stable/reference/generated/numpy.zeros_like.html
In this case, you have a simple 2d array so you could roll your own just by indexing through the whole array (all dimensions) and setting everything to zero.
var marker = Array(srcOriginal.rows()) {Array(srcOriginal.cols()) {0} }
i am a newcomer in OpenCV and i am creating an OMR(Optical Mark Recognition) system in Java in order to detect the answers on a multiple choice paper sheet.I have created a form that consists of one big rectangle that is used for answering the questions by drawing the right circle, and one smaller rectangle that is for detecting a unique number that is the identity of the one answering.
Here is the image of the form:
Now my programm is detecting the upper rectangle the AM one but cannot detect the bigger one. My image is passing though 6 stages 1st dilation,2nd gray, 3rd threshold,4th blur,5th canny and 6th adaptiveThreshold. Here you can see that
dilated1 = new Mat(source1.size(), CV_8UC1);
dilate(source1, dilated1, getStructuringElement(MORPH_RECT, new Size(3, 3)));
gray1 = new Mat(dilated1.size(), CV_8UC1);
cvtColor(dilated1, gray1, COLOR_BGR2GRAY);
thresh1 = new Mat(gray1.rows(), gray1.cols(), gray1.type());
threshold(gray1, thresh1, 0, 255, THRESH_BINARY + THRESH_OTSU );
blur1 = new Mat(thresh1.size(), CV_8UC1);
blur(gray1, blur1, new Size(5.,5.));
canny1 = new Mat(blur1.size(), CV_8UC1);
Canny(blur1, canny1,160, 80);
adaptiveThresh1 = new Mat(canny1.rows(), gray1.cols(), gray1.type());
adaptiveThreshold(canny1, adaptiveThresh1, 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 11,2);
I am also using findContours like that
findContours(adaptiveThresh1.clone(), contours1, hierarchy1, RETR_TREE, CHAIN_APPROX_SIMPLE);
I have created two different java classes because there are other things to detect in the small rectangle and other on the bigger. The code above is what i am using in order to try detecting the bigger rectangle. I have tried many different numbers in all the steps and still nothing.
When i am using only the bigger rectangle in an image it works just fine, but in combination with the other one it cannot detect it. It's for my thesis and its really important for me. Any help is appreciated and whatever you want me to add in order to help you please let me know.
You can find the largest contour and second largest contour.
Suggested stages:
Convert image to Grayscale (as you did).
Draw thick white rectangle around the image - making sure there is not black contour around the image.
Apply threshold and convert to binary (as you did).
The code I posted also inverse polarity, for the contours to be white.
Find contours.
Use RETR_EXTERNAL instead of RETR_TREE, because you don't need to find contours within contours.
Iterate the contours, and find the one with the largest area, and the one with the second largest area.
The contours with the largest area is the lower rectangle.
The contours with the second largest area is the upper rectangle.
Here is a Python implementation (not JAVA, but close enough):
import cv2
# Read input image
img = cv2.imread('image.png')
# Draw thick rectangle around the image - making sure there is not black contour around the image
cv2.rectangle(img, (0, 0), (img.shape[1], img.shape[0]), (255, 255, 255), thickness = 5)
# Convert from BGR to Grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Apply threshold on gray image - use automatic threshold algorithm (use THRESH_OTSU) and invert polarity.
ret, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
# Find contours
cnts, heir = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
max_a = 0 # Maximum area
smax_a = 0 # Second maximum area
max_c = [] # Contour with maximum area
smax_c = [] # Contour with second maximum area (maximum excluding max_c)
# Iterate contours
for c in cnts:
area = cv2.contourArea(c)
if area > max_a: # If area is grater than maximum, second max = max, and max = area
smax_a = max_a
smax_c = max_c # Second max contour gets maximum contour
max_a = area
max_c = c # Maximum contour gets c
elif area > smax_a: # If area is grater than second maximum, replace second maximum
smax_a = area
smax_c = c
#Get bounding rectangle of contour with maximum area, and mark it with green rectangle
x, y, w, h = cv2.boundingRect(max_c)
cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), thickness = 2)
#Get bounding rectangle of contour with second maximum area, and mark it with blue rectangle
x, y, w, h = cv2.boundingRect(smax_c)
cv2.rectangle(img, (x, y), (x+w, y+h), (255, 0, 0), thickness = 2)
# Show result (for testing).
cv2.imshow('img', img)
cv2.waitKey(0)
cv2.destroyAllWindows()
Result:
I was able to localize the content of the following image:
This is the current Java code:
Mat image = Imgcodecs.imread("test.png");
Mat gray = new Mat();
Imgproc.cvtColor(image, gray, Imgproc.COLOR_BGR2GRAY);
Core.absdiff(gray, new Scalar(255), gray);
Imgproc.threshold(gray, gray, 5, 255, Imgproc.THRESH_TOZERO);
Mat kernel1 = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new Size(11, 11));
Mat kernel2 = Mat.ones(3, 3, CvType.CV_8U);
Mat erosion = new Mat();
Imgproc.erode(gray, erosion, kernel2, new Point(-1, -1), 1);
Mat dilation = new Mat();
Imgproc.dilate(erosion, dilation, kernel1, new Point(-1, -1), 7);
final List<MatOfPoint> contours = new ArrayList<>();
final Mat hierarchy = new Mat();
Imgproc.findContours(dilation, contours, hierarchy,
Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE);
for (MatOfPoint contour : contours) {
RotatedRect rect = Imgproc.minAreaRect(new MatOfPoint2f(contour.toArray()));
Mat box = new Mat();
Imgproc.boxPoints(rect, box);
Imgproc.drawContours(image, contours, -1, new Scalar(0,0,255));
}
This is the resulting image:
As you may see - together with the useful content there are still a few scanning artifacts located with the red contours.
Is it possible to remove these scanning artifacts in some common way(that will work not only for this picture) without damage to content?
Also, how to properly rotate the content inside of this image(not the image itself) based on contours?
This problem can be treated as a Text Detection situation.
We can use some static image analysis:
Convert to Grey Scale
Apply Blurring/Smoothing
Threshold Image
Apply Morphological Dilation
Find Connected Components
Filter out components of small area
--
Gaussian Blur
Thresholding
Inverted Colors
Dilation
Detected Areas (after filtering) UPDATED
--
System.load("opencv_java320.dll");
Mat dst = new Mat();
Mat src = Imgcodecs.imread("path/to/your/image.png");
// Converting to Grey Scale
Imgproc.cvtColor(src, dst, Imgproc.COLOR_RGB2GRAY, 0);
// Blurring/Smoothing
Imgproc.GaussianBlur(dst, src, new Size(15.0,15.0),0.0,0.0);
// Thresholding / Binarization
Imgproc.threshold(src, dst, 150,255,Imgproc.THRESH_BINARY);
Mat painted = new Mat(); // UPDATED
src.copyTo(painted); // UPDATED
// Invert colors (helps with dilation)
Core.bitwise_not(dst,src);
// Image Dilation
Mat structuringElement = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(55.0,55.0));
Imgproc.dilate(src, dst, structuringElement);
// Detect Text Areas
List<Rect> textBlocks = findTextBlocks(dst);
// Paint detected text areas
paintTextBlocks(textBlocks, painted);
static List<Rect> findTextBlocks(Mat dilated)
{
Mat labels = new Mat();
Mat stats = new Mat();
Mat centroids = new Mat();
// Find connected components
int numberOfLabels = Imgproc.connectedComponentsWithStats(dilated,labels,stats,centroids,8, CvType.CV_16U);
List<Rect> textBlocks = new ArrayList<>();
// adjust this threshold as your desire
double sizeThreshold = 0.01;
// Label 0 is considered to be the background label, so we skip it
for (int i = 1; i < numberOfLabels; i++)
{
// stats columns; [0-4] : [left top width height area}
Rect textBlock = new Rect(new Point(stats.get(i,0)[0],stats.get(i,1)[0]),new Size(stats.get(i,2)[0],
stats.get(i,3)[0]));
// stats.get(i,4)[0] is the area of the connected component / Filtering out small areas
if (Double.compare(stats.get(i,4)[0],dilated.height() * dilated.width() * sizeThreshold) > 0){
textBlocks.add(textBlock);
}
}
return textBlocks;
}
static void paintTextBlocks(List<Rect> textBlocks, Mat original)
{
for (Rect r : textBlocks)
{
Imgproc.rectangle(original, new Point(r.x,r.y), new Point(r.x+r.width,r.y+r.height),
new Scalar(100.0),2);
}
}
You can tune/adjust the following:
1) 3rd parameter of Imgproc.threshold method. Looking at the code it means that any pixel with color value higher of 150 will be replaced with 255 (white). Hence, increasing this number will result in getting fewer black/text pixels.
Decreasing the number will result in more black areas e.g. artifacts.
2) Size of Dilation structuring element (rectangle). Width and height should be the same and both odd numbers. Smaller dimensions of the structuring element means weaker dilation; smaller connected components. Larger dimensions means wider dilation with bigger connected components.
3) sizeThreshold in findTextBlocks() method. This variable controls the strength of the filtering of the connected components based on their size/area. Very small threshold will result in getting small areas e.g. artifacts and a big threshold will result in very big detected areas only.
I am developing OMR scanner android application using opencv library.
I have detected my circles inside the sheet as contours and now I want to get filled circle contours from all the obtains contours
Since java support for opencv is very less I couldnt figure out anything,
please suggest some method for the same.
//paramview is my image
Utils.bitmapToMat(paramView, localMat1);
Mat localMat2 = new Mat();
double[] lo;
Imgproc.GaussianBlur(localMat1, localMat2, new Size(5.0D, 5.0D), 7.0D, 6.5D);
Object localObject = new Mat();
Imgproc.cvtColor(localMat2, (Mat)localObject, COLOR_RGB2GRAY);
Mat cloneMat= ((Mat) localObject).clone();
localMat2 = localMat1.clone();
bitwise_not(cloneMat,cloneMat);
Imgproc.threshold(cloneMat,localMat2,127,255,Imgproc.THRESH_OTSU);
Mat thresh=localMat2.clone();
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
List<MatOfPoint> questions = new ArrayList<MatOfPoint>();
List<MatOfPoint> sorted = new ArrayList<MatOfPoint>();
//All contours detected
Mat hierarchy = new Mat();
Imgproc.findContours(localMat2, contours, hierarchy,
Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
Image of Detected circles here
I reworked my own code and found this solution. Hope it might help.
for (int contourIdx = 0; contourIdx < questionSortedR.size(); contourIdx++) {
//creating rectangle around identified contour
Rect rectCrop = boundingRect(questionSortedR.get(contourIdx));
//creating crop of that contour from actual image
Mat imageROI= thresh.submat(rectCrop);
//apply countnonzero method to that crop
int total = countNonZero(imageROI);
double pixel =total/contourArea(questionSortedR.get(contourIdx))*100;
//pixel is in percentage of area that is filled
if(pixel>=100 && pixel<=130){
//counting filled circles
count++;
}
}
I propose an alternative to the accepted answer: instead of counting pixels inside a bounding rectangle, paint the contour into a mask, then mask the original image and count the pixels inside it. I was counting black pixels on a white background, where the contour kept several pixels on the edge, so your mileage may vary. Here is my code in Python:
mask = np.zeros(bw_image.shape, np.uint8)
cv.drawContours(mask, [contour], 0, 255, -1)
inverted = cv.bitwise_not(bw_image)
masked = cv.bitwise_not(cv.bitwise_and(inverted, inverted, mask = mask))
# Grab masked image inside contour
x, y, w, h = cv.boundingRect(contour)
pixels = masked[y:y+h, x:x+w]
# Check if black is only a line, in which case whiteness is 1
kernel = np.ones((3, 3), np.uint8)
dilated = cv.dilate(pixels, kernel, iterations = 1)
whiteness = np.sum(dilated) / (255 * w * h)
I am using OpenCV in an Android application. I want the mobile application to automatically take a photo when a rectangle (something in the shape of a receipt for example) is in view. I am using Canny edge detection but when I am looking for contours, the array size is greater than 1500. Obviously it is not optimal to loop through all the contours and find the largest one so I was wondering is it possible to filter out the largest contour automatically through an api?
My code so far:
ArrayList contours;
#Override
public Mat onCameraFrame(final CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
// Clear contours array on each frame
contours.clear();
// Get Grayscale image
final Mat gray = inputFrame.gray();
// Canny edge detection
Imgproc.Canny(gray, gray, 300, 1000, 5, true);
// New empty black matrix to store the edges captured
Mat dest = new Mat();
Core.add(dest, Scalar.all(0), dest);
// Copy the edge data over to the empty black matrix
gray.copyTo(dest);
// Is there a way to filter the size of contours so that not everything is returned? Right now this function is returning a lot of contours (1500 +)
Imgproc.findContours(gray, contours, hirearchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
return dest;
}
EDIT
The user will be holding the phone and I want the application to automatically take a photo when the receipt is in view.
Example receipt
I have covered the basic techniques you may use, in the following Python code, it won't be hard to translate the code in the language of your choice, java in this case. So the technique involves:
Estimate the color of object you want to segment, which is white in your case, so safe limits for upper and lower bound can be approximated as:
RECEIPT_LOWER_BOUND = np.array([200, 200, 200])
RECEIPT_UPPER_BOUND = np.array([255, 255, 255])
Apply some Blur to input image to make the color distribution smooth, which would reduce the smaller contours in future.
img_blurred = cv2.blur(img, (5, 5))
Apply dilation to the binary image to remove the neighbouring smaller contours which surround your target largest contour
kernel = np.ones((10, 10), dtype=np.uint8)
mask = cv2.dilate(mask, kernel)
Now find contours in the mask after applying above operations and filter out the contour on the basis of contourArea.
im, contours, hierarchy = cv2.findContours(receipt_mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
largest_contour = max(contours, key=lambda x: cv2.contourArea(x))
Finally you may apply some threshold over the area to verify if the input was really a ticket or not.
Code:
import cv2
import numpy as np
# You may change the following ranges to define your own lower and upper BGR bounds.
RECEIPT_LOWER_BOUND = np.array([200, 200, 200])
RECEIPT_UPPER_BOUND = np.array([255, 255, 255])
def segment_receipt(img):
# Blur the input image to reduce the noise which in-turn reduces the number of contours
img_blurred = cv2.blur(img, (5, 5))
mask = cv2.inRange(img_blurred, RECEIPT_LOWER_BOUND, RECEIPT_UPPER_BOUND)
# Also dilate the binary mask which further reduces the salt and pepper noise
kernel = np.ones((10, 10), dtype=np.uint8)
mask = cv2.dilate(mask, kernel)
return mask
def get_largest_contour_rect(image):
receipt_mask = segment_receipt(image)
im, contours, hierarchy = cv2.findContours(receipt_mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
print "Number of contours found :", len(contours)
# Sorting the contours to get the largest one
largest_contour = max(contours, key=lambda x: cv2.contourArea(x))
# Return the last contour in sorted list as the list is sorted in increasing order.
return cv2.boundingRect(largest_contour)
image = cv2.imread("path/to/your/image.jpg")
rect = get_largest_contour_rect(image)
Output:
#J.Doe I am currently working on such a project and I have successfully being able to isolate the largest contour in the image after a whole lot of processing. The only part remaining is recognizing a rectangular contour and taking a picture.
mRgba = inputFrame.rgba();
Imgproc.Canny(mRgba,mCanny,50,200);
Imgproc.cvtColor(mRgba, mGray, Imgproc.COLOR_RGB2GRAY);
Imgproc.GaussianBlur(mGray, mGray1, new Size(3, 3), 1);
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT,new Size(9,9));
Imgproc.dilate(mGray1, mGray2, kernel);
Imgproc.Canny(mGray2, mCanny, 50, 200);
Imgproc.findContours(mCanny,contours,hierarchy,Imgproc.RETR_TREE,Imgproc.CHAIN_APPROX_SIMPLE);
double maxVal = 0;
int maxValIdx = 0;
for(int contourIdx = 0; contourIdx < contours.size(); contourIdx++){
double contourArea = Imgproc.contourArea(contours.get(contourIdx));
if(maxVal < contourArea)
{
maxVal = contourArea;
maxValIdx = contourIdx;
}
}
Imgproc.drawContours(mRgba,contours,maxValIdx,new Scalar(0,255,255),-1);
return mRgba;
Be wary of the image names i changed them over different processes.