Develop Reference

OpenCV - find blackboard edges on video and images

UPDATE
You can find all the images I have for testing on my GitHub here:
GitHub repository with sources
There are also 2 videos, where the detection should work on as well
ORIGINAL QUESTION
I tried to use OpenCV 4.x.x to find the edges of a blackboard (image following), but somehow I cannot succeed. My code at the moment looks like this: (Android with OpenCV and live camera feed), where imgMat is a Mat from the camera feed:
Mat gray = new Mat();
Imgproc.cvtColor(imgMat, gray, Imgproc.COLOR_RGB2BGR);
Mat blurred = new Mat();
Imgproc.blur(gray, blurred, new org.opencv.core.Size(3, 3));
Mat canny = new Mat();
Imgproc.Canny(blurred, canny, 80, 230);
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new org.opencv.core.Size(2, 2));
Mat dilated = new Mat();
Imgproc.morphologyEx(canny, dilated, Imgproc.MORPH_DILATE, kernel, new Point(0, 0), 10);
Mat rectImage = new Mat();
Imgproc.morphologyEx(dilated, rectImage, Imgproc.MORPH_CLOSE, kernel, new Point(0, 0), 5);
Mat endproduct = new Mat();
Imgproc.Canny(rectImage, endproduct, 120, 230);
List<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
Imgproc.findContours(endproduct, contours, hierarchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
double maxArea = 0;
boolean hasContour = false;
MatOfPoint2f biggestContour = new MatOfPoint2f();
Iterator<MatOfPoint> each = contours.iterator();
while (each.hasNext()) {
MatOfPoint wrapper = each.next();
double area = Imgproc.contourArea(wrapper);
if (area > maxArea) {
maxArea = area;
biggestContour = new MatOfPoint2f(wrapper.toArray());
hasContour = true;
}
}
if (hasContour) {
Mat output = imgMat.clone();
MatOfPoint2f approx = new MatOfPoint2f();
MatOfPoint poly = new MatOfPoint();
Imgproc.approxPolyDP(biggestContour, approx, Imgproc.arcLength(biggestContour, true) * .02, true);
approx.convertTo(poly, CvType.CV_32S);
Rect rect = Imgproc.boundingRect(poly);
}
Somehow I am not able to get it working, although the same code(written in python) worked on my computer with a video. I take the output from the rectangle and display it on my mobile screen, where it flickers around a lot and does not work properly.
These are my images I tried the python program on, and they worked:
What am I doing wrong? I am not able to constantly detect the edges of the blackboard.
Additional information about the blackboard:
always rectangular
may have different lighting
the text should be ignored, only the main board should be detected
the outer blackboard should be ignored as well
only the contour for the main board should be shown/returned
Thanks for any advice or code!

I used HSV because that's the easiest way to detect specific colors. I used an abundancy test to automatically select the color threshold (so this will work for green or blue boards). However, this test will fail on white or black boards since white and black count as all colors according to hue. Instead, in HSV, white and black are easiest to detect as very low saturation (white) or as very low value (black).
I did a 3-way check for each and selected the mask that had the most pixels in it (I assume that the boards are the majority of the image). I'm not sure how this will work on other images since we only have one here, so this may or may not work for other boards.
I used approxPolyDP to cut down on the number of points in the contour until I had 4 points and used that to draw the shape.
import cv2
import numpy as np
# get unique colors (to speed up search) and return the most abundant mask
def getAbundantColor(channel, margin):
# get uniques
unique_colors, counts = np.unique(channel, return_counts=True);
# check for the most abundant color
most = None;
biggest_count = -1;
for col in unique_colors:
# count number of white pixels
mask = cv2.inRange(channel, int(col - margin), int(col + margin));
count = np.count_nonzero(mask);
# if bigger, set new "most"
if count > biggest_count:
biggest_count = count;
most = mask;
return most, biggest_count;
# load image
img = cv2.imread("blackboard.jpg");
# it's huge, scale down so that we can see the whole thing
h, w = img.shape[:2];
scale = 0.25;
h = int(scale*h);
w = int(scale*w);
img = cv2.resize(img, (w,h));
# hsv
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV);
h,s,v = cv2.split(hsv);
# median blur to get rid of most of the text
h = cv2.medianBlur(h, 5);
s = cv2.medianBlur(s, 5);
v = cv2.medianBlur(v, 5);
# get most abundant color
color_margin = 30;
hmask, hcount = getAbundantColor(h, color_margin);
# detect white and black separately
light_margin = 30;
# white
wmask = cv2.inRange(s, 0, light_margin);
wcount = np.count_nonzero(wmask);
# black
bmask = cv2.inRange(v, 0, light_margin);
bcount = np.count_nonzero(bmask);
# check which is biggest
sorter = [[hcount, hmask], [wcount, wmask], [bcount, bmask]];
sorter.sort();
mask = sorter[-1][1];
# dilate and erode to close holes
kernel = np.ones((3,3), np.uint8);
mask = cv2.dilate(mask, kernel, iterations = 2);
mask = cv2.erode(mask, kernel, iterations = 4);
mask = cv2.dilate(mask, kernel, iterations = 2);
# get contours # OpenCV 3.4, in OpenCV 2* or 4* it returns (contours, _)
_, contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE);
# for each contour, approximate a simpler shape until we have 4 points
simplified = [];
for con in contours:
# go until we have 4 points
num_points = 999999;
step_size = 0.01;
percent = step_size;
while num_points >= 4:
# get number of points
epsilon = percent * cv2.arcLength(con, True);
approx = cv2.approxPolyDP(con, epsilon, True);
num_points = len(approx);
# increment
percent += step_size;
# step back and get the points
# there could be more than 4 points if our step size misses it
percent -= step_size * 2;
epsilon = percent * cv2.arcLength(con, True);
approx = cv2.approxPolyDP(con, epsilon, True);
simplified.append(approx);
cv2.drawContours(img, simplified, -1, (0,0,200), 2);
# print out the number of points
for points in simplified:
print("Num Points: " + str(len(points)));
# show image
cv2.imshow("Image", img);
cv2.imshow("Hue", h);
cv2.imshow("Mask", mask);
cv2.waitKey(0);
Edit: In order to accommodate the uncertainty in the board's color and appearance I run the assumption that the board itself will be the majority of the picture. The lines involving the sorter are looking for the most abundant color in the image. If the white wall behind the board takes up more space in the image then that'll be the color that gets selected for the mask.
There are other ways to try and select just the board, but it's really difficult to come up with a catch-all solution. The rest of the code should do its job the same if you can come up with some way of masking the board. If you're willing to budge on the unknown color assumption and provide the original pictures of the failing cases then I can probably come up with an appropriate mask.

How to remove a contour inside contour in Java OpenCV?

I'm trying to run an OCR on a speed sign. I'm getting the contours like below :
static ArrayList<MatOfPoint> getContours(Mat fgMask) {
ArrayList<MatOfPoint> contours = new ArrayList<>();
float threshold = 100.0f;
Mat cannyOutput = new Mat();
Imgproc.Canny(fgMask, cannyOutput, threshold, threshold * 3);
Mat hierarchy = new Mat();
Imgproc.findContours(cannyOutput, contours, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE);
hierarchy.release();
return contours;
}
But sometimes, I detect a 0 inside a 0. As you can see here :
I know that we can use the hierachy mat to do what I want. I just don't understand how to do it in Java.
Here is a solution in python

draw contours in Android using opencv

I have code in Android which processes an image and returns a binary image.
Imgproc.cvtColor(source, middle, Imgproc.COLOR_RGB2GRAY);
Mat element = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new Size(15, 15), new Point(0, 0));
Imgproc.morphologyEx(middle, middle, MORPH_TOPHAT, element, new Point(0, 0));
Imgproc.threshold(middle, middle, 20, 255, Imgproc.THRESH_BINARY);
[][2
Now, my requirement is, that instead of binary image i need to highlight the dents on the original image. Like this:
What I figured out is to use
Core.findnonzero()
to get the coordinates of the dents and then use drawcontours on the original image.This is just an idea.
My questions are:
1. What is the best way to do it?
2. what is matofpoint?

You can use findContours() function of OpenCV on the binary image (middle as in your code) and then iterate on all over the top level contours in the list and draw them on the original (source) image, just something like this:
List<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
Imgproc.findContours(middle, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
// now iterate over all top level contours
for(int idx = 0; idx >= 0; idx = (int) hierarchy.get(0, idx)[0]) {
MatOfPoint matOfPoint = contours.get(idx);
Rect rect = Imgproc.boundingRect(matOfPoint);
Imgproc.rectangle(originalImage, rect.tl(), rect.br(), new Scalar(0, 0, 255));
}

how to count non-zero pixels inside a contour opencv

I am developing OMR scanner android application using opencv library.
I have detected my circles inside the sheet as contours and now I want to get filled circle contours from all the obtains contours
Since java support for opencv is very less I couldnt figure out anything,
please suggest some method for the same.
//paramview is my image
Utils.bitmapToMat(paramView, localMat1);
Mat localMat2 = new Mat();
double[] lo;
Imgproc.GaussianBlur(localMat1, localMat2, new Size(5.0D, 5.0D), 7.0D, 6.5D);
Object localObject = new Mat();
Imgproc.cvtColor(localMat2, (Mat)localObject, COLOR_RGB2GRAY);
Mat cloneMat= ((Mat) localObject).clone();
localMat2 = localMat1.clone();
bitwise_not(cloneMat,cloneMat);
Imgproc.threshold(cloneMat,localMat2,127,255,Imgproc.THRESH_OTSU);
Mat thresh=localMat2.clone();
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
List<MatOfPoint> questions = new ArrayList<MatOfPoint>();
List<MatOfPoint> sorted = new ArrayList<MatOfPoint>();
//All contours detected
Mat hierarchy = new Mat();
Imgproc.findContours(localMat2, contours, hierarchy,
Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
Image of Detected circles here

I reworked my own code and found this solution. Hope it might help.
for (int contourIdx = 0; contourIdx < questionSortedR.size(); contourIdx++) {
//creating rectangle around identified contour
Rect rectCrop = boundingRect(questionSortedR.get(contourIdx));
//creating crop of that contour from actual image
Mat imageROI= thresh.submat(rectCrop);
//apply countnonzero method to that crop
int total = countNonZero(imageROI);
double pixel =total/contourArea(questionSortedR.get(contourIdx))*100;
//pixel is in percentage of area that is filled
if(pixel>=100 && pixel<=130){
//counting filled circles
count++;
}
}

I propose an alternative to the accepted answer: instead of counting pixels inside a bounding rectangle, paint the contour into a mask, then mask the original image and count the pixels inside it. I was counting black pixels on a white background, where the contour kept several pixels on the edge, so your mileage may vary. Here is my code in Python:
mask = np.zeros(bw_image.shape, np.uint8)
cv.drawContours(mask, [contour], 0, 255, -1)
inverted = cv.bitwise_not(bw_image)
masked = cv.bitwise_not(cv.bitwise_and(inverted, inverted, mask = mask))
# Grab masked image inside contour
x, y, w, h = cv.boundingRect(contour)
pixels = masked[y:y+h, x:x+w]
# Check if black is only a line, in which case whiteness is 1
kernel = np.ones((3, 3), np.uint8)
dilated = cv.dilate(pixels, kernel, iterations = 1)
whiteness = np.sum(dilated) / (255 * w * h)

OpenCV Java limit contours

I am using OpenCV in an Android application. I want the mobile application to automatically take a photo when a rectangle (something in the shape of a receipt for example) is in view. I am using Canny edge detection but when I am looking for contours, the array size is greater than 1500. Obviously it is not optimal to loop through all the contours and find the largest one so I was wondering is it possible to filter out the largest contour automatically through an api?
My code so far:
ArrayList contours;
#Override
public Mat onCameraFrame(final CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
// Clear contours array on each frame
contours.clear();
// Get Grayscale image
final Mat gray = inputFrame.gray();
// Canny edge detection
Imgproc.Canny(gray, gray, 300, 1000, 5, true);
// New empty black matrix to store the edges captured
Mat dest = new Mat();
Core.add(dest, Scalar.all(0), dest);
// Copy the edge data over to the empty black matrix
gray.copyTo(dest);
// Is there a way to filter the size of contours so that not everything is returned? Right now this function is returning a lot of contours (1500 +)
Imgproc.findContours(gray, contours, hirearchy, Imgproc.RETR_LIST, Imgproc.CHAIN_APPROX_SIMPLE);
return dest;
}
EDIT
The user will be holding the phone and I want the application to automatically take a photo when the receipt is in view.
Example receipt

I have covered the basic techniques you may use, in the following Python code, it won't be hard to translate the code in the language of your choice, java in this case. So the technique involves:
Estimate the color of object you want to segment, which is white in your case, so safe limits for upper and lower bound can be approximated as:
RECEIPT_LOWER_BOUND = np.array([200, 200, 200])
RECEIPT_UPPER_BOUND = np.array([255, 255, 255])
Apply some Blur to input image to make the color distribution smooth, which would reduce the smaller contours in future.
img_blurred = cv2.blur(img, (5, 5))
Apply dilation to the binary image to remove the neighbouring smaller contours which surround your target largest contour
kernel = np.ones((10, 10), dtype=np.uint8)
mask = cv2.dilate(mask, kernel)
Now find contours in the mask after applying above operations and filter out the contour on the basis of contourArea.
im, contours, hierarchy = cv2.findContours(receipt_mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
largest_contour = max(contours, key=lambda x: cv2.contourArea(x))
Finally you may apply some threshold over the area to verify if the input was really a ticket or not.
Code:
import cv2
import numpy as np
# You may change the following ranges to define your own lower and upper BGR bounds.
RECEIPT_LOWER_BOUND = np.array([200, 200, 200])
RECEIPT_UPPER_BOUND = np.array([255, 255, 255])
def segment_receipt(img):
# Blur the input image to reduce the noise which in-turn reduces the number of contours
img_blurred = cv2.blur(img, (5, 5))
mask = cv2.inRange(img_blurred, RECEIPT_LOWER_BOUND, RECEIPT_UPPER_BOUND)
# Also dilate the binary mask which further reduces the salt and pepper noise
kernel = np.ones((10, 10), dtype=np.uint8)
mask = cv2.dilate(mask, kernel)
return mask
def get_largest_contour_rect(image):
receipt_mask = segment_receipt(image)
im, contours, hierarchy = cv2.findContours(receipt_mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
print "Number of contours found :", len(contours)
# Sorting the contours to get the largest one
largest_contour = max(contours, key=lambda x: cv2.contourArea(x))
# Return the last contour in sorted list as the list is sorted in increasing order.
return cv2.boundingRect(largest_contour)
image = cv2.imread("path/to/your/image.jpg")
rect = get_largest_contour_rect(image)
Output:

#J.Doe I am currently working on such a project and I have successfully being able to isolate the largest contour in the image after a whole lot of processing. The only part remaining is recognizing a rectangular contour and taking a picture.
mRgba = inputFrame.rgba();
Imgproc.Canny(mRgba,mCanny,50,200);
Imgproc.cvtColor(mRgba, mGray, Imgproc.COLOR_RGB2GRAY);
Imgproc.GaussianBlur(mGray, mGray1, new Size(3, 3), 1);
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT,new Size(9,9));
Imgproc.dilate(mGray1, mGray2, kernel);
Imgproc.Canny(mGray2, mCanny, 50, 200);
Imgproc.findContours(mCanny,contours,hierarchy,Imgproc.RETR_TREE,Imgproc.CHAIN_APPROX_SIMPLE);
double maxVal = 0;
int maxValIdx = 0;
for(int contourIdx = 0; contourIdx < contours.size(); contourIdx++){
double contourArea = Imgproc.contourArea(contours.get(contourIdx));
if(maxVal < contourArea)
{
maxVal = contourArea;
maxValIdx = contourIdx;
}
}
Imgproc.drawContours(mRgba,contours,maxValIdx,new Scalar(0,255,255),-1);
return mRgba;
Be wary of the image names i changed them over different processes.