Creating your own corner detector

Goal

In this tutorial you will learn how to:

• Use the OpenCV function cv::cornerEigenValsAndVecs to find the eigenvalues and eigenvectors to determine if a pixel is a corner.
• Use the OpenCV function cv::cornerMinEigenVal to find the minimum eigenvalues for corner detection.
• Implement our own version of the Harris detector as well as the Shi-Tomasi detector, by using the two functions above.

Theory

Code

C++JavaPython

C++

This tutorial code's is shown lines below. You can also download it from here

#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include <iostream>

using namespace cv;
using namespace std;

Mat src, src_gray;
Mat myHarris_dst, myHarris_copy, Mc;
Mat myShiTomasi_dst, myShiTomasi_copy;

int myShiTomasi_qualityLevel = 50;
int myHarris_qualityLevel = 50;
int max_qualityLevel = 100;

double myHarris_minVal, myHarris_maxVal;
double myShiTomasi_minVal, myShiTomasi_maxVal;

RNG rng(12345);

const char* myHarris_window = "My Harris corner detector";
const char* myShiTomasi_window = "My Shi Tomasi corner detector";

void myShiTomasi_function( int, void* );
void myHarris_function( int, void* );

int main( int argc, char** argv )
{
    CommandLineParser parser( argc, argv, "{@input | ../data/building.jpg | input image}" );
    src = imread( parser.get<String>( "@input" ) );
    if ( src.empty() )
    {
        cout << "Could not open or find the image!\n" << endl;
        cout << "Usage: " << argv[0] << " <Input image>" << endl;
        return -1;
    }
    cvtColor( src, src_gray, COLOR_BGR2GRAY );

    int blockSize = 3, apertureSize = 3;

    cornerEigenValsAndVecs( src_gray, myHarris_dst, blockSize, apertureSize );

    /* calculate Mc */
    Mc = Mat( src_gray.size(), CV_32FC1 );
    for( int i = 0; i < src_gray.rows; i++ )
    {
        for( int j = 0; j < src_gray.cols; j++ )
        {
            float lambda_1 = myHarris_dst.at<Vec6f>(i, j)[0];
            float lambda_2 = myHarris_dst.at<Vec6f>(i, j)[1];
            Mc.at<float>(i, j) = lambda_1*lambda_2 - 0.04f*pow( ( lambda_1 + lambda_2 ), 2 );
        }
    }

    minMaxLoc( Mc, &myHarris_minVal, &myHarris_maxVal );

    /* Create Window and Trackbar */
    namedWindow( myHarris_window );
    createTrackbar( "Quality Level:", myHarris_window, &myHarris_qualityLevel, max_qualityLevel, myHarris_function );
    myHarris_function( 0, 0 );

    cornerMinEigenVal( src_gray, myShiTomasi_dst, blockSize, apertureSize );

    minMaxLoc( myShiTomasi_dst, &myShiTomasi_minVal, &myShiTomasi_maxVal );

    /* Create Window and Trackbar */
    namedWindow( myShiTomasi_window );
    createTrackbar( "Quality Level:", myShiTomasi_window, &myShiTomasi_qualityLevel, max_qualityLevel, myShiTomasi_function );
    myShiTomasi_function( 0, 0 );

    waitKey();
    return 0;
}

void myShiTomasi_function( int, void* )
{
    myShiTomasi_copy = src.clone();
    myShiTomasi_qualityLevel = MAX(myShiTomasi_qualityLevel, 1);

    for( int i = 0; i < src_gray.rows; i++ )
    {
        for( int j = 0; j < src_gray.cols; j++ )
        {
            if( myShiTomasi_dst.at<float>(i,j) > myShiTomasi_minVal + ( myShiTomasi_maxVal - myShiTomasi_minVal )*myShiTomasi_qualityLevel/max_qualityLevel )
            {
                circle( myShiTomasi_copy, Point(j,i), 4, Scalar( rng.uniform(0,256), rng.uniform(0,256), rng.uniform(0,256) ), FILLED );
            }
        }
    }
    imshow( myShiTomasi_window, myShiTomasi_copy );
}

void myHarris_function( int, void* )
{
    myHarris_copy = src.clone();
    myHarris_qualityLevel = MAX(myHarris_qualityLevel, 1);

    for( int i = 0; i < src_gray.rows; i++ )
    {
        for( int j = 0; j < src_gray.cols; j++ )
        {
            if( Mc.at<float>(i,j) > myHarris_minVal + ( myHarris_maxVal - myHarris_minVal )*myHarris_qualityLevel/max_qualityLevel )
            {
                circle( myHarris_copy, Point(j,i), 4, Scalar( rng.uniform(0,256), rng.uniform(0,256), rng.uniform(0,256) ), FILLED );
            }
        }
    }
    imshow( myHarris_window, myHarris_copy );
}

Java

This tutorial code's is shown lines below. You can also download it from here

import java.awt.BorderLayout;
import java.awt.Container;
import java.awt.Image;
import java.util.Random;

import javax.swing.BoxLayout;
import javax.swing.ImageIcon;
import javax.swing.JFrame;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.JSlider;
import javax.swing.event.ChangeEvent;
import javax.swing.event.ChangeListener;

import org.opencv.core.Core;
import org.opencv.core.Core.MinMaxLocResult;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.highgui.HighGui;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;

class CornerDetector {
    private Mat src = new Mat();
    private Mat srcGray = new Mat();
    private Mat harrisDst = new Mat();
    private Mat shiTomasiDst = new Mat();
    private Mat harrisCopy = new Mat();
    private Mat shiTomasiCopy = new Mat();
    private Mat Mc = new Mat();
    private JFrame frame;
    private JLabel harrisImgLabel;
    private JLabel shiTomasiImgLabel;
    private static final int MAX_QUALITY_LEVEL = 100;
    private int qualityLevel = 50;
    private double harrisMinVal;
    private double harrisMaxVal;
    private double shiTomasiMinVal;
    private double shiTomasiMaxVal;
    private Random rng = new Random(12345);

    public CornerDetector(String[] args) {
        String filename = args.length > 0 ? args[0] : "../data/building.jpg";
        src = Imgcodecs.imread(filename);
        if (src.empty()) {
            System.err.println("Cannot read image: " + filename);
            System.exit(0);
        }

        Imgproc.cvtColor(src, srcGray, Imgproc.COLOR_BGR2GRAY);

        // Create and set up the window.
        frame = new JFrame("Creating your own corner detector demo");
        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        // Set up the content pane.
        Image img = HighGui.toBufferedImage(src);
        addComponentsToPane(frame.getContentPane(), img);
        // Use the content pane's default BorderLayout. No need for
        // setLayout(new BorderLayout());
        // Display the window.
        frame.pack();
        frame.setVisible(true);

        int blockSize = 3, apertureSize = 3;

        Imgproc.cornerEigenValsAndVecs(srcGray, harrisDst, blockSize, apertureSize);

        /* calculate Mc */
        Mc = Mat.zeros(srcGray.size(), CvType.CV_32F);

        float[] harrisData = new float[(int) (harrisDst.total() * harrisDst.channels())];
        harrisDst.get(0, 0, harrisData);
        float[] McData = new float[(int) (Mc.total() * Mc.channels())];
        Mc.get(0, 0, McData);

        for( int i = 0; i < srcGray.rows(); i++ ) {
            for( int j = 0; j < srcGray.cols(); j++ ) {
                float lambda1 = harrisData[(i*srcGray.cols() + j) * 6];
                float lambda2 = harrisData[(i*srcGray.cols() + j) * 6 + 1];
                McData[i*srcGray.cols()+j] = (float) (lambda1*lambda2 - 0.04f*Math.pow( ( lambda1 + lambda2 ), 2 ));
            }
        }
        Mc.put(0, 0, McData);

        MinMaxLocResult res = Core.minMaxLoc(Mc);
        harrisMinVal = res.minVal;
        harrisMaxVal = res.maxVal;

        Imgproc.cornerMinEigenVal(srcGray, shiTomasiDst, blockSize, apertureSize);
        res = Core.minMaxLoc(shiTomasiDst);
        shiTomasiMinVal = res.minVal;
        shiTomasiMaxVal = res.maxVal;

        update();
    }

    private void addComponentsToPane(Container pane, Image img) {
        if (!(pane.getLayout() instanceof BorderLayout)) {
            pane.add(new JLabel("Container doesn't use BorderLayout!"));
            return;
        }

        JPanel sliderPanel = new JPanel();
        sliderPanel.setLayout(new BoxLayout(sliderPanel, BoxLayout.PAGE_AXIS));

        sliderPanel.add(new JLabel("Max  corners:"));
        JSlider slider = new JSlider(0, MAX_QUALITY_LEVEL, qualityLevel);
        slider.setMajorTickSpacing(20);
        slider.setMinorTickSpacing(10);
        slider.setPaintTicks(true);
        slider.setPaintLabels(true);
        slider.addChangeListener(new ChangeListener() {
            @Override
            public void stateChanged(ChangeEvent e) {
                JSlider source = (JSlider) e.getSource();
                qualityLevel = source.getValue();
                update();
            }
        });
        sliderPanel.add(slider);
        pane.add(sliderPanel, BorderLayout.PAGE_START);

        JPanel imgPanel = new JPanel();
        harrisImgLabel = new JLabel(new ImageIcon(img));
        shiTomasiImgLabel = new JLabel(new ImageIcon(img));
        imgPanel.add(harrisImgLabel);
        imgPanel.add(shiTomasiImgLabel);
        pane.add(imgPanel, BorderLayout.CENTER);
    }

    private void update() {
        int qualityLevelVal = Math.max(qualityLevel, 1);

        //Harris
        harrisCopy = src.clone();

        float[] McData = new float[(int) (Mc.total() * Mc.channels())];
        Mc.get(0, 0, McData);
        for (int i = 0; i < srcGray.rows(); i++) {
            for (int j = 0; j < srcGray.cols(); j++) {
                if (McData[i * srcGray.cols() + j] > harrisMinVal
                        + (harrisMaxVal - harrisMinVal) * qualityLevelVal / MAX_QUALITY_LEVEL) {
                    Imgproc.circle(harrisCopy, new Point(j, i), 4,
                            new Scalar(rng.nextInt(256), rng.nextInt(256), rng.nextInt(256)), Imgproc.FILLED);
                }
            }
        }

        //Shi-Tomasi
        shiTomasiCopy = src.clone();

        float[] shiTomasiData = new float[(int) (shiTomasiDst.total() * shiTomasiDst.channels())];
        shiTomasiDst.get(0, 0, shiTomasiData);
        for (int i = 0; i < srcGray.rows(); i++) {
            for (int j = 0; j < srcGray.cols(); j++) {
                if (shiTomasiData[i * srcGray.cols() + j] > shiTomasiMinVal
                        + (shiTomasiMaxVal - shiTomasiMinVal) * qualityLevelVal / MAX_QUALITY_LEVEL) {
                    Imgproc.circle(shiTomasiCopy, new Point(j, i), 4,
                            new Scalar(rng.nextInt(256), rng.nextInt(256), rng.nextInt(256)), Imgproc.FILLED);
                }
            }
        }

        harrisImgLabel.setIcon(new ImageIcon(HighGui.toBufferedImage(harrisCopy)));
        shiTomasiImgLabel.setIcon(new ImageIcon(HighGui.toBufferedImage(shiTomasiCopy)));
        frame.repaint();
    }
}

public class CornerDetectorDemo {
    public static void main(String[] args) {
        // Load the native OpenCV library
        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);

        // Schedule a job for the event dispatch thread:
        // creating and showing this application's GUI.
        javax.swing.SwingUtilities.invokeLater(new Runnable() {
            @Override
            public void run() {
                new CornerDetector(args);
            }
        });
    }
}

Python

This tutorial code's is shown lines below. You can also download it from here

from __future__ import print_function
import cv2 as cv
import numpy as np
import argparse
import random as rng

myHarris_window = 'My Harris corner detector'
myShiTomasi_window = 'My Shi Tomasi corner detector'
myHarris_qualityLevel = 50
myShiTomasi_qualityLevel = 50
max_qualityLevel = 100
rng.seed(12345)

def myHarris_function(val):
    myHarris_copy = np.copy(src)
    myHarris_qualityLevel = max(val, 1)

    for i in range(src_gray.shape[0]):
        for j in range(src_gray.shape[1]):
            if Mc[i,j] > myHarris_minVal + ( myHarris_maxVal - myHarris_minVal )*myHarris_qualityLevel/max_qualityLevel:
                cv.circle(myHarris_copy, (j,i), 4, (rng.randint(0,256), rng.randint(0,256), rng.randint(0,256)), cv.FILLED)

    cv.imshow(myHarris_window, myHarris_copy)

def myShiTomasi_function(val):
    myShiTomasi_copy = np.copy(src)
    myShiTomasi_qualityLevel = max(val, 1)

    for i in range(src_gray.shape[0]):
        for j in range(src_gray.shape[1]):
            if myShiTomasi_dst[i,j] > myShiTomasi_minVal + ( myShiTomasi_maxVal - myShiTomasi_minVal )*myShiTomasi_qualityLevel/max_qualityLevel:
                cv.circle(myShiTomasi_copy, (j,i), 4, (rng.randint(0,256), rng.randint(0,256), rng.randint(0,256)), cv.FILLED)

    cv.imshow(myShiTomasi_window, myShiTomasi_copy)

# Load source image and convert it to gray
parser = argparse.ArgumentParser(description='Code for Creating your own corner detector tutorial.')
parser.add_argument('--input', help='Path to input image.', default='building.jpg')
args = parser.parse_args()

src = cv.imread(cv.samples.findFile(args.input))
if src is None:
    print('Could not open or find the image:', args.input)
    exit(0)

src_gray = cv.cvtColor(src, cv.COLOR_BGR2GRAY)

# Set some parameters
blockSize = 3
apertureSize = 3

# My Harris matrix -- Using cornerEigenValsAndVecs
myHarris_dst = cv.cornerEigenValsAndVecs(src_gray, blockSize, apertureSize)

# calculate Mc
Mc = np.empty(src_gray.shape, dtype=np.float32)
for i in range(src_gray.shape[0]):
    for j in range(src_gray.shape[1]):
        lambda_1 = myHarris_dst[i,j,0]
        lambda_2 = myHarris_dst[i,j,1]
        Mc[i,j] = lambda_1*lambda_2 - 0.04*pow( ( lambda_1 + lambda_2 ), 2 )

myHarris_minVal, myHarris_maxVal, _, _ = cv.minMaxLoc(Mc)

# Create Window and Trackbar
cv.namedWindow(myHarris_window)
cv.createTrackbar('Quality Level:', myHarris_window, myHarris_qualityLevel, max_qualityLevel, myHarris_function)
myHarris_function(myHarris_qualityLevel)

# My Shi-Tomasi -- Using cornerMinEigenVal
myShiTomasi_dst = cv.cornerMinEigenVal(src_gray, blockSize, apertureSize)

myShiTomasi_minVal, myShiTomasi_maxVal, _, _ = cv.minMaxLoc(myShiTomasi_dst)

# Create Window and Trackbar
cv.namedWindow(myShiTomasi_window)
cv.createTrackbar('Quality Level:', myShiTomasi_window, myShiTomasi_qualityLevel, max_qualityLevel, myShiTomasi_function)
myShiTomasi_function(myShiTomasi_qualityLevel)

cv.waitKey()

Explanation

Result