How to identify whether the area of an object is within another area desired by the user, drawn by the mouse

Question

Hello! I recently started to venture into the Opencv library in C++. Today, I want to develop a program that applies a classifier to a polygonal ROI (5 points), obtained through 5 clicks with the mouse on the cam image. Classifier can only stay within this ROI, not outside.

Thanks to several topics here, I was able to gather some examples and assemble the code below that does what I need but in a rectangular area. The "Onmouse" function draws the rectangle. However, I can’t get past this by trying to use other topics that mention polygon ROI. Could someone help with code suggestion =).

My desire is to get a polygonal ROI with mouse instead of rectangle.

Thank you so much for your help! =)

Follows the code.

#include "opencv2/highgui/highgui.hpp"
#include <opencv2/opencv.hpp>
#include <iostream>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <iostream>
#include <opencv2/opencv.hpp>
#include <opencv2/features2d/features2d.hpp>
#include "opencv2/core/core.hpp"
#include "opencv2/flann/miniflann.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/photo/photo.hpp"
#include "opencv2/video/video.hpp"
#include "opencv2/features2d/features2d.hpp"
#include "opencv2/objdetect/objdetect.hpp"
#include "opencv2/calib3d/calib3d.hpp"
#include "opencv2/ml/ml.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/core/core_c.h"
#include "opencv2/highgui/highgui_c.h"
#include "opencv2/imgproc/imgproc_c.h"

using namespace std;
using namespace cv;


Mat src,img,ROI;
Rect cropRect(0,0,0,0);
 Point P1(0,0);
 Point P2(0,0);

const char* winName="Imagem de Origem";
bool clicked=false;
int i=0;
char imgName[15];

const char* WinComClassificador="Imagem COM CLASSIFICADOR";
const char* WinSemClassificador="Imagem SEM CLASSIFICADOR";

void detectAndDisplay( Mat frame);

String face_cascade_name =     "C:/Code/Projects_Tests/Cascade_Classifier_not_displaying/Cascade_Classifier/haarcascade_frontalface_alt.xml";
String eyes_cascade_name = "C:/Code/Projects_Tests/Cascade_Classifier_not_displaying/Cascade_Classifier/haarcascade_eye_tree_eyeglasses.xml";
CascadeClassifier face_cascade;
CascadeClassifier eyes_cascade;


void detectAndDisplay( Mat frame )
{
    std::vector<Rect> faces;
    Mat frame_gray;

    cvtColor( frame, frame_gray, COLOR_BGR2GRAY );
    equalizeHist( frame_gray, frame_gray );

    //-- Detect faces
    face_cascade.detectMultiScale( frame_gray, faces, 1.1, 2,     0|CASCADE_SCALE_IMAGE, Size(30, 30) );

    for ( size_t i = 0; i < faces.size(); i++ )
    {
        Point center( faces[i].x + faces[i].width/2, faces[i].y +     faces[i].height/2 );
        ellipse( frame, center, Size( faces[i].width/2, faces[i].height/2 ),     0, 0, 360, Scalar( 255, 0, 0 ), 4, 8, 0 );

        Mat faceROI = frame_gray( faces[i] );
        std::vector<Rect> eyes;

        //-- In each face, detect eyes
        eyes_cascade.detectMultiScale( faceROI, eyes, 1.1, 2, 0     |CASCADE_SCALE_IMAGE, Size(30, 30) );

        for ( size_t j = 0; j < eyes.size(); j++ )
        {
            Point eye_center( faces[i].x + eyes[j].x + eyes[j].width/2, faces[i].y + eyes[j].y + eyes[j].height/2 );
            int radius = cvRound( (eyes[j].width + eyes[j].height)*0.25 );
            circle( frame, eye_center, radius, Scalar( 0, 255, 255), 4, 8, 0     );
        }
    }



    //-- Show what you got
    //imshow( window_name, frame );
}

void checkBoundary(){
       if(cropRect.width>img.cols-cropRect.x)
         cropRect.width=img.cols-cropRect.x;

       if(cropRect.height>img.rows-cropRect.y)
         cropRect.height=img.rows-cropRect.y;

        if(cropRect.x<0)
         cropRect.x=0;

       if(cropRect.y<0)
         cropRect.height=0;
}

void showImage(){
    img=src.clone();
    checkBoundary();
    if(cropRect.width>0&&cropRect.height>0){
        ROI=src(cropRect);
         imshow("Regiao de Interesse",ROI);
    }
    rectangle(img, cropRect, Scalar(0,255,0), 1, 8, 0 );
    imshow(winName,img);
}


void onMouse( int event, int x, int y, int f, void* ){

    switch(event){

        case  CV_EVENT_LBUTTONDOWN  :
                                        clicked=true;

                                        P1.x=x;
                                        P1.y=y;
                                        P2.x=x;
                                        P2.y=y;
                                        break;

        case  CV_EVENT_LBUTTONUP    :
                                        P2.x=x;
                                        P2.y=y;
                                        clicked=false;
                                        break;

        case  CV_EVENT_MOUSEMOVE    :
                                        if(clicked){
                                        P2.x=x;
                                        P2.y=y;
                                        }
                                        break;

        default                     :   break;

    }


    if(clicked){
     if(P1.x>P2.x){ cropRect.x=P2.x;
                       cropRect.width=P1.x-P2.x; }
        else {         cropRect.x=P1.x;
                       cropRect.width=P2.x-P1.x; }

        if(P1.y>P2.y){ cropRect.y=P2.y;
                       cropRect.height=P1.y-P2.y; }
        else {         cropRect.y=P1.y;
                       cropRect.height=P2.y-P1.y; }

    }

showImage();

}
int main()
{
    string msg("OI FELIPE");
    VideoCapture cap(0);
    cap.open( 0 );
    cap >> src;
    if ( ! cap.isOpened() ) { printf("--(!)Error opening video capture\n");     return -4; }

    cout << "Recording..." << endl;
    namedWindow(winName,WINDOW_NORMAL);

        //-- 1. Load the cascades
    if( !face_cascade.load( face_cascade_name ) ){ printf("--(!)Error     loading face cascade\n"); return -1; };
    if( !eyes_cascade.load( eyes_cascade_name ) ){ printf("--(!)Error     loading eyes cascade\n"); return -1; };


    while (cap.read(src)) {  

   // imshow( "MyVideo", src );

    if (!clicked) {
        imshow(winName,src);
        setMouseCallback(winName,onMouse,NULL );

        if( src.empty() )
                break;

    }
   else {
        imshow(winName,src);
        setMouseCallback(winName,onMouse,NULL );

        if( src.empty() )
                break;
        }

    showImage();

        int c=waitKey(10);   

    }

    waitKey(10);
    return 0;
    }

Answer 1

After much arguing I finally understood what you wanted do (and it was not on its own merit, let it be said in passing; therefore, it remains the tip: in the future, focus on the problem instead of focusing on one solution that you stifle that works).

What you want is not to run the Cascade within a limited area (because it doesn’t make any sense!). What you want is to know if the area of an object, as detected by Cascade (in this case, the region where a human face is) is inside another area marked by the user.

Since the Cascade area is an ROI (which is rectangular in principle, since it is a sub-area of an image) and its verification area is a polygon with more sides, this comparison is not trivial. Since both polygons are closed, you could do a simple check of the "is completely inside or is completely outside" type by simply checking whether all vertices of the ROI are within the limits of the polygon. It could also do a "collision" check, just as games do (and has a series of techniques for this). However, you don’t need to know if one area "touched" the other, but whether one contains the other. Therefore, using collision techniques would potentially produce many false positives.

As you use Opencv, a very simple and effective way is for you to do so:

1. Draw each area in a distinct binary image (i.e., in an image that has only black and white), filling the area of the region completely blank.
2. Make a logical "E" of the two images, pixel by pixel. Pixels all in black are worth 0, and pixels all blank are worth 1. Like 0 E 0 or 0 E 1 result in 0, only the pixels that are 1 (white) on both images will be kept in the resulting image.
3. Count the blank (non-zero) pixels in the resulting image, and divide this value by the number of pixels in the area originally detected. This is a percentage, which therefore varies between 0 and 1. A detected ROI that is fully within your polygon will result in 1 (i.e., 100% inside! ) and an ROI that is totally outside your polygon will result in 0 (i.e., 100% off).
4. With the possession of this value, create some decision criterion to consider as "invasion" or not (for example, consider as invaded only when the given value is greater than 0.7, ie 70%).

The code

#include <opencv2\core.hpp>
#include <opencv2\highgui.hpp>
#include <opencv2\imgproc.hpp>

#include <vector>
#include <iostream>

using namespace std;
using namespace cv;

Size sz = Size(800, 600);
Mat image = Mat::zeros(sz, CV_8UC3);
Rect roi;
vector<Point> polygon;
bool buttonDown = false;

typedef enum
{
    CAPTURING_POLYGON,
    CAPTURING_ROI
} Capturing;
Capturing cap;

void roiCapture(int event, int x, int y, int flags, void *userdata)
{
    if(event == EVENT_LBUTTONDOWN)
    {
        roi = Rect(x, y, 0, 0);
        buttonDown = true;
    }
    else if(event == EVENT_LBUTTONUP)
    {
        buttonDown = false;
        roi = Rect(Point(roi.x, roi.y), Point(x, y));
    }
    else if(buttonDown)
        roi = Rect(Point(roi.x, roi.y), Point(x, y));       
}

void polygonCapture(int event, int x, int y, int flags, void *userdata)
{
    if(event == EVENT_LBUTTONUP)
        polygon.push_back(Point(x, y));
    else if(event == EVENT_RBUTTONDOWN)
        polygon.clear();
}

float calcOverlapping(const Rect &roi, const vector<Point> &polygon, const Size &sz, Mat &resp)
{
    // Desenha o polígono preenchido com branco em uma imagem binária toda em preto
    Mat imgPol = Mat::zeros(sz, CV_8UC1);
    vector<vector<Point>> points;
    points.push_back(polygon);
    int npts = polygon.size();
    fillPoly(imgPol, points, Scalar(255));

    // Desenha o retângulo preenchido com branco em uma imagem binária toda em preto
    Mat imgRoi = Mat::zeros(sz, CV_8UC1);
    rectangle(imgRoi, roi, Scalar(255, 255, 255), CV_FILLED);

    // Faz um "E" lógico das imagens. Isso causa uma interseção das áreas, pois apenas os pixels
    // que forem branco NAS DUAS imagens irão permanecer em branco na imagem de resposta.
    bitwise_and(imgPol, imgRoi, resp);

    // Conta os pixels que são diferentes de preto (0) na imagem resultante
    int intersec = countNonZero(resp);

    // Conta os pixels que são diferentes de preto (0) na imagem da ROI (porque a área dela é
    // usada como base (isto é, que se saber qual é a porcentagem da ROI que está dentro do 
    // polígono)
    int base = countNonZero(imgRoi);

    // A resposta é um percentual (entre 0 e 1), indicando quanto da ROI está na interseção.
    // Se a ROI estiver inteira dentro do polígono, a interseção vai ser total e esse valor vai
    // dar 1. Se estiver totalmente fora, a interseção vai ser nula e esse valor vai dar 0.
    return float(intersec) / float(base);
}

int main()
{
    // ===========================================
    // Captura das áreas para teste.
    // Essa parte você já faz (ou sabe fazer).
    // ===========================================
    namedWindow("Captura", WINDOW_AUTOSIZE);

    int fontFace = FONT_HERSHEY_SIMPLEX;
    double fontScale = 0.5;
    int thickness = 1;

    // Começa capturando a área do polígono
    cap = CAPTURING_POLYGON;
    setMouseCallback("Captura", polygonCapture, NULL);
    bool ok = true;
    while(ok)
    {
        // Pinta tudo de preto
        image = Scalar(0, 0, 0);

        // Desenha o texto de ajuda
        string text;
        if(cap == CAPTURING_ROI)
            text = "Desenhe o ROI com o mouse. Pressione [C] para continuar.";
        else
            text = "Adicione pontos ao POLIGONO com o mouse (botao direito limpa). Pressione [C] para continuar.";
        putText(image, text, Point(0, 15), fontFace, fontScale, Scalar(255, 255, 255), thickness);

        // Desenha o polígono, se ele tiver dados
        if(polygon.size() == 1)
            circle(image, polygon[0], 1, Scalar(255, 0, 255));
        else if(polygon.size() > 1)
        {
            vector<vector<Point>> points;
            points.push_back(polygon);
            int npts = polygon.size();
            fillPoly(image, points, Scalar(255, 0, 255));
        }

        // Desenha a ROI, se ela tiver dados
        if(roi.width != 0 || roi.height != 0)
            rectangle(image, roi, Scalar(0, 255, 255));

        imshow("Captura", image);
        int k = waitKey(10);
        switch(k)
        {
            case 27: // ESC
            case 'q':
            case 'Q':
                destroyAllWindows();
                return 0;

            case 'c':
            case 'C':
                if(cap == CAPTURING_POLYGON)
                {
                    cap = CAPTURING_ROI;
                    // Troca pra captura da ROI
                    setMouseCallback("Captura", roiCapture, NULL);
                }
                else
                    ok = false;
                break;
        }
    }

    // ===========================================
    // Cálculo do percentual de interseção.
    // Essa parte é a novidade.
    // ===========================================

    Mat resp;
    float val = calcOverlapping(roi, polygon, sz, resp);

    cout << "Valor de intereseção calculado: " << val << endl;

    imshow("Debug", resp);
    waitKey(0);

    return 0;
}

The really important function is to calcOverlapping, who does the "magic". :)

Examples of Result

In the following examples, the left image indicates the ROI (in yellow) and the polygon (in magenta), and the right image indicates only the area of intersection in the resulting binary image of the logical E between the two binary images constructed from the figures in the left image.

Valor de interseção calculado: 0.748836

Valor de interseção calculado: 0.012016

Editing: Real Use Example

You created another question to ask essentially the same thing, but this time provided a concrete example of the image below:

Well, if you want the people detector (Cascade) to find only people within the region delimited by the polygon in red, just use what I explained above and do the logical operation to have only the pixels of the polygon region in an ROI still rectangular (the "final" image in the example below). Then apply the Cascade in this new image. Example (in Python this time, because I did it in a hurry - but it’s the same thing in C++):

import numpy as np
import cv2

image = cv2.imread('teste.png', cv2.IMREAD_COLOR)

# Define a Região de Interesse (ROI)
left = 15
top = 118
width = 615
height = 365
roi = image[top:height, left:width]

# Define o polígono delimitador
points = np.array([[(0, 184), (165, 40), (402, 0), (598, 20), (482, 56), (342, 245)]])
mask = np.zeros(roi.shape, roi.dtype)
cv2.fillPoly(mask, points, (255, 255, 255)) # Preenche com branco (note que a cor é RGB - i.e. 3 bandas, já que a imagem não é binária!)

final = cv2.bitwise_and(roi, mask)

cv2.imshow('Imagem Original', image)
cv2.imshow('ROI', roi)
cv2.imshow('Mascara', mask)
cv2.imshow('Final', final)

cv2.waitKey(0)

Upshot: