cxcore_introduction.tex

\ifCpp
\chapter{Introduction}

Starting from OpenCV 2.0 the new modern C++ interface has been introduced.
It is crisp (less typing is needed to code the same thing), type-safe (no more \texttt{CvArr*} a.k.a. \texttt{void*})
and, in general, more convenient to use. Here is a short example of what it looks like:

\begin{lstlisting}
//
// Simple retro-style photo effect done by adding noise to
// the luminance channel and reducing intensity of the chroma channels
//

// include standard OpenCV headers, same as before
#include "cv.h"
#include "highgui.h"

// all the new API is put into "cv" namespace. Export its content
using namespace cv;

// enable/disable use of mixed API in the code below.
#define DEMO_MIXED_API_USE 1

int main( int argc, char** argv )
{
    const char* imagename = argc > 1 ? argv[1] : "lena.jpg";
#if DEMO_MIXED_API_USE
    // Ptr<T> is safe ref-conting pointer class
    Ptr<IplImage> iplimg = cvLoadImage(imagename);
    
    // cv::Mat replaces the CvMat and IplImage, but it's easy to convert
    // between the old and the new data structures
    // (by default, only the header is converted and the data is shared)
    Mat img(iplimg); 
#else
    // the newer cvLoadImage alternative with MATLAB-style name
    Mat img = imread(imagename);
#endif

    if( !img.data ) // check if the image has been loaded properly
        return -1;

    Mat img_yuv;
    // convert image to YUV color space.
    // The output image will be allocated automatically
    cvtColor(img, img_yuv, CV_BGR2YCrCb); 

    // split the image into separate color planes
    vector<Mat> planes;
    split(img_yuv, planes);

    // another Mat constructor; allocates a matrix of the specified
	// size and type
    Mat noise(img.size(), CV_8U);
    
    // fills the matrix with normally distributed random values;
    // there is also randu() for uniformly distributed random numbers. 
    // Scalar replaces CvScalar, Scalar::all() replaces cvScalarAll().
    randn(noise, Scalar::all(128), Scalar::all(20));
                                                     
    // blur the noise a bit, kernel size is 3x3 and both sigma's 
	// are set to 0.5
    GaussianBlur(noise, noise, Size(3, 3), 0.5, 0.5);

    const double brightness_gain = 0;
    const double contrast_gain = 1.7;
#if DEMO_MIXED_API_USE
    // it's easy to pass the new matrices to the functions that
    // only work with IplImage or CvMat:
    // step 1) - convert the headers, data will not be copied
    IplImage cv_planes_0 = planes[0], cv_noise = noise;
    // step 2) call the function; do not forget unary "&" to form pointers
    cvAddWeighted(&cv_planes_0, contrast_gain, &cv_noise, 1,
                 -128 + brightness_gain, &cv_planes_0);
#else
    addWeighted(planes[0], constrast_gain, noise, 1,
                -128 + brightness_gain, planes[0]);
#endif
    const double color_scale = 0.5;
    // Mat::convertTo() replaces cvConvertScale.
    // One must explicitly specify the output matrix type
    // (we keep it intact, i.e. pass planes[1].type())
    planes[1].convertTo(planes[1], planes[1].type(),
                        color_scale, 128*(1-color_scale));

    // alternative form of convertTo if we know the datatype
    // at compile time ("uchar" here).
    // This expression will not create any temporary arrays
    // and should be almost as fast as the above variant
    planes[2] = Mat_<uchar>(planes[2]*color_scale + 128*(1-color_scale));

    // Mat::mul replaces cvMul(). Again, no temporary arrays are
    // created in the case of simple expressions.
    planes[0] = planes[0].mul(planes[0], 1./255);

    // now merge the results back
    merge(planes, img_yuv);
    // and produce the output RGB image
    cvtColor(img_yuv, img, CV_YCrCb2BGR);

    // this is counterpart for cvNamedWindow
    namedWindow("image with grain", CV_WINDOW_AUTOSIZE);
#if DEMO_MIXED_API_USE
    // this is to demonstrate that img and iplimg really share the data -
    // the result of the above processing is stored to img and thus 
	// in iplimg too.
    cvShowImage("image with grain", iplimg);
#else
    imshow("image with grain", img);
#endif
    waitKey();

    return 0;
    // all the memory will automatically be released
    // by vector<>, Mat and Ptr<> destructors.
}
\end{lstlisting}

Following a summary "cheatsheet" below, the rest of the introduction will discuss the key features of the new interface in more detail.



\section{C++ Cheatsheet}\label{cheatSheet}
 The section is just a summary "cheatsheet" of common things you may want to do with cv::Mat:. The code snippets below all assume the correct 
 namespace is used:
 \begin{lstlisting}
using namespace cv;
using namespace std; 
 \end{lstlisting} 
 
 
 Convert an IplImage or CvMat to an cv::Mat and a cv::Mat to an IplImage or CvMat:
 \begin{lstlisting}
 // Assuming somewhere IplImage *iplimg; exists 
 // and has been allocated and cv::Mat Mimg has been defined
 Mat imgMat(iplimg);  //Construct an Mat image "img" out of an IplImage
 Mimg = iplimg;       //Or just set the header of pre existing cv::Mat 
                      //Ming to iplimg's data (no copying is done)
 
 //Convert to IplImage or CvMat, no data copying
 IplImage ipl_img = img;
 CvMat cvmat = img; // convert cv::Mat -> CvMat
 \end{lstlisting} 
 
 A very simple way to operate on a rectanglular sub-region of an image (ROI -- "Region of Interest"):
\begin{lstlisting}
 //Make a rectangle 
 Rect roi(10, 20, 100, 50);
 //Point a cv::Mat header at it (no allocation is done)
 Mat image_roi = image(roi);
\end{lstlisting}

A bit advanced, but should you want efficiently to sample from a circular region in an image  
(below, instead of sampling, we just draw into a BGR image) :

\begin{lstlisting}
// the function returns x boundary coordinates of 
// the circle for each y. RxV[y1] = x1 means that 
// when y=y1, -x1 <=x<=x1 is inside the circle
void getCircularROI(int R, vector < int > & RxV)
{
    RxV.resize(R+1);
    for( int y = 0; y <= R; y++ )
        RxV[y] = cvRound(sqrt((double)R*R - y*y));
}

// This draws a circle in the green channel
// (note the "[1]" for a BGR" image,
// blue and red channels are not modified),
// but is really an example of how to *sample* from a circular region. 
void drawCircle(Mat &image, int R, Point center)
{
    vector<int> RxV;
    getCircularROI(R, RxV);
	
    Mat_<Vec3b>& img = (Mat_<Vec3b>&)image; //3 channel pointer to image
    for( int dy = -R; dy <= R; dy++ )
    {
        int Rx = RxV[abs(dy)];
        for( int dx = -Rx; dx <= Rx; dx++ )
            img(center.y+dy, center.x+dx)[1] = 255;
    }
}
\end{lstlisting}

\section{Namespace \texttt{cv} and Function Naming}

All the newly introduced classes and functions are placed into \texttt{cv} namespace. Therefore, to access this functionality from your code, use \texttt{cv::} specifier or \texttt{"using namespace cv;"} directive:
\begin{lstlisting}
#include "cv.h"

...
cv::Mat H = cv::findHomography(points1, points2, cv::RANSAC, 5);
...
\end{lstlisting}
or
\begin{lstlisting}
#include "cv.h"

using namespace cv;

...
Mat H = findHomography(points1, points2, RANSAC, 5 );
...
\end{lstlisting}

It is probable that some of the current or future OpenCV external names conflict with STL
or other libraries, in this case use explicit namespace specifiers to resolve the name conflicts:
\begin{lstlisting}
Mat a(100, 100, CV_32F);
randu(a, Scalar::all(1), Scalar::all(std::rand()%256+1));
cv::log(a, a);
a /= std::log(2.);
\end{lstlisting}

For the most of the C functions and structures from OpenCV 1.x you may find the direct counterparts in the new C++ interface. The name is usually formed by omitting \texttt{cv} or \texttt{Cv} prefix and turning the first letter to the low case (unless it's a own name, like Canny, Sobel etc). In case when there is no the new-style counterpart, it's possible to use the old functions with the new structures, as shown the first sample in the chapter.

\section{Memory Management}

When using the new interface, the most of memory deallocation and even memory allocation operations are done automatically when needed.

First of all, \cross{Mat}, \cross{SparseMat} and other classes have destructors
that deallocate memory buffers occupied by the structures when needed.

Secondly, this "when needed" means that the destructors do not always deallocate the buffers, they take into account possible data sharing.
That is, in a destructor the reference counter associated with the underlying data is decremented and the data is deallocated
if and only if the reference counter becomes zero, that is, when no other structures refer to the same buffer. When such a structure
containing a reference counter is copied, usually just the header is duplicated, while the underlying data is not; instead, the reference counter is incremented to memorize that there is another owner of the same data.
Also, some structures, such as \texttt{Mat}, can refer to the user-allocated data.
In this case the reference counter is \texttt{NULL} pointer and then no reference counting is done - the data is not deallocated by the destructors and should be deallocated manually by the user. We saw this scheme in the first example in the chapter:
\begin{lstlisting}
// allocates IplImages and wraps it into shared pointer class.
Ptr<IplImage> iplimg = cvLoadImage(...);

// constructs Mat header for IplImage data;
// does not copy the data;
// the reference counter will be NULL
Mat img(iplimg);
...
// in the end of the block img destructor is called,
// which does not try to deallocate the data because
// of NULL pointer to the reference counter.
//
// Then Ptr<IplImage> destructor is called that decrements
// the reference counter and, as the counter becomes 0 in this case,
// the destructor calls cvReleaseImage().
\end{lstlisting}

The copying semantics was mentioned in the above paragraph, but deserves a dedicated discussion.
By default, the new OpenCV structures implement shallow, so called O(1) (i.e. constant-time) assignment operations. It gives user possibility to pass quite big data structures to functions (though, e.g. passing \texttt{const Mat\&} is still faster than passing \texttt{Mat}), return them (e.g. see the example with \cross{findHomography} above), store them in OpenCV and STL containers etc. - and do all of this very efficiently. On the other hand, most of the new data structures provide \texttt{clone()} method that creates a full copy of an object. Here is the sample:
\begin{lstlisting}
// create a big 8Mb matrix
Mat A(1000, 1000, CV_64F);

// create another header for the same matrix;
// this is instant operation, regardless of the matrix size.
Mat B = A;
// create another header for the 3-rd row of A; no data is copied either
Mat C = B.row(3);
// now create a separate copy of the matrix
Mat D = B.clone();
// copy the 5-th row of B to C, that is, copy the 5-th row of A 
// to the 3-rd row of A.
B.row(5).copyTo(C);
// now let A and D share the data; after that the modified version
// of A is still referenced by B and C.
A = D;
// now make B an empty matrix (which references no memory buffers),
// but the modified version of A will still be referenced by C,
// despite that C is just a single row of the original A
B.release(); 
             
// finally, make a full copy of C. In result, the big modified
// matrix will be deallocated, since it's not referenced by anyone
C = C.clone();
\end{lstlisting}

Memory management of the new data structures is automatic and thus easy. If, however, your code uses \cross{IplImage},
\cross{CvMat} or other C data structures a lot, memory management can still be automated without immediate migration
to \cross{Mat} by using the already mentioned template class \cross{Ptr}, similar to \texttt{shared\_ptr} from Boost and C++ TR1.
It wraps a pointer to an arbitrary object, provides transparent access to all the object fields and associates a reference counter with it.
Instance of the class can be passed to any function that expects the original pointer. For correct deallocation of the object, you should specialize \texttt{Ptr<T>::delete\_obj()} method. Such specialized methods already exist for the classical OpenCV structures, e.g.:
\begin{lstlisting}
// cxoperations.hpp:
...
template<> inline Ptr<IplImage>::delete_obj() {
    cvReleaseImage(&obj);
}
...
\end{lstlisting}
See \cross{Ptr} description for more details and other usage scenarios.


\section{Memory Management Part II. Automatic Data Allocation}\label{AutomaticMemoryManagement2}

With the new interface not only explicit memory deallocation is not needed anymore,
but the memory allocation is often done automatically too. That was demonstrated in the example
in the beginning of the chapter when \texttt{cvtColor} was called, and here are some more details.

\cross{Mat} and other array classes provide method \texttt{create} that allocates a new buffer for array
data if and only if the currently allocated array is not of the required size and type.
If a new buffer is needed, the previously allocated buffer is released
(by engaging all the reference counting mechanism described in the previous section).
Now, since it is very quick to check whether the needed memory buffer is already allocated,
most new OpenCV functions that have arrays as output parameters call the \texttt{create} method and
this way the \emph{automatic data allocation} concept is implemented. Here is the example:
\begin{lstlisting}
#include "cv.h"
#include "highgui.h"

using namespace cv;

int main(int, char**)
{
    VideoCapture cap(0);
    if(!cap.isOpened()) return -1;

    Mat edges;
    namedWindow("edges",1);
    for(;;)
    {
        Mat frame;
        cap >> frame;
        cvtColor(frame, edges, CV_BGR2GRAY);
        GaussianBlur(edges, edges, Size(7,7), 1.5, 1.5);
        Canny(edges, edges, 0, 30, 3);
        imshow("edges", edges);
        if(waitKey(30) >= 0) break;
    }
    return 0;
}
\end{lstlisting}
The matrix \texttt{edges} is allocated during the first frame processing and unless the resolution will suddenly change,
the same buffer will be reused for every next frame's edge map.

In many cases the output array type and size can be inferenced from the input arrays' respective characteristics, but not always.
In these rare cases the corresponding functions take separate input parameters that specify the data type and/or size of the output arrays,
like \cross{resize}. Anyway, a vast majority of the new-style array processing functions call \texttt{create}
for each of the output array, with just a few exceptions like \texttt{mixChannels}, \texttt{RNG::fill} and some others.

Note that this output array allocation semantic is only implemented in the new functions. If you want to pass the new structures to some old OpenCV function, you should first allocate the output arrays using \texttt{create} method, then make \texttt{CvMat} or \texttt{IplImage} headers and after that call the function.

\section{Algebraic Operations}

Just like in v1.x, OpenCV 2.x provides some basic functions operating on matrices, like \texttt{add},
\texttt{subtract}, \texttt{gemm} etc. In addition, it introduces overloaded operators that give the user a convenient
algebraic notation, which is nearly as fast as using the functions directly. For example, here is how the least squares problem $Ax=b$
can be solved using normal equations:
\begin{lstlisting}
Mat x = (A.t()*A).inv()*(A.t()*b);
\end{lstlisting}

The complete list of overloaded operators can be found in \cross{Matrix Expressions}.

\section{Fast Element Access}

Historically, OpenCV provided many different ways to access image and matrix elements, and none of them was both fast and convenient.
With the new data structures, OpenCV 2.x introduces a few more alternatives, hopefully more convenient than before. For detailed description of the operations, please, check \cross{Mat} and \hyperref[MatT]{Mat\_} description. Here is part of the retro-photo-styling example rewritten (in simplified form) using the element access operations:

\begin{lstlisting}
...
// split the image into separate color planes
vector<Mat> planes;
split(img_yuv, planes);

// method 1. process Y plane using an iterator
MatIterator_<uchar> it = planes[0].begin<uchar>(),
                    it_end = planes[0].end<uchar>();
for(; it != it_end; ++it)
{
    double v = *it*1.7 + rand()%21-10;
    *it = saturate_cast<uchar>(v*v/255.);
}

// method 2. process the first chroma plane using pre-stored row pointer.
// method 3. process the second chroma plane using
//           individual element access operations
for( int y = 0; y < img_yuv.rows; y++ )
{
    uchar* Uptr = planes[1].ptr<uchar>(y);
    for( int x = 0; x < img_yuv.cols; x++ )
    {
        Uptr[x] = saturate_cast<uchar>((Uptr[x]-128)/2 + 128);
        uchar& Vxy = planes[2].at<uchar>(y, x);
        Vxy = saturate_cast<uchar>((Vxy-128)/2 + 128);
    }
}

merge(planes, img_yuv);
...
\end{lstlisting}


\section{Saturation Arithmetics}

In the above sample you may have noticed \hyperref[saturatecast]{saturate\_cast} operator, and that's how all the pixel processing is done in OpenCV. When a result of image operation is 8-bit image with pixel values ranging from 0 to 255, each output pixel value is clipped to this available range:

\[
I(x,y)=\min(\max(value, 0), 255)
\]

and the similar rules are applied to 8-bit signed and 16-bit signed and unsigned types. This "saturation" semantics (different from usual C language "wrapping" semantics, where lowest bits are taken, is implemented in every image processing function, from the simple \texttt{cv::add} to 
\texttt{cv::cvtColor}, \texttt{cv::resize}, \texttt{cv::filter2D} etc.
It is not a new feature of OpenCV v2.x, it was there from very beginning. In the new version this special \hyperref[saturatecast]{saturate\_cast} template operator is introduced to simplify implementation of this semantic in your own functions.


\section{Error handling}

The modern error handling mechanism in OpenCV uses exceptions, as opposite to the manual stack unrolling used in previous versions.

If you to check some conditions in your code and raise an error if they are not satisfied, use \hyperref[CV Assert]{CV\_Assert}() or \hyperref[cppfunc.error]{CV\_Error}().

\begin{lstlisting}
CV_Assert(mymat.type() == CV_32FC1);
...

if( scaleValue < 0 || scaleValue > 1000 )
   CV_Error(CV_StsOutOfRange, "The scale value is out of range");
\end{lstlisting}

There is also \hyperref[CV Assert]{CV\_DbgAssert} that yields no code in the Release configuration.

To handle the errors, use the standard exception handling mechanism:

\begin{lstlisting}
try
{
    ...
}
catch( cv::Exception& e )
{
    const char* err_msg = e.what();
    ...
}
\end{lstlisting}
 
instead of \hyperref[Exception]{cv::Exception} you can write \texttt{std::exception}, since the former is derived from the latter.

Then, obviously, to make it all work and do not worry about the object destruction, try not to use \texttt{IplImage*}, \texttt{CvMat*} and plain pointers in general. Use \hyperref[Mat]{cv::Mat}, \texttt{std::vector<>}, \hyperref[Ptr]{cv::Ptr} etc.

\section{Threading and Reenterability}

OpenCV uses OpenMP to run some time-consuming operations in parallel. Threading can be explicitly controlled by \cross{setNumThreads} function. Also, functions and "const" methods of the classes are generally re-enterable, that is, they can be called from different threads asynchronously.

\fi

\ifPy
\chapter{Introduction}

Starting with release 2.0, OpenCV has a new Python interface. This replaces the previous 
\href{http://opencv.willowgarage.com/wiki/SwigPythonInterface}{SWIG-based Python interface}.

Some highlights of the new bindings:

\begin{itemize}
\item{single import of all of OpenCV using \texttt{import cv}}
\item{OpenCV functions no longer have the "cv" prefix}
\item{simple types like CvRect and CvScalar use Python tuples}
\item{sharing of Image storage, so image transport between OpenCV and other systems (e.g. numpy and ROS) is very efficient}
\item{complete documentation for the Python functions}
\end{itemize}

This cookbook section contains a few illustrative examples of OpenCV Python code.

\section{Cookbook}

Here is a collection of code fragments demonstrating some features
of the OpenCV Python bindings.

\subsection{Convert an image}

\begin{lstlisting}
>>> import cv
>>> im = cv.LoadImageM("building.jpg")
>>> print type(im)
<type 'cv.cvmat'>
>>> cv.SaveImage("foo.png", im)
\end{lstlisting}

\subsection{Resize an image}

To resize an image in OpenCV, create a destination image of the appropriate size, then call \cross{Resize}.

\begin{lstlisting}
>>> import cv
>>> original = cv.LoadImageM("building.jpg")
>>> thumbnail = cv.CreateMat(original.rows / 10, original.cols / 10, cv.CV_8UC3)
>>> cv.Resize(original, thumbnail)
\end{lstlisting}

\subsection{Compute the Laplacian}

\begin{lstlisting}
>>> import cv
>>> im = cv.LoadImageM("building.jpg", 1)
>>> dst = cv.CreateImage(cv.GetSize(im), cv.IPL_DEPTH_16S, 3)
>>> laplace = cv.Laplace(im, dst)
>>> cv.SaveImage("foo-laplace.png", dst)
\end{lstlisting}


\subsection{Using GoodFeaturesToTrack}

To find the 10 strongest corner features in an image, use \cross{GoodFeaturesToTrack} like this:

\begin{lstlisting}
>>> import cv
>>> img = cv.LoadImageM("building.jpg", cv.CV_LOAD_IMAGE_GRAYSCALE)
>>> eig_image = cv.CreateMat(img.rows, img.cols, cv.CV_32FC1)
>>> temp_image = cv.CreateMat(img.rows, img.cols, cv.CV_32FC1)
>>> for (x,y) in cv.GoodFeaturesToTrack(img, eig_image, temp_image, 10, 0.04, 1.0, useHarris = True):
...    print "good feature at", x,y
good feature at 198.0 514.0
good feature at 791.0 260.0
good feature at 370.0 467.0
good feature at 374.0 469.0
good feature at 490.0 520.0
good feature at 262.0 278.0
good feature at 781.0 134.0
good feature at 3.0 247.0
good feature at 667.0 321.0
good feature at 764.0 304.0
\end{lstlisting}


\subsection{Using GetSubRect}

GetSubRect returns a rectangular part of another image.  It does this without copying any data.

\begin{lstlisting}
>>> import cv
>>> img = cv.LoadImageM("building.jpg")
>>> sub = cv.GetSubRect(img, (60, 70, 32, 32))  # sub is 32x32 patch within img
>>> cv.SetZero(sub)                             # clear sub to zero, which also clears 32x32 pixels in img
\end{lstlisting}

\subsection{Using CreateMat, and accessing an element}

\begin{lstlisting}
>>> import cv
>>> mat = cv.CreateMat(5, 5, cv.CV_32FC1)
>>> cv.Set(mat, 1.0)
>>> mat[3,1] += 0.375
>>> print mat[3,1]
1.375
>>> print [mat[3,i] for i in range(5)]
[1.0, 1.375, 1.0, 1.0, 1.0]
\end{lstlisting}

\subsection{ROS image message to OpenCV}

See this tutorial: \href{http://www.ros.org/wiki/cv\_bridge/Tutorials/UsingCvBridgeToConvertBetweenROSImagesAndOpenCVImages}{Using CvBridge to convert between ROS images And OpenCV images}.

\subsection{PIL Image to OpenCV}

(For details on PIL see the \href{http://www.pythonware.com/library/pil/handbook/image.htm}{PIL handbook}.)

\begin{lstlisting}
>>> import Image, cv
>>> pi = Image.open('building.jpg')       # PIL image
>>> cv_im = cv.CreateImageHeader(pi.size, cv.IPL_DEPTH_8U, 3)
>>> cv.SetData(cv_im, pi.tostring())
>>> print pi.size, cv.GetSize(cv_im)
(868, 600) (868, 600)
>>> print pi.tostring() == cv_im.tostring()
True
\end{lstlisting}

\subsection{OpenCV to PIL Image}

\begin{lstlisting}
>>> import Image, cv
>>> cv_im = cv.CreateImage((320,200), cv.IPL_DEPTH_8U, 1)
>>> pi = Image.fromstring("L", cv.GetSize(cv_im), cv_im.tostring())
>>> print pi.size
(320, 200)
\end{lstlisting}

\subsection{NumPy and OpenCV}

Using the \href{http://docs.scipy.org/doc/numpy/reference/arrays.interface.html}{array interface}, to use an OpenCV CvMat in NumPy:

\begin{lstlisting}
>>> import cv, numpy
>>> mat = cv.CreateMat(3, 5, cv.CV_32FC1)
>>> cv.Set(mat, 7)
>>> a = numpy.asarray(mat)
>>> print a
[[ 7.  7.  7.  7.  7.]
 [ 7.  7.  7.  7.  7.]
 [ 7.  7.  7.  7.  7.]]
\end{lstlisting}

and to use a NumPy array in OpenCV:

\begin{lstlisting}
>>> import cv, numpy
>>> a = numpy.ones((480, 640))
>>> mat = cv.fromarray(a)
>>> print mat.rows
480
>>> print mat.cols
640
\end{lstlisting}

also, most OpenCV functions can work on NumPy arrays directly, for example:

\begin{lstlisting}
>>> picture = numpy.ones((640, 480))
>>> cv.Smooth(picture, picture, cv.CV_GAUSSIAN, 15, 15)
\end{lstlisting}

Given a 2D array, 
the \cross{fromarray} function (or the implicit version shown above)
returns a single-channel \cross{CvMat} of the same size.
For a 3D array of size $j \times k \times l$, it returns a 
\cross{CvMat} sized $j \times k$ with $l$ channels.

Alternatively, use \cross{fromarray} with the \texttt{allowND} option to always return a \cross{cvMatND}.

\fi