OpenCV 匹配兴趣点：SIFT、SURF 和二值描述子

上一篇文章中讲到如何检测图像中的兴趣点，以便后续的局部图像分析。为了进行基于兴趣点的图像分析，我们需要构建多种表征方式，精确地描述每个关键点。这些描述子通常是二值类型、整数型或浮点数型组成的向量。好的描述子要具有足够的独特性和鲁棒性，能唯一地表示图像中的每个特征点，并且在亮度和视角变化时仍能提取出同一批点集。此外，尽量能够简洁，以减少计算资源的占用。

SIFT算法提出者 David Lowe

1.Harris、FAST

这两个特征检测算子不具有尺度不变等特性，所以使用这两个算子进行检测并匹配的效果一般不会很好。常见的方案是通过比较特征点附近的一个方块的像素集合的相似度，算法使用差的平方和（SSD），效果如下，这里不进行代码演示。可以看到，即使在视角差别不大的情况下，就已经有非常多的错误匹配项。

FAST & SSD

2.SIFT、SURF

so，尺度不变检测算子的优势就体现出来了。SIFT、SURF在检测出特征点之后，可以生成相应的描述子（Descriptor）。这些描述子具有的信息量比单纯地比较像素块的SSD多得多，于是能够更好地进行图像的匹配。至于描述子的数据结构，上一篇文章中提到过，这里不再赘述。其中SIFT是128维的向量，SURF则是检测Haar小波特征。

SIFT SURF

直接进行匹配的话，也会有很多的错误匹配项，比上面那两位好不到哪里去。那么，算法研究员们想出了一些匹配策略，能够在一定程度上减少错误项。主要有：

• 交叉检查匹配项

交叉检查是指在第一幅图像匹配到第二幅图像后，再用第二幅图像的关键点再逐个跟第一幅的图像进行比较，只有在两个方向都匹配了同一个关键点时，才认为是一个有效的匹配项。

• 比率检测法

我们为每个关键点找到两个最佳的匹配项，方法是使用kNN最近邻（可以看我的这篇文章，其实在这里只是用了欧氏距离）。接下来计算排名第二的匹配项与排名第一的匹配项的差值之比（如果两个匹配项近乎相等，则结果接近为1）。比率过高的匹配项作为模糊匹配项，从结果中被排除。

• 匹配差值的阈值化

很简单，就是将差值过大的匹配项排除掉。

上面的一些匹配策略可以结合使用来提升匹配效果。代码实现如下

/******************************************************
 * Created by 杨帮杰 on 10/5/18
 * Right to use this code in any way you want without
 * warranty, support or any guarantee of it working
 * E-mail: yangbangjie1998@qq.com
 * Association: SCAU 华南农业大学
 ******************************************************/

#include <iostream>
#include <vector>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/objdetect.hpp>
#include <opencv2/xfeatures2d.hpp>

#define IMAGE1_PATH "/home/jacob/下载/church01.jpg"
#define IMAGE2_PATH "/home/jacob/下载/church02.jpg"
#define IMAGE3_PATH "/home/jacob/下载/church03.jpg"

using namespace cv;
using namespace std;

int main()
{

    /*******************SIFT、SURF：描述并匹配局部强度值模式***********************/

    Mat image1= imread(IMAGE1_PATH,IMREAD_GRAYSCALE);
    Mat image2= imread(IMAGE2_PATH,IMREAD_GRAYSCALE);

    vector<KeyPoint> keypoints1;
    vector<KeyPoint> keypoints2;

    //创建SURF特征检测器
    Ptr<Feature2D> ptrFeature2D = xfeatures2d::SURF::create(2000.0);
    //创建SIFT特征检测器
    //Ptr<Feature2D> ptrFeature2D = xfeatures2d::SIFT::create(74);

    //检测特征点
    ptrFeature2D->detect(image1,keypoints1);
    ptrFeature2D->detect(image2,keypoints2);

    Mat featureImage;
    drawKeypoints(image1,keypoints1,featureImage,
                  Scalar(255,255,255),DrawMatchesFlags::DRAW_RICH_KEYPOINTS);

    imshow("SURF",featureImage);

    cout << "Number of SURF keypoints (image 1): " << keypoints1.size() << endl;
    cout << "Number of SURF keypoints (image 2): " << keypoints2.size() << endl;

    //提取特征描述子
    Mat descriptors1;
    Mat descriptors2;
    ptrFeature2D->compute(image1,keypoints1,descriptors1);
    ptrFeature2D->compute(image2,keypoints2,descriptors2);

    //使用L2范式（欧氏距离）进行配对
    BFMatcher matcher(NORM_L2);
    //进行交叉匹配
    //BFMatcher matcher(NORM_L2, true);

    vector<DMatch> matches;
    matcher.match(descriptors1,descriptors2, matches);

    Mat imageMatches;
    drawMatches(
    image1, keypoints1, // 1st image and its keypoints
    image2, keypoints2, // 2nd image and its keypoints
    matches,            // the matches
    imageMatches,       // the image produced
    Scalar(255, 255, 255),  // color of lines
    Scalar(255, 255, 255),  // color of points
    vector< char >(),      // masks if any
    DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS | DrawMatchesFlags::DRAW_RICH_KEYPOINTS);

    imshow("SURF Matches",imageMatches);

    cout << "Number of matches: " << matches.size() << endl;

    //使用比率检测法
    //为每个关键点找出两个最佳匹配项
    vector<vector<DMatch> > matches2;
    matcher.knnMatch(descriptors1, descriptors2,
                     matches2,
                     2); // find the k (2) best matches
    matches.clear();

    //比率设定为0.6
    double ratioMax= 0.6;
    vector<vector<DMatch> >::iterator it;
    for (it= matches2.begin(); it!= matches2.end(); ++it) {
        //   first best match/second best match
        if ((*it)[0].distance/(*it)[1].distance < ratioMax) {
            matches.push_back((*it)[0]);
        }
    }

     drawMatches(
     image1,keypoints1, // 1st image and its keypoints
     image2,keypoints2, // 2nd image and its keypoints
     matches,           // the matches
     imageMatches,      // the image produced
     Scalar(255,255,255),  // color of lines
     Scalar(255,255,255),  // color of points
     vector< char >(),    // masks if any
     DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS | DrawMatchesFlags::DRAW_RICH_KEYPOINTS);

    cout << "Number of matches (after ratio test): " << matches.size() << endl;

    imshow("SURF Matches (ratio test at 0.6)",imageMatches);

    //差值阈值化匹配，这里设为0.3
    float maxDist = 0.3;
    matches2.clear();
    matcher.radiusMatch(descriptors1, descriptors2, matches2,
                        maxDist); // maximum acceptable distance

    drawMatches(
    image1, keypoints1, // 1st image and its keypoints
    image2, keypoints2, // 2nd image and its keypoints
    matches2,          // the matches
    imageMatches,      // the image produced
    Scalar(255, 255, 255),  // color of lines
    Scalar(255, 255, 255),  // color of points
    vector<vector< char >>(),    // masks if any
    DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS | DrawMatchesFlags::DRAW_RICH_KEYPOINTS);

    int nmatches = 0;
    for (int i = 0; i< matches2.size(); i++)
        nmatches += matches2[i].size();

    cout << "Number of matches (with max radius): " << nmatches << endl;

    imshow("SURF Matches (with max radius)", imageMatches);

    /****************************尺度无关的匹配**************************************/

    image1= imread(IMAGE1_PATH,CV_LOAD_IMAGE_GRAYSCALE);
    image2= imread(IMAGE3_PATH,CV_LOAD_IMAGE_GRAYSCALE);

    cout << "Number of SIFT keypoints (image 1): " << keypoints1.size() << endl;
    cout << "Number of SIFT keypoints (image 2): " << keypoints2.size() << endl;

    ptrFeature2D = xfeatures2d::SIFT::create();
    ptrFeature2D->detectAndCompute(image1, noArray(), keypoints1, descriptors1);
    ptrFeature2D->detectAndCompute(image2, noArray(), keypoints2, descriptors2);

    matcher.match(descriptors1,descriptors2, matches);

    //选取最好的50个
    nth_element(matches.begin(),matches.begin()+50,matches.end());
    matches.erase(matches.begin()+50,matches.end());

    drawMatches(
    image1, keypoints1, // 1st image and its keypoints
    image2, keypoints2, // 2nd image and its keypoints
    matches,            // the matches
    imageMatches,      // the image produced
    Scalar(255, 255, 255),  // color of lines
    Scalar(255, 255, 255), // color of points
    vector<char>(), cv::DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS| cv::DrawMatchesFlags::DRAW_RICH_KEYPOINTS);

    imshow("Multi-scale SIFT Matches",imageMatches);

    cout << "Number of matches: " << matches.size() << endl;

    waitKey();
    return 0;
}

结果如下

SURF和不同尺度下的SIFT匹配
SURF的比率检测和阈值化

3.ORB、BRISK

SIFT和SURF的描述子向量分别是浮点型的128位和64位，对他们操作耗资巨大，为了减少计算资源的使用，算法研究员们又引入了二值描述子的概念。其中ORB和BRISK生成的就是二值描述子。

其中，ORB实际上是在BRIEF描述子基础上构建的。实现过程是在关键点周围的邻域内随机选取一对像素点，从而创建一个二值描述子。比较这两个像素点的强度值，如果第一个点的强度值较大，就把对应描述子的位（bit）设为1，否则为0。对一批随机像素点进行上述处理，就产生了一个由若干位组成的描述子，通常在128到512位。对于ORB，为了解决旋转不变性，对256个随机点对进行旋转后进行判别。

ORB

二值描述子之间的比较一般使用Hamming Distance（汉明距离），表示的是两个等长子串或者二进制数之间不同位的个数，如

# 举例说明以下字符串间的汉明距离为：
"karolin" and "kathrin" is 3.
"karolin" and "kerstin" is 3.
1011101 and 1001001 is 2.
2173896 and 2233796 is 3.

代码实现如下

/******************************************************
 * Created by 杨帮杰 on 10/5/18
 * Right to use this code in any way you want without
 * warranty, support or any guarantee of it working
 * E-mail: yangbangjie1998@qq.com
 * Association: SCAU 华南农业大学
 ******************************************************/

#include <iostream>
#include <vector>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/objdetect.hpp>
#include <opencv2/xfeatures2d.hpp>

#define IMAGE1_PATH "/home/jacob/下载/church01.jpg"
#define IMAGE2_PATH "/home/jacob/下载/church02.jpg"
#define IMAGE3_PATH "/home/jacob/下载/church03.jpg"

using namespace cv;
using namespace std;

int main()
{

    Mat image1= imread(IMAGE1_PATH,CV_LOAD_IMAGE_GRAYSCALE);
    Mat image2= imread(IMAGE2_PATH,CV_LOAD_IMAGE_GRAYSCALE);

    vector<KeyPoint> keypoints1;
    vector<KeyPoint> keypoints2;
    Mat descriptors1;
    Mat descriptors2;

    //构建ORB特征检测器
    //Ptr<Feature2D> feature =ORB::create(60);
    //构建BRISK特征检测器
    Ptr<Feature2D> feature = BRISK::create(80);

    feature->detectAndCompute(image1, noArray(), keypoints1, descriptors1);
    feature->detectAndCompute(image2, noArray(), keypoints2, descriptors2);

    Mat featureImage;
    drawKeypoints(image1,keypoints1,featureImage,
                  Scalar(255,255,255),DrawMatchesFlags::DRAW_RICH_KEYPOINTS);

    imshow("ORB",featureImage);

    cout << "Number of ORB keypoints (image 1): " << keypoints1.size() << endl;
    cout << "Number of ORB keypoints (image 2): " << keypoints2.size() << endl;

    // 使用FREAK（快速视网膜关键点），配合BRISK
    // feature = xfeatures2d::FREAK::create();
    // feature->compute(image1, keypoints1, descriptors1);
    // feature->compute(image1, keypoints2, descriptors2);

    //二值描述子必须用Hamming规范
    BFMatcher matcher(NORM_HAMMING);

    vector<DMatch> matches;
    matcher.match(descriptors1,descriptors2, matches);

    Mat imageMatches;
    drawMatches(
    image1,keypoints1, // 1st image and its keypoints
    image2,keypoints2, // 2nd image and its keypoints
    matches,           // the matches
    imageMatches,      // the image produced
    Scalar(255,255,255),  // color of lines
    Scalar(255,255,255),  // color of points
    vector< char >(),    // masks if any
    DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS | DrawMatchesFlags::DRAW_RICH_KEYPOINTS);

    imshow("ORB Matches", imageMatches);
    //imshow("BRISK Matches", imageMatches);
    //imshow("FREAK with BRISK Matches", imageMatches);

    cout << "Number of matches: " << matches.size() << endl;

    waitKey();
    return 0;
}

结果如下

ORB Matches
BRISK Matches

References:
OpenCV尺度不变特征检测：SIFT、SURF、BRISK、ORB
Hamming Distance (汉明距离)
【特征检测】ORB特征提取算法
opencv计算机视觉编程攻略（第三版） —— Robert