Fawkes API Fawkes Development Version
sift.cpp
1
2/***************************************************************************
3 * sift.cpp - Feature-based classifier using OpenCV structures
4 *
5 * Created: Mon Mar 15 15:47:11 2008
6 * Copyright 2008 Stefan Schiffer [stefanschiffer.de]
7 *
8 ****************************************************************************/
9
10/* This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version. A runtime exception applies to
14 * this software (see LICENSE.GPL_WRE file mentioned below for details).
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU Library General Public License for more details.
20 *
21 * Read the full text in the LICENSE.GPL_WRE file in the doc directory.
22 */
23
24#include <fvclassifiers/sift.h>
25
26#include <iostream>
27#include <vector>
28//#ifdef SIFT_TIMETRACKER
29#include <utils/time/clock.h>
30#include <utils/time/tracker.h>
31//#endif
32
33extern "C" {
34#include <sift/imgfeatures.h>
35#include <sift/kdtree.h>
36#include <sift/sift.h>
37#include <sift/utils.h>
38#include <sift/xform.h>
39}
40
41#include <core/exception.h>
42#include <core/exceptions/software.h>
43#include <fvutils/color/colorspaces.h>
44#include <fvutils/color/conversions.h>
45#include <opencv/cv.h>
46#include <opencv/cxcore.h>
47#include <opencv/highgui.h>
48
49using namespace fawkes;
50
51namespace firevision {
52
53/** @class SiftClassifier <fvclassifiers/sift.h>
54 * SIFT classifier.
55 *
56 * This class provides a classifier that uses OpenCV to detect objects in a given
57 * image by matching features using SIFT. The objects are reported back as regions
58 * of interest. Each ROI contains an object.
59 *
60 * This code is based on the sift package provided by Rob Hess.
61 * at http://web.engr.oregonstate.edu/~hess/
62 *
63 * @author Stefan Schiffer
64 */
65
66/** Constructor.
67 * @param object_file file that contains the object to detect
68 * @param pixel_width width of images that will be processed
69 * @param pixel_height height of images that will be processed
70 * @param kdtree_bbf_max_nn_chks maximum number of keypoint NN candidates to check during BBF search
71 * @param nn_sq_dist_ratio_thr threshold on squared ratio of distances between NN and 2nd NN
72 * @param flags flags, not used yet.
73 */
74SiftClassifier::SiftClassifier(const char * object_file,
75 unsigned int pixel_width,
76 unsigned int pixel_height,
77 int kdtree_bbf_max_nn_chks,
78 float nn_sq_dist_ratio_thr,
79 int flags)
80: Classifier("SiftClassifier")
81{
82 kdtree_bbf_max_nn_chks_ = kdtree_bbf_max_nn_chks;
83 nn_sq_dist_ratio_thr_ = nn_sq_dist_ratio_thr;
84 flags_ = flags;
85
86 //#ifdef SIFT_TIMETRACKER
87 tt_ = new TimeTracker();
88 loop_count_ = 0;
89 ttc_objconv_ = tt_->add_class("ObjectConvert");
90 ttc_objfeat_ = tt_->add_class("ObjectFeatures");
91 ttc_imgconv_ = tt_->add_class("ImageConvert");
92 ttc_imgfeat_ = tt_->add_class("ImageFeatures");
93 ttc_matchin_ = tt_->add_class("Matching");
94 ttc_roimerg_ = tt_->add_class("MergeROIs");
95 //#endif
96
97 //#ifdef SIFT_TIMETRACKER
98 tt_->ping_start(ttc_objconv_);
99 //#endif
100 obj_img_ = cvLoadImage(object_file, 1);
101 if (!obj_img_) {
102 throw Exception("Could not load object file");
103 }
104 //#ifdef SIFT_TIMETRACKER
105 tt_->ping_end(ttc_objconv_);
106 //#endif
107
108 //#ifdef SIFT_TIMETRACKER
109 tt_->ping_start(ttc_objfeat_);
110 //#endif
111 obj_num_features_ = 0;
112 obj_num_features_ = sift_features(obj_img_, &obj_features_);
113 if (!obj_num_features_ > 0) {
114 throw Exception("Could not compute object features");
115 }
116 std::cout << "SiftClassifier(classify): computed '" << obj_num_features_
117 << "' features from object" << std::endl;
118 //cvReleaseImage(&obj_img_);
119 //#ifdef SIFT_TIMETRACKER
120 tt_->ping_end(ttc_objfeat_);
121 //#endif
122
123 // create space for OpenCV image
124 image_ = cvCreateImage(cvSize(pixel_width, pixel_height), IPL_DEPTH_8U, 3);
125}
126
127/** Destructor. */
129{
130 //
131 cvReleaseImage(&obj_img_);
132 cvReleaseImage(&image_);
133}
134
135std::list<ROI> *
137{
138 //#ifdef SIFT_TIMETRACKER
139 tt_->ping_start(0);
140 //#endif
141
142 // list of ROIs to return
143 std::list<ROI> *rv = new std::list<ROI>();
144
145 struct feature * feat;
146 struct feature **nbrs;
147 struct kd_node * kd_root;
148 CvPoint pt1, pt2;
149
150 // for ROI calculation
151 CvPoint ftpt;
152 std::vector<CvPoint> ftlist;
153 //= new std::vector< CvPoint >();
154 int x_min = _width;
155 int y_min = _height;
156 int x_max = 0;
157 int y_max = 0;
158
159 double d0, d1; // = 0.0;
160 int k, m = 0;
161
162 //#ifdef SIFT_TIMETRACKER
163 tt_->ping_start(ttc_imgconv_);
164 //#endif
165 //std::cout << "SiftClassifier(classify): convert frame to IplImage" << std::endl;
166 convert(YUV422_PLANAR, BGR, _src, (unsigned char *)image_->imageData, _width, _height);
167 //#ifdef SIFT_TIMETRACKER
168 tt_->ping_end(ttc_imgconv_);
169 //#endif
170
171 //#ifdef SIFT_TIMETRACKER
172 tt_->ping_start(ttc_imgfeat_);
173 //#endif
174 //std::cout << "SiftClassifier(classify): compute features on current frame " << std::endl;
175 int num_img_ft = sift_features(image_, &img_features_);
176 kd_root = kdtree_build(img_features_, num_img_ft);
177 //#ifdef SIFT_TIMETRACKER
178 tt_->ping_end(ttc_imgfeat_);
179 //#endif
180
181 if (!kd_root) {
182 std::cerr << "SiftClassifier(classify): KD-Root NULL!" << std::endl;
183 }
184
185 //#ifdef SIFT_TIMETRACKER
186 tt_->ping_start(ttc_matchin_);
187 //#endif
188 std::cout << "SiftClassifier(classify): matching ..." << std::endl;
189 for (int i = 0; i < obj_num_features_; ++i) {
190 //std::cout << "SiftClassifier(classify): ... feature '" << i << "'" << std::endl;
191 feat = obj_features_ + i;
192 k = kdtree_bbf_knn(kd_root, feat, 2, &nbrs, kdtree_bbf_max_nn_chks_);
193 if (k == 2) {
194 d0 = descr_dist_sq(feat, nbrs[0]);
195 d1 = descr_dist_sq(feat, nbrs[1]);
196 if (d0 < d1 * nn_sq_dist_ratio_thr_) {
197 pt1 = cvPoint(cvRound(feat->x), cvRound(feat->y));
198 pt2 = cvPoint(cvRound(nbrs[0]->x), cvRound(nbrs[0]->y));
199 m++;
200 obj_features_[i].fwd_match = nbrs[0];
201 // save matched feature points
202 ftpt = cvPoint(cvRound(nbrs[0]->x), cvRound(nbrs[0]->y));
203 ftlist.push_back(ftpt);
204 // save matched features as ROIs
205 ROI r(pt2.x - 5, pt2.y - 5, 11, 11, _width, _height);
206 rv->push_back(r);
207 }
208 }
209 free(nbrs);
210 }
211 std::cout << "SiftClassifier(classify): found '" << m << "' matches" << std::endl;
212 kdtree_release(kd_root);
213 //#ifdef SIFT_TIMETRACKER
214 tt_->ping_end(ttc_matchin_);
215 //#endif
216
217 //#ifdef SIFT_TIMETRACKER
218 tt_->ping_start(ttc_roimerg_);
219 //#endif
220 std::cout << "SiftClassifier(classify): computing ROI" << std::endl;
221 //for ( int i = 0; i < m; ++i) {
222 for (std::vector<CvPoint>::size_type i = 0; i < ftlist.size(); ++i) {
223 if (ftlist[i].x < x_min)
224 x_min = ftlist[i].x;
225 if (ftlist[i].y < y_min)
226 y_min = ftlist[i].y;
227 if (ftlist[i].x > x_max)
228 x_max = ftlist[i].x;
229 if (ftlist[i].y > y_max)
230 y_max = ftlist[i].y;
231 }
232 if (m != 0) {
233 ROI r(x_min, y_min, x_max - x_min, y_max - y_min, _width, _height);
234 rv->push_back(r);
235 }
236 //#ifdef SIFT_TIMETRACKER
237 tt_->ping_end(ttc_roimerg_);
238 //#endif
239
240 //#ifdef SIFT_TIMETRACKER
241 tt_->ping_end(0);
242 //#endif
243
244 //#ifdef SIFT_TIMETRACKER
245 tt_->print_to_stdout();
246 //#endif
247
248 std::cout << "SiftClassifier(classify): done ... returning '" << rv->size() << "' ROIs."
249 << std::endl;
250 return rv;
251}
252
253} // end namespace firevision
Base class for exceptions in Fawkes.
Definition: exception.h:36
Time tracking utility.
Definition: tracker.h:37
void print_to_stdout()
Print results to stdout.
Definition: tracker.cpp:307
unsigned int add_class(std::string name)
Add a new class.
Definition: tracker.cpp:149
void ping_end(unsigned int cls)
End of given class task.
Definition: tracker.cpp:243
void ping_start(unsigned int cls)
Start of given class task.
Definition: tracker.cpp:218
Classifier to extract regions of interest.
Definition: classifier.h:36
unsigned int _height
Height in pixels of _src buffer.
Definition: classifier.h:53
unsigned char * _src
Source buffer, encoded as YUV422_PLANAR.
Definition: classifier.h:49
unsigned int _width
Width in pixels of _src buffer.
Definition: classifier.h:51
Region of interest.
Definition: roi.h:55
SiftClassifier(const char *features_file, unsigned int pixel_width, unsigned int pixel_height, int kdtree_bbf_max_nn_chks=200, float nn_sq_dist_ratio_thr=0.49, int flags=0)
Constructor.
Definition: sift.cpp:74
virtual ~SiftClassifier()
Destructor.
Definition: sift.cpp:128
virtual std::list< ROI > * classify()
Classify image.
Definition: sift.cpp:136
Fawkes library namespace.