gaze  0.1.0
Perform gaze tracking with common webcams.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Groups Pages
pupil_localization.cpp
1 #include "gaze/pipeline_steps/pupil_localization.h"
2 
3 #include <algorithm>
4 #include <limits>
5 #include <string>
6 #include <vector>
7 
8 #include "dlib/image_processing.h"
9 #include "dlib/image_transforms.h"
10 #include "dlib/matrix.h"
11 #include "dlib/opencv.h"
12 #include "yaml-cpp/yaml.h"
13 
14 #include "gaze/util/config.h"
15 #include "gaze/util/data.h"
16 #include "gaze/util/pipeline_utils.h"
17 
18 
19 namespace gaze {
20 
21 namespace util {
22 
24  dlib::matrix<double>& table_x, // NOLINT
25  dlib::matrix<double>& table_y, // NOLINT
26  int size) {
27  // Ensure size is bigger than the current size (i.e. if an update is needed
28  // at all)
29  if (size <= table_x.nr()) {
30  return;
31  }
32 
33  // Ensure size is odd
34  size += (1 - size & 1);
35  int half_size = (size - 1) / 2;
36 
37  // Resize tables
38  table_x.set_size(size, size);
39  table_y.set_size(size, size);
40 
41  // Fill with indices
42  for (int row = 0; row < table_x.nr(); ++row) {
43  dlib::set_rowm(table_x, row) = dlib::range(-half_size, half_size);
44  }
45  table_y = dlib::trans(table_x);
46 
47  // Calculate norms and store to table_x, then transpose to table_y
48  const double epsilon = std::numeric_limits<double>::min();
49  for (int row = 0; row < table_x.nr(); ++row) {
50  for (int col = 0; col < table_x.nc(); ++col) {
51  table_x(row, col) = table_x(row, col) /
52  (std::sqrt(table_x(row, col) * table_x(row, col)
53  + table_y(row, col) * table_y(row, col))
54  // Avoid div0
55  + epsilon);
56  }
57  }
58  table_y = dlib::trans(table_x);
59 }
60 
61 } // namespace util
62 
63 namespace pipeline {
64 
65 PupilLocalization::PupilLocalization()
66  // See
67  // http://thume.ca/projects/2012/11/04/simple-accurate-eye-center-tracking-in-opencv/
68  // for parameters
69  : relative_threshold_factor(0.3),
70  sigma(1.1) {
71  YAML::Node config = util::get_config(this->number);
72  this->name = config["name"] ?
73  config["name"].as<std::string>() : "PupilLocalization";
74 
75  if (config["relative_threshold"]) {
76  this->relative_threshold_factor =
77  config["relative_threshold"]
78  .as<decltype(this->relative_threshold_factor)>();
79  }
80 
81  // Pre-calculate displacement table
83  this->displacement_table_x,
84  this->displacement_table_y,
85  config["table_size"] ? config["table_size"].as<int>() : 131);
86 }
87 
88 void PupilLocalization::process(util::Data& data) {
89  if (data.landmarks.num_parts() < 5) {
90  return;
91  }
92 
93  // Extract eye patches
96  dlib::extract_image_chips(data.image, details, data.eyes);
97 
98  // Resize lookup table if needed
99  int max_size = std::max({data.eyes[0].nr(), data.eyes[0].nc(),
100  data.eyes[1].nr(), data.eyes[1].nc()});
101  if (2 * max_size > this->displacement_table_x.nr()) {
103  this->displacement_table_x,
104  this->displacement_table_y,
105  2 * max_size + 7);
106  }
107 
108  for (int i = 0; i < 2; ++i) {
109  dlib::matrix<double> eye_in;
110  dlib::assign_image(eye_in, data.eyes[i]);
111 
112  // Calculate gradients
113  dlib::matrix<double> eye_horizontal;
114  dlib::matrix<double> eye_vertical;
115  dlib::sobel_edge_detector(eye_in, eye_horizontal, eye_vertical);
116 
117  util::normalize_and_threshold_gradients(eye_horizontal, eye_vertical,
118  this->relative_threshold_factor);
119 
120  // Blur eyes as weights
121  int boundary_offset = 6;
122  dlib::matrix<double> eye_gaussian;
123  dlib::gaussian_blur(eye_in, eye_gaussian, this->sigma, boundary_offset);
124  eye_gaussian = 255 - eye_gaussian;
125 
126  // Compute objective function t and select highest value
127  double max_t = std::numeric_limits<double>::min();
128  int argmax_r;
129  int argmax_c;
130 
131  int center = (this->displacement_table_x.nr() - 1) / 2;
132  auto nr = eye_in.nr();
133  auto nc = eye_in.nc();
134  dlib::matrix<double> zero = dlib::zeros_matrix<double>(nr, nc);
135  dlib::matrix<double> ts = dlib::zeros_matrix<double>(nr, nc);
136  for (auto row = decltype(nr){boundary_offset};
137  row < nr - boundary_offset; ++row) {
138  for (auto col = decltype(nc){boundary_offset};
139  col < nc - boundary_offset; ++col) {
140  dlib::matrix<double> d_x = dlib::subm(this->displacement_table_x,
141  center - row, center - col, nr, nc);
142  dlib::matrix<double> d_y = dlib::subm(this->displacement_table_y,
143  center - row, center - col, nr, nc);
144 
145  double t =
146  dlib::sum(
147  dlib::squared(dlib::max_pointwise(
148  dlib::pointwise_multiply(d_x, eye_horizontal) +
149  dlib::pointwise_multiply(d_y, eye_vertical), zero))) *
150  eye_gaussian(row, col);
151  if (t > max_t) {
152  max_t = t;
153  argmax_r = row;
154  argmax_c = col;
155  }
156  }
157  }
158  data.centers[i] = dlib::point(argmax_c, argmax_r);
159  }
160 }
161 
162 void PupilLocalization::visualize(util::Data& data) {
163  if (data.landmarks.num_parts() < 5) {
164  return;
165  }
166  // Ensures output of this pipeline step is visualized
167  this->process(data);
168 
169  // define offsets
170  dlib::point offset_x(5, 0);
171  dlib::point offset_y(0, 5);
174 
175  // assign eyes to new images
176  dlib::array<dlib::array2d<dlib::bgr_pixel>> eyes(3);
177  dlib::assign_image(eyes[2], data.image);
178  for (auto i = decltype(data.eyes.size()){0};
179  i < data.eyes.size(); ++i) {
180  dlib::assign_image(eyes[i], data.eyes[i]);
181 
182  dlib::draw_line(eyes[i],
183  data.centers[i] + offset_x,
184  data.centers[i] - offset_x,
185  dlib::rgb_pixel(255, 0, 0));
186  dlib::draw_line(eyes[i],
187  data.centers[i] + offset_y,
188  data.centers[i] - offset_y,
189  dlib::rgb_pixel(255, 0, 0));
190 
191  dlib::draw_line(eyes[2],
192  data.centers[i] + offset_x + chips[i].rect.tl_corner(),
193  data.centers[i] - offset_x + chips[i].rect.tl_corner(),
194  dlib::rgb_pixel(255, 0, 0));
195  dlib::draw_line(eyes[2],
196  data.centers[i] + offset_y + chips[i].rect.tl_corner(),
197  data.centers[i] - offset_y + chips[i].rect.tl_corner(),
198  dlib::rgb_pixel(255, 0, 0));
199  }
200  dlib::resize_image(0.5, eyes[2]);
201 
202  this->widget->set_image(dlib::tile_images(eyes));
203 }
204 
205 } // namespace pipeline
206 
207 } // namespace gaze
std::vector< dlib::chip_details > get_eyes_chip_details(const dlib::full_object_detection object_detection)
static softfloat zero()
dlib::array< dlib::array2d< double > > eyes
Definition: data.h:69
dlib::array< dlib::point > centers
Definition: data.h:75
size_t size() const
dlib::array2d< dlib::bgr_pixel > image
Definition: data.h:52
void fill_displacement_tables(dlib::matrix< double > &table_x, dlib::matrix< double > &table_y, int size)
_GLIBCXX14_CONSTEXPR const _Tp & max(const _Tp &__a, const _Tp &__b)
static constexpr _Tp min() noexcept
dlib::full_object_detection landmarks
Definition: data.h:64
Wraps the data acquired per frame into a single instance.
Definition: data.h:27
complex< _Tp > sqrt(const complex< _Tp > &)
void normalize_and_threshold_gradients(dlib::matrix< T > &horizontal, dlib::matrix< T > &vertical, double relative_threshold=-1)
YAML::Node get_config()
Definition: config.in.cpp:12
_Tp sum() const