gaze::pipeline::PupilLocalization Class Reference

Implements Timm and Barth (2011) Timm2011 to detect eye centers. More...

#include "gaze/pipeline_steps/pupil_localization.h"

Inheritance diagram for gaze::pipeline::PupilLocalization:

Public Member Functions
void	process (util::Data &data) override
void	visualize (util::Data &data) override
Public Member Functions inherited from gaze::PipelineStep
	PipelineStep ()
std::string	get_name ()
virtual	~PipelineStep ()=default
Public Member Functions inherited from gaze::gui::Visualizeable< widget_type >
std::shared_ptr< dlib::drawable >	init (dlib::drawable_window &parent, int width, int height, std::string text="No visualization.") final
void	remove_widget () override
Public Member Functions inherited from gaze::gui::VisualizeableBase
virtual	~VisualizeableBase ()=default

Additional Inherited Members
Protected Attributes inherited from gaze::PipelineStep
int	number
std::string	name
Protected Attributes inherited from gaze::gui::Visualizeable< widget_type >
std::shared_ptr< widget_type >	widget

Detailed Description

Implements Timm and Barth (2011) Timm2011 to detect eye centers.

This implementation uses a lookup table to pre-calculate the distance function. Additionally, some tweaks are introduced, as explained in the similar implementation eyeLike by Tristan Hume.

For details you can also refer to Tristan's blog post Simple, accurate eye center tracking in OpenCV.

Definition at line 125 of file pupil_localization.h.

Member Function Documentation

void gaze::pipeline::PupilLocalization::process ( util::Data &  data )

overridevirtual

Detects the eye center for each eye.

Parameters

data	The data object to be updated.

Returns

via data the modified data object.

Implements gaze::PipelineStep.

Definition at line 88 of file pupil_localization.cpp.

References gaze::util::Data::centers, gaze::util::Data::eyes, gaze::util::fill_displacement_tables(), gaze::util::get_eyes_chip_details(), gaze::util::Data::image, gaze::util::Data::landmarks, std::max(), std::numeric_limits< class >::min(), gaze::util::normalize_and_threshold_gradients(), sum(), and zero().

                                              {
  if (data.landmarks.num_parts() < 5) {
    return;
  }

  // Extract eye patches
  std::vector<dlib::chip_details> details =
    util::get_eyes_chip_details(data.landmarks);
  dlib::extract_image_chips(data.image, details, data.eyes);

  // Resize lookup table if needed
  int max_size = std::max({data.eyes[0].nr(), data.eyes[0].nc(),
                           data.eyes[1].nr(), data.eyes[1].nc()});
  if (2 * max_size > this->displacement_table_x.nr()) {
    util::fill_displacement_tables(
        this->displacement_table_x,
        this->displacement_table_y,
        2 * max_size + 7);
  }

  for (int i = 0; i < 2; ++i) {
    dlib::matrix<double> eye_in;
    dlib::assign_image(eye_in, data.eyes[i]);

    // Calculate gradients
    dlib::matrix<double> eye_horizontal;
    dlib::matrix<double> eye_vertical;
    dlib::sobel_edge_detector(eye_in, eye_horizontal, eye_vertical);

    util::normalize_and_threshold_gradients(eye_horizontal, eye_vertical,
        this->relative_threshold_factor);

    // Blur eyes as weights
    int boundary_offset = 6;
    dlib::matrix<double> eye_gaussian;
    dlib::gaussian_blur(eye_in, eye_gaussian, this->sigma, boundary_offset);
    eye_gaussian = 255 - eye_gaussian;

    // Compute objective function t and select highest value
    double max_t = std::numeric_limits<double>::min();
    int argmax_r;
    int argmax_c;

    int center = (this->displacement_table_x.nr() - 1) / 2;
    auto nr = eye_in.nr();
    auto nc = eye_in.nc();
    dlib::matrix<double> zero = dlib::zeros_matrix<double>(nr, nc);
    dlib::matrix<double> ts = dlib::zeros_matrix<double>(nr, nc);
    for (auto row = decltype(nr){boundary_offset};
         row < nr - boundary_offset; ++row) {
      for (auto col = decltype(nc){boundary_offset};
           col < nc - boundary_offset; ++col) {
        dlib::matrix<double> d_x = dlib::subm(this->displacement_table_x,
            center - row, center - col, nr, nc);
        dlib::matrix<double> d_y = dlib::subm(this->displacement_table_y,
            center - row, center - col, nr, nc);

        double t =
          dlib::sum(
              dlib::squared(dlib::max_pointwise(
                dlib::pointwise_multiply(d_x, eye_horizontal) +
                dlib::pointwise_multiply(d_y, eye_vertical), zero))) *
          eye_gaussian(row, col);
        if (t > max_t) {
          max_t = t;
          argmax_r = row;
          argmax_c = col;
        }
      }
    }
    data.centers[i] = dlib::point(argmax_c, argmax_r);
  }
}

void gaze::pipeline::PupilLocalization::visualize ( util::Data &  data )

overridevirtual

Parameters

data	The data object to be updated.

Implements gaze::gui::VisualizeableBase.

Definition at line 162 of file pupil_localization.cpp.

References gaze::util::Data::centers, gaze::util::Data::eyes, gaze::util::get_eyes_chip_details(), gaze::util::Data::image, and gaze::util::Data::landmarks.

                                                {
  if (data.landmarks.num_parts() < 5) {
    return;
  }
  // Ensures output of this pipeline step is visualized
  this->process(data);

  // define offsets
  dlib::point offset_x(5, 0);
  dlib::point offset_y(0, 5);
  std::vector<dlib::chip_details> chips =
    util::get_eyes_chip_details(data.landmarks);

  // assign eyes to new images
  dlib::array<dlib::array2d<dlib::bgr_pixel>> eyes(3);
  dlib::assign_image(eyes[2], data.image);
  for (auto i = decltype(data.eyes.size()){0};
       i < data.eyes.size(); ++i) {
    dlib::assign_image(eyes[i], data.eyes[i]);

    dlib::draw_line(eyes[i],
                    data.centers[i] + offset_x,
                    data.centers[i] - offset_x,
                    dlib::rgb_pixel(255, 0, 0));
    dlib::draw_line(eyes[i],
                    data.centers[i] + offset_y,
                    data.centers[i] - offset_y,
                    dlib::rgb_pixel(255, 0, 0));

    dlib::draw_line(eyes[2],
                    data.centers[i] + offset_x + chips[i].rect.tl_corner(),
                    data.centers[i] - offset_x + chips[i].rect.tl_corner(),
                    dlib::rgb_pixel(255, 0, 0));
    dlib::draw_line(eyes[2],
                    data.centers[i] + offset_y + chips[i].rect.tl_corner(),
                    data.centers[i] - offset_y + chips[i].rect.tl_corner(),
                    dlib::rgb_pixel(255, 0, 0));
  }
  dlib::resize_image(0.5, eyes[2]);

  this->widget->set_image(dlib::tile_images(eyes));
}

---

The documentation for this class was generated from the following files:

• include/gaze/pipeline_steps/pupil_localization.h
• src/gaze/pipeline_steps/pupil_localization.cpp