gaze::pipeline::GazePointCalculation Class Reference

Calculates the gaze point. More...

#include "gaze/pipeline_steps/gaze_point_calculation.h"

Inheritance diagram for gaze::pipeline::GazePointCalculation:

Public Member Functions
void	process (util::Data &data) override
void	visualize (util::Data &data) override
Public Member Functions inherited from gaze::PipelineStep
	PipelineStep ()
std::string	get_name ()
virtual	~PipelineStep ()=default
Public Member Functions inherited from gaze::gui::Visualizeable< widget_type >
std::shared_ptr< dlib::drawable >	init (dlib::drawable_window &parent, int width, int height, std::string text="No visualization.") final
void	remove_widget () override
Public Member Functions inherited from gaze::gui::VisualizeableBase
virtual	~VisualizeableBase ()=default

Protected Member Functions
void	set_sensor_size (double sensor_diagonal, double aspect_ratio)

Additional Inherited Members
Protected Attributes inherited from gaze::PipelineStep
int	number
std::string	name
Protected Attributes inherited from gaze::gui::Visualizeable< widget_type >
std::shared_ptr< widget_type >	widget

Detailed Description

Calculates the gaze point.

///

Todo:

shoeffner: Write documentation for GazePointCalculation.

Definition at line 53 of file gaze_point_calculation.h.

Member Function Documentation

void gaze::pipeline::GazePointCalculation::process ( util::Data &  data )

overridevirtual

Calculates the gaze point given a head pose and eye centers.

Parameters

data	The data object to be updated.

Returns

via data the modified data object.

Implements gaze::PipelineStep.

Definition at line 214 of file gaze_point_calculation.cpp.

References gaze::util::Data::centers, gaze::util::clamp(), gaze::util::Data::estimated_gaze_point, gaze::util::Data::gaze_points, gaze::util::get_eyes_chip_details(), gaze::util::Data::head_rotation, gaze::util::Data::head_translation, gaze::util::Data::landmarks, gaze::util::Data::pupils, std::vector< _Tp, _Alloc >::push_back(), and cv::Rodrigues().

                                                 {
  if (data.landmarks.num_parts() < 5) {
    return;
  }

  // Convert landmarks to OpenCV data structures
  std::vector<cv::Vec3d> landmarks;
  for (auto i : this->landmark_indices) {
    cv::Vec3d landmark(data.landmarks.part(i).x(), data.landmarks.part(i).y(),
        0);
    landmarks.push_back(landmark);
  }

  // Project pupils into model
  cv::Matx34d transformation = this->invertProjection(landmarks);

  std::vector<dlib::chip_details> eyes =
    util::get_eyes_chip_details(data.landmarks);

  for (auto i = decltype(data.pupils.size()){0};
       i < data.pupils.size(); ++i) {
    dlib::point eye = data.centers[i] + eyes[i].rect.tl_corner();
    data.pupils[i] = transformation * cv::Vec4d(eye.x(), eye.y(), 0, 1);
  }

  // Calculate distance
  double distance = this->calculate_distance(
      data.landmarks.part(36), data.landmarks.part(45), 0.08671);
  cv::Matx33d R;
  cv::Rodrigues(data.head_rotation, R);

  cv::Vec3d model_to_camera_dir =
    this->get_model_to_camera_dir(data.head_translation, R);
  cv::Vec3d camera_pos = this->get_camera_pos(model_to_camera_dir, distance);

  std::vector<cv::Vec3d> screen_tl_tr_br_bl =
    this->get_screen_corners(camera_pos, R);

  // Ray cast
  cv::Vec2d screen_coord_coeffs(0, 0);
  for (auto i = decltype(data.gaze_points.size()){0};
       i < data.gaze_points.size(); ++i) {
    cv::Matx32d gaze_point = this->calculate_gaze_point(
        this->eye_ball_centers[i], data.pupils[i],
        screen_tl_tr_br_bl[0], screen_tl_tr_br_bl[1], screen_tl_tr_br_bl[3]);
    data.gaze_points[i] =
        cv::Vec3d(gaze_point(0, 0), gaze_point(1, 0), gaze_point(2, 0));
    screen_coord_coeffs[0] += gaze_point(1, 1);
    screen_coord_coeffs[1] += gaze_point(1, 2);
  }
  screen_coord_coeffs /= 2;
  // swap left and right
  screen_coord_coeffs[0] = 1 - util::clamp(screen_coord_coeffs[0], 0, 1);
  screen_coord_coeffs[1] = util::clamp(screen_coord_coeffs[1], 0, 1);

  data.estimated_gaze_point = cv::Vec2d(
      this->target_width * screen_coord_coeffs[0],
      this->target_height * screen_coord_coeffs[1]);
}

void gaze::pipeline::GazePointCalculation::set_sensor_size ( double  sensor_diagonal, double  aspect_ratio  )

protected

Calculates the sensor width and sensor height from the sensor diagonal and its aspect ratio.

Definition at line 206 of file gaze_point_calculation.cpp.

References std::sqrt().

                         {
  this->sensor_height = std::sqrt(sensor_diagonal * sensor_diagonal
      / (1 + aspect_ratio * aspect_ratio));
  this->sensor_width = aspect_ratio * this->sensor_height;
}

void gaze::pipeline::GazePointCalculation::visualize ( util::Data &  data )

overridevirtual

Visualizes the estimated gaze point.

Parameters

data	The data object to be visualized.

Todo:

shoeffner: Add pipeline step to visualize 2D gaze point / area

Implements gaze::gui::VisualizeableBase.

Definition at line 274 of file gaze_point_calculation.cpp.

References gaze::util::Data::head_rotation, gaze::util::Data::head_translation, gaze::util::Data::landmarks, gaze::util::Data::pupils, std::vector< _Tp, _Alloc >::push_back(), and cv::Rodrigues().

                                                   {
  // TODO(shoeffner): Add pipeline step to visualize 2D gaze point / area
  this->widget->clear_overlay();
  if (data.landmarks.num_parts() < 5) {
    return;
  }

  // Recalculate distance, etc.
  double distance = this->calculate_distance(
      data.landmarks.part(36), data.landmarks.part(45), 0.08671);
  cv::Matx33d R;
  cv::Rodrigues(data.head_rotation, R);

  // Move landmarks into model coordinates for an intuitive visualization of
  // where the direction of the camera plane is
  std::vector<cv::Vec2d> image_landmarks;
  for (auto i = decltype(data.landmarks.num_parts()){0};
       i < data.landmarks.num_parts(); ++i) {
    image_landmarks.push_back(
        cv::Vec2d(data.landmarks.part(i).x(), data.landmarks.part(i).y()));
  }
  std::vector<cv::Vec3d> world_landmarks = this->unprojectPoints(
      image_landmarks, data.head_translation, R, distance);

  // Find camera direction
  cv::Vec3d camera_dir =
    this->get_model_to_camera_dir(data.head_translation, R);

  cv::Vec3d camera_pos = this->get_camera_pos(camera_dir, distance);

  // Calculate screen corners
  std::vector<cv::Vec3d> screen_corners =
    this->get_screen_corners(camera_pos, R);

  // Helpers to draw points
  std::vector<dlib::perspective_display::overlay_dot>
    overlay_dots;

  auto add_to_overlay =
    [&overlay_dots] (std::vector<cv::Vec3d> points,
                     dlib::rgb_pixel color) -> void {
      for (auto p : points) {
        overlay_dots.push_back({{p[0], p[1], p[2]}, color});
      }
    };

  add_to_overlay(this->model, {255, 255, 0});
  add_to_overlay(data.pupils, {0, 255, 255});
  add_to_overlay(this->eye_ball_centers, {255, 0, 255});
  add_to_overlay(world_landmarks, {0, 255, 0});

  std::vector<dlib::chip_details> chips =
    util::get_eyes_chip_details(data.landmarks);
  std::vector<dlib::point> pupils_image = {data.centers[0], data.centers[1]};
  for (auto i = decltype(pupils_image.size()){0}; i < pupils_image.size();
       ++i) {
    pupils_image[i] += chips[i].rect.tl_corner();
  }
  std::vector<cv::Vec3d> pupils_world =
    this->unprojectPoints({cv::Vec2d(pupils_image[0].x(), pupils_image[0].y()),
        cv::Vec2d(pupils_image[1].x(), pupils_image[1].y())},
        data.head_translation, R, distance);

  add_to_overlay(pupils_world, {255, 255, 255});

  // Visualize camera position
  cv::Vec3d camera = this->get_camera_pos(camera_dir, distance);
  overlay_dots.push_back({{camera[0], camera[1], camera[2]},
      dlib::rgb_pixel(255, 255, 255)});

  this->widget->add_overlay(overlay_dots);

  auto to_line = [](const cv::Vec3d& p0, const cv::Vec3d& p1,
      dlib::rgb_pixel color) ->
    dlib::perspective_display::overlay_line {
      return dlib::perspective_display::overlay_line(
          {p0[0], p0[1], p0[2]}, {p1[0], p1[1], p1[2]}, color);
    };

  // Draw coordinate axes
  cv::Vec3d origin(0, 0, 0);
  cv::Vec3d xdir(.1, 0, 0);
  cv::Vec3d ydir(0, .1, 0);
  cv::Vec3d zdir(0, 0, .1);

  this->widget->add_overlay({
      to_line(origin, xdir, {255, 0, 0}),
      to_line(origin, ydir, {0, 255, 0}),
      to_line(origin, zdir, {0, 0, 255}),
      to_line(origin, origin + camera_dir, {255, 255, 0})});

  this->widget->add_overlay({
      to_line(screen_corners[0], screen_corners[1], {255, 255, 0}),
      to_line(screen_corners[1], screen_corners[2], {255, 255, 0}),
      to_line(screen_corners[2], screen_corners[3], {255, 255, 0}),
      to_line(screen_corners[3], screen_corners[0], {255, 255, 0})});

  this->widget->add_overlay({
      to_line(this->eye_ball_centers[0], data.pupils[0], {0, 255, 255}),
      to_line(this->eye_ball_centers[1], data.pupils[1], {0, 255, 255})});

  this->widget->add_overlay({
      to_line(this->eye_ball_centers[0], data.gaze_points[0], {255, 255, 255}),
      to_line(this->eye_ball_centers[1], data.gaze_points[1],
          {255, 255, 255})});
}

---

The documentation for this class was generated from the following files:

• include/gaze/pipeline_steps/gaze_point_calculation.h
• src/gaze/pipeline_steps/gaze_point_calculation.cpp