gaze  0.1.0
Perform gaze tracking with common webcams.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Groups Pages
itracker.cpp
1 #include "itracker/itracker.h"
2 
3 #include <iostream>
4 #include <memory>
5 #include <string>
6 
7 #include "caffe/caffe.hpp"
8 #include "opencv2/opencv.hpp"
9 
10 namespace itracker {
11 
15 class ITrackerImpl {
18  public:
19  ITrackerImpl(const std::string& model_file,
20  const std::string& weights_file,
21  const std::string& mean_left_eye_file,
22  const std::string& mean_right_eye_file,
23  const std::string& mean_face_file);
24 
25  cv::Vec2f predict(const cv::Mat& left_eye, const cv::Mat& right_eye,
26  const cv::Mat& face, const cv::Mat& face_mask);
27 };
28 
29 namespace util {
30 
40 cv::Mat load_mean(const std::string& mean_file) {
41  caffe::BlobProto blob_proto;
42  caffe::ReadProtoFromBinaryFileOrDie(mean_file.c_str(), &blob_proto);
43  // Convert from BlobProto to Blob<float>
44  caffe::Blob<float> mean_blob;
45  mean_blob.FromProto(blob_proto);
46 
48  float* data = mean_blob.mutable_cpu_data();
49  for (int i = 0; i < 3; ++i) {
50  cv::Mat channel(mean_blob.height(), mean_blob.width(), CV_32FC1, data);
51  channels.push_back(channel);
52  data += mean_blob.height() * mean_blob.width();
53  }
54 
55  cv::Mat mean;
56  cv::merge(channels, mean);
57 
58  cv::Scalar channel_mean = cv::mean(mean);
59  return cv::Mat(224, 224, mean.type(), channel_mean);
60 }
61 
62 } // namespace util
63 
64 ITrackerImpl::ITrackerImpl(const std::string& model_file,
65  const std::string& weights_file,
66  const std::string& mean_left_eye_file,
67  const std::string& mean_right_eye_file,
68  const std::string& mean_face_file) {
69  // Init network
71  new caffe::Net<float>(model_file, caffe::TEST));
72  // Load stored weights
73  this->net->CopyTrainedLayersFrom(weights_file);
74 
75  // Resize network (batchsize 1)
76  for (auto i = 0; i < 4; ++i) {
77  auto input = this->net->input_blobs()[i];
78  std::vector<int> shape = input->shape();
79  shape[0] = 1;
80  input->Reshape(shape);
81  }
82  this->net->Reshape();
83 
84  // Load means
85  this->means = {
86  util::load_mean(mean_left_eye_file),
87  util::load_mean(mean_right_eye_file),
88  util::load_mean(mean_face_file)
89  };
90 }
91 
92 cv::Vec2f ITrackerImpl::predict(
93  const cv::Mat& left_eye, const cv::Mat& right_eye,
94  const cv::Mat& face, const cv::Mat& face_mask) {
95  // Gather input
96  std::vector<cv::Mat> input_images = {left_eye, right_eye, face, face_mask};
97 
98  // Convert and feed input
99  for (int i = 0; i < 4; ++i) {
100  auto input = this->net->input_blobs()[i];
101  std::vector<int> shape = input->shape();
102 
103  // Create mat object per channel with pointer to layer data
105  float* input_data = input->mutable_cpu_data();
106  for (int j = 0; j < shape[1]; ++j) {
107  cv::Mat channel(shape[2], shape[3], CV_32FC1, input_data);
108  channels.push_back(channel);
109  input_data += shape[2] * shape[3];
110  }
111 
112  // Resize image to channel shape (or reshape for the map)
113  cv::Mat input_image;
114  if (i != 3) { // Default case is just filling the zero-mean images
115  cv::Mat resized(shape[2], shape[3], CV_32FC3);
116  input_images[i].convertTo(input_image, CV_32FC3);
117  cv::resize(input_image, resized, resized.size());
118  cv::subtract(resized, this->means[i], resized);
119  cv::split(resized, channels);
120  } else { // Mask has special layout, thus copying value by value
121  input_images[i].convertTo(input_image, CV_32FC1);
122  for (int j = 0; j < shape[1]; ++j) {
123  channels[j] = input_image.at<float>(j);
124  }
125  }
126  }
127 
128  // Inference step
129  this->net->Forward();
130 
131  // Read out results
132  caffe::Blob<float>* output_layer = this->net->output_blobs()[0];
133  const float* begin = output_layer->cpu_data();
134  std::vector<float> out(begin, begin + output_layer->channels());
135 
136  // Store result
137  return cv::Vec2f(out[0], out[1]);
138 }
139 
140 ITracker::ITracker(const std::string& model_file,
141  const std::string& weights_file,
142  const std::string& mean_left_eye_file,
143  const std::string& mean_right_eye_file,
144  const std::string& mean_face_file)
145  : impl(new ITrackerImpl(model_file, weights_file, mean_left_eye_file,
146  mean_left_eye_file, mean_face_file)) {
147 }
148 
149 cv::Vec2f ITracker::predict(const cv::Mat& left_eye, const cv::Mat& right_eye,
150  const cv::Mat& face, const cv::Mat& face_mask) {
151  return this->impl->predict(left_eye, right_eye, face, face_mask);
152 }
153 
154 ITracker::~ITracker() {
155  delete this->impl;
156 }
157 
158 } // namespace itracker
Scalar mean(InputArray src, InputArray mask=noArray())
void split(const Mat &src, Mat *mvbegin)
cv::Vec2f predict(const cv::Mat &left_eye, const cv::Mat &right_eye, const cv::Mat &face, const cv::Mat &face_mask)
Definition: itracker.cpp:149
int type() const
void resize(InputArray src, OutputArray dst, Size dsize, double fx=0, double fy=0, int interpolation=INTER_LINEAR)
MatSize size
void push_back(const value_type &__x)
void subtract(InputArray src1, InputArray src2, OutputArray dst, InputArray mask=noArray(), int dtype=-1)
void merge(const Mat *mv, size_t count, OutputArray dst)
Vec< float, 2 > Vec2f
const char * c_str() const noexcept
int channels() const
A private implementation of ITracker to hide dependencies from including classes. ...
Definition: itracker.cpp:15
#define CV_32FC1
_Tp & at(int i0=0)
ITracker(const std::string &model_file, const std::string &weights_file, const std::string &mean_left_eye_file, const std::string &mean_right_eye_file, const std::string &mean_face_file)
Definition: itracker.cpp:140