利用手势识别控制播放器

分享一个大二课程设计的自选课题“手势控制播放器”，。
利用OpenCV跨平台计算机视觉库模拟计算机视觉，实时检测用户的手势动作并翻译为计算机指令并且执行，本次实验以网易云音乐播放器为实验软件（酷狗、酷我、QQ等音乐播放器也可用）实现用户手势操作播放/暂停、上一首、下一首简单操作。

工具：

c++、Opencv、网易云音乐播放器、当然还有你的hand!

系统结构分析

流程图

源码

#include "stdafx.h"
#include "CVObject.h"

#include <cmath>
#include <queue>
#include <algorithm>

using namespace std;

void ThresholdBidirection(Mat & img, int lower, int upper) {
if (img.channels() != 1)
throw exception("图像通道数目不为1");

for (int h = 0; h < img.rows; h++) {
for (int w = 0; w < img.cols; w++) {
uchar * data = &img.at<uchar>(w, h);
if (*data <= upper && *data >= lower) {
*data = 255;
}
else {
*data = 0;
}
}
}
}

void FindTargets(const Mat & img, const int erea_threshold, Rect & rect, Mat & filtered) {
if (img.channels() != 1)
throw exception("FindTargets : 通道数必须为 1");
vector<vector<Point>> contours;         // contours被定义成二维浮点型向量，这里面将来会存储找到的边界的（x,y）坐标。
findContours(img, contours, CV_RETR_EXTERNAL, CHAIN_APPROX_NONE);
vector<Point> max_contour;        //寻找面积最大的轮廓画矩形框架
double max_area = 0;
for (auto item : contours) {
auto area = contourArea(item);
if (area > max_area && area > erea_threshold) {
max_area = area;
max_contour = item;
}
}

rect = Rect();
if (max_contour.size() > 0)
rect = boundingRect(max_contour);

filtered = Mat::zeros(img.rows, img.cols, CV_8UC3);   //指定数组的形状的整数数组
drawContours(filtered, vector<vector<Point>>{max_contour}, -1, Scalar(255, 255, 255), CV_FILLED);                                                //轮廓绘制函数指定范围
rectangle(filtered, rect, Scalar(0, 255, 255), 3);

#ifdef DEBUG
imshow("contours", filtered);
#endif
}

bool Locus::InRegionLeftOfTurn(Point p, Size winSize) {
if (p.x < winSize.width / 3) {
return true;
}
return false;
}

bool Locus::InRegionRightOfTurn(Point p, Size winSize) {
if (p.x>winSize.width * 2 / 3) {
return true;
}
return false;
}                                       //轨迹添加中心点
void Locus::addPoint(Point p, Size winSize) {
tracks.push_back(p);
newPoint = p;
}

Gesture Locus::analyseLocus() {
Gesture g = NOOPERATION;

Point first = tracks.front();
if (InRegionRightOfTurn(first, winSize) && InRegionLeftOfTurn(newPoint, winSize)) {
g = PREVIOUS;
tracks.clear();
tracks.push_back(newPoint);
}
else if (InRegionLeftOfTurn(first, winSize) && InRegionRightOfTurn(newPoint, winSize)) {
g = NEXT;
tracks.clear();
tracks.push_back(newPoint);
}
else if (InRegionTop(first, winSize) && InRegionBottom(newPoint, winSize)) {
g = PAUSEPLAY;
tracks.clear();
tracks.push_back(newPoint);
}
else if (InRegionBottom(first, winSize) && InRegionTop(newPoint, winSize)) {
g = PAUSEPLAY;
tracks.clear();
tracks.push_back(newPoint);
}

return g;
}
//取中心区域的顶部
bool Locus::InRegionTop(Point p, Size winSize) {
if (p.y < winSize.height / 3) {
return true;
}
return false;
}
//取中心区域的底部
bool Locus::InRegionBottom(Point p, Size winSize) {
if (p.y > winSize.height * 2 / 3) {
return true;
}
return false;
}






#include "stdafx.h"
#include "CVObject.h"
#include <exception>


void sendHotKey(BYTE key) {              //创建热键
keybd_event(VK_CONTROL, 0, 0, 0);
keybd_event(VK_MENU, 0, 0, 0);
keybd_event(key, 0, 0, 0);
keybd_event(key, 0, KEYEVENTF_KEYUP, 0);
keybd_event(VK_MENU, 0, KEYEVENTF_KEYUP, 0);
keybd_event(VK_CONTROL, 0, KEYEVENTF_KEYUP, 0);
}
//发送内部命令
void postCommand(Gesture g) {
switch (g) {
case NOOPERATION:
return;
case PREVIOUS:
sendHotKey(VK_RIGHT);
cout << "previous" << endl;   //触发上一首
break;
case NEXT:
sendHotKey(VK_LEFT);
cout << "next" << endl;   //触发下一首
break;
case PAUSEPLAY:
sendHotKey('X');
cout << "pause/play" << endl;    //触发暂停/播放
break;
default:
return;
}
}

bool PreSet(int & threshold_lower, int & threshold_higher) {
namedWindow("skin", CV_WINDOW_NORMAL);
namedWindow("video", CV_WINDOW_NORMAL);
//为皮肤窗口创建轨迹条
createTrackbar("Lower Threshold", "skin", &threshold_lower, 255, NULL);
createTrackbar("Higher Threshold2", "skin", &threshold_higher, 255, NULL);

moveWindow("skin", 0, 320);
moveWindow("video", 0, 0);

return true;
}

VideoCapture openCamera(int cameraIndex) {
VideoCapture cap(cameraIndex);
if (!cap.isOpened())
throw exception("初始化相机失败");

return cap;
}

Mat getFrame(VideoCapture & cap) {
Mat img;
bool ret = cap.read(img);
if (!ret)
throw exception("读取相机图像失败");

return img;
}
//框架句柄
void FramePreHandle(Mat ImgFrame, int & threshold_lower, int & threshold_higher, Mat & cb, Mat & ImgFrameSmall) {
Mat ImgSkin;
vector<Mat> channels;      //用来存放三个通道的像素值
resize(ImgFrame, ImgFrameSmall, winSize);
cvtColor(ImgFrameSmall, ImgSkin, CV_BGR2YCrCb);     //转YCrCb
split(ImgSkin, channels);
ThresholdBidirection(channels[2], threshold_lower, threshold_higher);

Mat element = getStructuringElement(MORPH_RECT, Size(5, 5));
morphologyEx(channels[2], channels[2], MORPH_OPEN, element);
cb = channels[2];

#ifdef DEBUG
imshow("cb", cb);      //显示cd窗口
#endif
}

bool SetControlUI(const Size & winSize, Mat & ImgSkin) {
ImgSkin = Mat::zeros(winSize.height, winSize.width, CV_8UC3);

rectangle(ImgSkin, Point(0, 0), Point(winSize.width / 6, winSize.height), Scalar(0, 255, 255), CV_FILLED);
rectangle(ImgSkin, Point(winSize.width * 5 / 6, 0), Point(winSize.width, winSize.height), Scalar(0, 255, 255), CV_FILLED);
//指定的位置和大小初始化 rectangle 类的新实例。
return true;
}

int main(int argc, char**argv) {
try {
int cameraIndex = 0;
if (argc == 2) {
cameraIndex = argv[1][0] - '0';
}

int threshold_lower = 0, threshold_higher = 122;
PreSet(threshold_lower, threshold_higher);
VideoCapture cap = openCamera(cameraIndex);
Sleep(1000);
Mat ImgFrame;

Locus locusAnalyser;                //轨迹分析

while (true) {
ImgFrame = getFrame(cap);
if (ImgFrame.empty())
break;

Mat cb, ImgSkin, ImgFrameSmall, ImgSkin_s;
FramePreHandle(ImgFrame, threshold_lower, threshold_higher, cb, ImgFrameSmall);
SetControlUI(winSize, ImgSkin);
Rect target_rect;
FindTargets(cb, 1000, target_rect, ImgSkin_s);
// findContours会改变原图
if (target_rect.width > 0) {
rectangle(ImgFrameSmall, target_rect, Scalar(255, 0, 0), 3);
Point center(target_rect.x + target_rect.width / 2, target_rect.y + target_rect.height / 2);
ImgSkin += ImgSkin_s;

//在目标中心画十字
line(ImgSkin, Point(center.x - 5, center.y - 5), Point(center.x + 5, center.y + 5), Scalar(0, 0, 255), 3);
line(ImgSkin, Point(center.x + 5, center.y - 5), Point(center.x - 5, center.y + 5), Scalar(0, 0, 255), 3);

// 轨迹
locusAnalyser.addPoint(center, winSize);
Gesture g = locusAnalyser.analyseLocus();

// 手势姿态
postCommand(g);
}
else {
locusAnalyser.reset();
}


imshow("skin", ImgSkin);
imshow("video", ImgFrameSmall);

if (waitKey(3) == 27)
break;
}

return 0;
}
catch (exception & e) {
cerr << e.what() << endl;
return -1;
}
}