Khi đã đếm được số ngón tay, lúc này sinh viên thực hiện sẽ xét trạng thái ngón tay để vẽ:
- Trường hợp 1 ngón thì bắt đầu vẽ tại điểm đầu ngón tay dài nhất. - Trường hợp 5 ngón tay sẽ xóa đi nét vẽ.
CHƯƠNG 5: KẾT LUẬN VÀ ĐỊNH HƯỚNG ĐỀ TÀI 5.1. Kết quả đạt được
- Hoàn thành được khóa luận như mong muốn.
- Kỹ năng viết code gọi chương trình con cũng đã tiến bộ sau khóa luận trên.
5.2. Hạn chế
- Nét vẽ vẫn còn bị đứt nét và giai đoạn tiền xử lý ảnh cũng chưa tốt lắm. - Đôi khi đếm số ngón tay sai.
5.3. Hướng phát triển đề tài
PHỤ LỤC Code chương trình #include "opencv/cv.h" #include "opencv/highgui.h" #include "CblobLabeling.h" #include <opencv2/core/core_c.h> #include <opencv2/imgproc/imgproc.hpp> #include <iostream> #include <fstream> using namespace std;
void background_subtraction(IplImage*src, IplImage*dst, IplImage*bg); void skin_detection(IplImage*src, IplImage*dst);
void hand_detection(IplImage*src, IplImage*dst, IplImage*mask);
void fingers_counting(IplImage* src, IplImage* dst_hand, IplImage* dst); unsigned char num_fingers;
// pt is the center
// pt1 is the top left of blob // pt2 is the farthest point
CvPoint pt, pt1, pt2, pt2_f, pt2_l; #define no_fingers 65
#define threshold_fingers 25 //Đk sáng chói
//int minY = 53, maxY = 196, minU = 106, maxU = 140, minV = 128, maxV = 160; //Đk của video 1
//int minY = 170, maxY = 255, minU = 0, maxU = 140, minV = 119, maxV = 160; //Đk tại nhà
int minY = 190, maxY = 255, minU = 0, maxU = 140, minV = 119, maxV = 160; const char* window0 = "setupYUV";
void main() {
IplImage* frm1;
IplImage* frm_resize = cvCreateImage(cvSize(cvQueryFrame(capture)- >width / 2, (cvQueryFrame(capture))->height / 2), 8, 3);
IplImage* frm_bg_resize = cvCreateImage(cvGetSize(frm_resize), 8, 3); IplImage* frm_bg = cvCreateImage(cvGetSize(frm_resize), 8, 1);
IplImage* frm_bg_subtraction = cvCreateImage(cvGetSize(frm_resize), 8, 1); IplImage* frm_skin_detection = cvCreateImage(cvGetSize(frm_resize), 8, 1); IplImage* frm_hand_detection = cvCreateImage(cvGetSize(frm_resize), 8, 1); IplImage* hand = cvCreateImage(cvGetSize(frm_resize), 8, 3);
IplImage* frm_draw = cvCreateImage(cvGetSize(cvQueryFrame(capture)), 8, 3); IplImage* frm_draw_binary = cvCreateImage(cvGetSize(cvQueryFrame(capture)), 8, 1); frm1 = cvQueryFrame(capture); cvSetZero(frm_draw); cvSetZero(frm_draw_binary); cvResize(frm1, frm_bg_resize, 1); //cvFlip(frm_bg_resize, frm_bg_resize, 1); //cvShowImage("Nen", Old_frame); cvCvtColor(frm_bg_resize, frm_bg, CV_RGB2GRAY); cvSmooth(frm_bg, frm_bg, CV_GAUSSIAN, 5); //cvShowImage("Nen2", Old_frame_Gray); //save fps ofstream myfile;
myfile.open("C:\\Users\\Huynh Tan Cuong\\Desktop\\fps.csv"); while (1)
{
int64 now, then;
float ticks = cvGetTickFrequency()*1.0e6; then = cvGetTickCount();
// get frm from CAM
frm = cvQueryFrame(capture); if (!frm) break;
// flip frame
//cvFlip(frm, frm, 1);
cvShowImage("Webcam", frm); // resize 0.5 to increase speed cvResize(frm, frm_resize, 1); // Background Subtraction
background_subtraction(frm_resize, frm_bg_subtraction, frm_bg); //cvShowImage("Background Subtraction", frm_bg_subtraction); // skin detection
skin_detection(frm_resize, frm_skin_detection);
//cvShowImage("Skin Detection", frm_skin_detection); // hand detection
cvAnd(frm_skin_detection, frm_bg_subtraction, frm_hand_detection); cvShowImage("Hand Detection", frm_hand_detection);
// HAND
hand_detection(frm_resize, hand, frm_hand_detection); //
//finger counting
fingers_counting(frm_hand_detection, hand, frm); //cvShowImage("Webcam draw", frm);
//cvShowImage("Draw hand", hand); //Draw if (num_fingers == 1) { cvLine(frm_draw, pt2_l, pt2_f, CV_RGB(255, 0, 0), 5); cvLine(frm_draw_binary, pt2_l, pt2_f, CV_RGB(255, 255, 255), 5); } if (num_fingers == 5) { cvSetZero(frm_draw); cvSetZero(frm_draw_binary); } cvCopy(frm_draw, frm, frm_draw_binary); pt2_l = pt2_f; //printf("pointpt2_l: %d\n", pt2_l); cvShowImage("End", frm); /*cvShowImage("End2", frm_draw); cvShowImage("End3", frm_draw_binary);*/ // now = cvGetTickCount();
float frame_time = (now - then) / ticks; float fps = 1 / frame_time;
if (c >= 0) break; } cvReleaseCapture(&capture); cvDestroyAllWindows(); myfile.close(); }
void background_subtraction(IplImage*src, IplImage*dst, IplImage*bg) {
// khai bao cac bien trung gian de su dung
IplImage* tmp1 = cvCreateImage(cvSize(src->width, src->height), 8, 1); IplImage* tmp2 = cvCreateImage(cvSize(src->width, src->height), 8, 1); IplImage* tmp3 = cvCreateImage(cvSize(src->width, src->height), 8, 1); //Background Subtraction cvCvtColor(src, tmp1, CV_RGB2GRAY); //cvShowImage("Background Subtraction 1", tmp1); cvAbsDiff(tmp1, bg, tmp2); cvSmooth(tmp2, tmp2, CV_GAUSSIAN, 15); //cvShowImage("Background Subtraction 2", tmp2); cvThreshold(tmp2, tmp3, 30, 255, CV_THRESH_BINARY); //cvShowImage("Background Subtraction 3", tmp3); cvErode(tmp3, dst, cvCreateStructuringElementEx(3, 3, 0, 0, CV_SHAPE_ELLIPSE, 0)); //cvShowImage("Background Subtraction 4", dst); cvDilate(dst, dst, cvCreateStructuringElementEx(5, 5, 0, 0, CV_SHAPE_ELLIPSE, 0)); //cvShowImage("Background Subtraction 5", dst); }
void skin_detection(IplImage*src, IplImage*dst) {
// khai bao thong so trackbar dieu chinh nguong tmp1 cvNamedWindow(window0);
cvCreateTrackbar("MinY", window0, &minY, 255); cvCreateTrackbar("MaxY", window0, &maxY, 255); cvCreateTrackbar("MinU", window0, &minU, 140); cvCreateTrackbar("MaxU", window0, &maxU, 140); cvCreateTrackbar("MinV", window0, &minV, 140); cvCreateTrackbar("MaxV", window0, &maxV, 160); // khai bao bien trung gian
IplImage* tmp1 = cvCreateImage(cvSize(src->width, src->height), 8, 3); IplImage* tmp2 = cvCreateImage(cvSize(src->width, src->height), 8, 1); IplConvKernel* Element1;
IplConvKernel* Element2; //SkinColor
//cvShowImage("Skin Detection S0", src); cvCvtColor(src, tmp1, CV_RGB2YUV); //cvShowImage("Skin Detection S1", tmp1);
cvInRangeS(tmp1, cvScalar(minY, minU, minV), cvScalar(maxY, maxU, maxV), tmp2);
//cvShowImage("Skin Detection S2", tmp2);
Element1 = cvCreateStructuringElementEx(5, 5, 0, 0, CV_SHAPE_ELLIPSE, 0);
Element2 = cvCreateStructuringElementEx(7, 7, 0, 0, CV_SHAPE_ELLIPSE, 0); cvErode(tmp2, dst, Element1); //cvShowImage("Skin Detection S3", dst); cvDilate(dst, dst, Element2); //cvShowImage("Skin Detection S4", dst); }
void hand_detection(IplImage*src, IplImage*dst, IplImage*mask) {
cvErode(mask, mask, cvCreateStructuringElementEx(5, 5, 3, 3, CV_SHAPE_ELLIPSE, 0));
cvDilate(mask, mask, cvCreateStructuringElementEx(7, 7, 6, 6, CV_SHAPE_ELLIPSE, 0)); // BLOB LABELING CBlobLabeling blob; blob.SetParam(mask, 40); // area blob.DoLabeling(); // much time here
blob.BlobSmallSizeConstraint(50, 50); // size min blob.BlobBigSizeConstraint(200, 250); // size max int Hand_Blob_Index = 0;
//only select right most blob
for (int i = 0; i < blob.m_nBlobs; i++) {
if ((blob.m_recBlobs[Hand_Blob_Index].x < blob.m_recBlobs[i].x) && (blob.m_recBlobs[Hand_Blob_Index].y > blob.m_recBlobs[i].y)) { Hand_Blob_Index = i; } } // cut image if (blob.m_nBlobs) {
pt1 = cvPoint((blob.m_recBlobs[Hand_Blob_Index].x), (blob.m_recBlobs[Hand_Blob_Index].y));
int width_new = (blob.m_recBlobs[Hand_Blob_Index].width); int height_new = (blob.m_recBlobs[Hand_Blob_Index].height); IplImage* sub_skin_tmp = cvCreateImage(cvSize(width_new, height_new), 8, 1);
IplImage* sub_skin = cvCreateImage(cvSize(width_new, height_new), 8, 1);
sub_skin->origin = src->origin;
cvSetImageROI(mask, cvRect(pt1.x, pt1.y, width_new, height_new)); cvCopy(mask, sub_skin, 0);
cvResetImageROI(mask); cvZero(mask);
// DISTANT TRANSFORM
IplImage* sub_skin_temp = cvCreateImage(cvSize(width_new, height_new), IPL_DEPTH_32F, 1);
cvDistTransform(sub_skin, sub_skin_temp, CV_DIST_L2, 3); pt.x = pt.y = 0;
pt2.x = pt2.y = 0; float max_DT = 0.0; int radius = 40;
float* data_sub_1 = (float*)sub_skin_temp->imageData; int sub_w = sub_skin_temp->width;
int sub_h = sub_skin_temp->height; int sub_wh = sub_skin_temp->width; for (int j = 0; j < sub_h; j++)
for (int i = 0; i<sub_w; i++) {
// find the max distance transform
if (data_sub_1[j*sub_wh + i] > max_DT) {
max_DT = data_sub_1[j*sub_wh + i]; // hand center pt.x = i; pt.y = j; } } // REMOVE WRIST
uchar* data_sub_2 = (uchar*)sub_skin->imageData; sub_wh = sub_skin->widthStep;
for (int j = 0; j<sub_h; j++)
for (int i = 0; i < sub_w; i++) { if (data_sub_2[j*sub_wh + i] == 255) { if (pt2.x == 0 && pt2.y == 0) { pt2.x = i; pt2.y = j; } //ĐK dưới hình tròn thì xóa if (j >(pt.y + radius)) { data_sub_2[j*sub_wh + i] = 0; } } }
cvSetImageROI(mask, cvRect(pt1.x, pt1.y, width_new, height_new)); cvCopy(sub_skin, mask, 0);
cvResetImageROI(mask);
// show Distant transform as a skeleton of hand
cvNormalize(sub_skin_temp, sub_skin_temp, 0.0, 1.0, CV_MINMAX); cvShowImage("Skeleton", sub_skin_temp);
// pt is the center
// pt1 is the top left of blob // pt2 is the farthest point pt2.x = pt1.x + pt2.x; pt2.y = pt1.y + pt2.y; pt.x = pt1.x + pt.x; pt.y = pt1.y + pt.y;
//cvCircle(src, pt, 5, CV_RGB(0, 255, 0), 5); } // CREATE HAND cvSetZero(dst); cvCopy(src, dst, mask); //cvShowImage("result", dst); //cvShowImage("mask", mask); }
void fingers_counting(IplImage* src, IplImage* dst_hand, IplImage* dst) {
CvSeq* maxitem = NULL; double area = 0, areamax = 0; CvPoint pt3;
int maxn = 0;
int Nc = cvFindContours(src, storage, &first_contour, sizeof(CvContour), CV_RETR_LIST);
int n = 0; CvFont font;
//printf("Total Contours Detected: %d\n", Nc); if (Nc>0)
{
for (CvSeq* c = first_contour; c != NULL; c = c->h_next) {
area = cvContourArea(c, CV_WHOLE_SEQ); if (area>areamax) { areamax = area; maxitem = c; maxn = n; } n++; }
CvMemStorage* storage3 = cvCreateMemStorage(0); // the area should be larger than a fixed value
if (areamax>1000) {
maxitem = cvApproxPoly(maxitem, sizeof(CvContour), storage3, CV_POLY_APPROX_DP, 10, 1);
CvPoint pt0; CvPoint end_pt;
CvMemStorage* storage1 = cvCreateMemStorage(0); CvMemStorage* storage2 = cvCreateMemStorage(0); CvSeq* ptseq = cvCreateSeq(CV_SEQ_KIND_GENERIC | CV_32SC2, sizeof(CvContour), sizeof(CvPoint), storage1);
CvSeq* hull; CvSeq* defects; //
for (int i = 0; i < maxitem->total; i++) {
CvPoint* p = CV_GET_SEQ_ELEM(CvPoint, maxitem, i);
pt0.x = p->x; pt0.y = p->y;
cvSeqPush(ptseq, &pt0); }
pt3.x = 0; pt3.y = 0; //
for (int x = 0; x<hull->total; x++) {
CvPoint hull_pt = **CV_GET_SEQ_ELEM(CvPoint*, hull, x); if (pt3.x == 0 && pt3.y == 0) { pt3 = hull_pt; end_pt = pt3; } // draw on mask cvLine(dst_hand, pt3, hull_pt, CV_RGB(255, 0, 0), 1); if (x == hull->total - 1) cvLine(dst_hand, hull_pt, end_pt, CV_RGB(255, 0, 0), 1);
// draw on frame
CvPoint pt3_f, hull_pt_f, end_pt_f; pt3_f.x = pt3.x * 2; pt3_f.y = pt3.y * 2; hull_pt_f.x = hull_pt.x * 2; hull_pt_f.y = hull_pt.y * 2; end_pt_f.x = end_pt.x * 2; end_pt_f.y = end_pt.y * 2; cvLine(dst, pt3_f, hull_pt_f, CV_RGB(255, 0, 0), 1); pt3 = hull_pt; // draw on frame
if (x == hull->total - 1) cvLine(dst, hull_pt_f, end_pt_f, CV_RGB(255, 0, 0), 1);
}
int hullcount = hull->total;
defects = cvConvexityDefects(ptseq, hull, storage2); //printf(" defect no %d \n", defects->total);
CvConvexityDefect* defectArray; num_fingers = 1;
// This cycle marks all defects of convexity of current contours. for (; defects; defects = defects->h_next)
{
int nomdef = defects->total; // defect amount if (nomdef == 0)
continue;
// Alloc memory for defect set. defectArray =
cvCvtSeqToArray(defects, defectArray, CV_WHOLE_SEQ);
// Draw marks for all defects. for (int i = 0; i<nomdef; i++) {
if (defectArray[i].depth > threshold_fingers) {
num_fingers++;
//printf(" defect depth for defect %d %f \n",i,defectArray[i].depth); cvCircle(dst_hand, *(defectArray[i].start), 5, CV_RGB(255, 0, 0), 2, 8, 0); cvCircle(dst_hand, *(defectArray[i].depth_point), 5, CV_RGB(0, 0, 255), 2, 8, 0); cvLine(dst_hand, *(defectArray[i].start), *(defectArray[i].depth_point), CV_RGB(255, 255, 0), 1, CV_AA, 0); cvLine(dst_hand, *(defectArray[i].depth_point), *(defectArray[i].end), CV_RGB(255, 255, 0), 1, CV_AA, 0); } } // Free memory. free(defectArray); } // In case Num = 0
double fDist = sqrt((double)((pt.x - pt2.x)*(pt.x - pt2.x) + (pt.y - pt2.y)*(pt.y - pt2.y)));
if (num_fingers == 1 && fDist < no_fingers) num_fingers = 0; // draw on hand
cvLine(dst_hand, pt, pt2, CV_RGB(255, 255, 0), 2);
cvCircle(dst_hand, cvPoint(pt.x, pt.y), 4, CV_RGB(0, 255, 0), 2); cvCircle(dst_hand, cvPoint(pt2.x, pt2.y), 4, CV_RGB(0, 0, 255), 2); // draw on frame CvPoint pt_f; pt_f.x = pt.x * 2; pt_f.y = pt.y * 2; pt2_f.x = pt2.x * 2; pt2_f.y = pt2.y * 2; cvLine(dst, pt_f, pt2_f, CV_RGB(255, 255, 0), 2);
cvCircle(dst, cvPoint(pt_f.x, pt_f.y), 4, CV_RGB(0, 255, 0), 2); cvCircle(dst, cvPoint(pt2_f.x, pt2_f.y), 4, CV_RGB(0, 0, 255), 2); //printf("pointpt2: %d\n", pt2_f);
// print num of finger on the screen char txt[] = "0";
txt[0] = '0' + num_fingers;
cvInitFont(&font, CV_FONT_HERSHEY_DUPLEX, 1.0, 1.0, 0, 2, CV_AA);
cvPutText(dst, txt, cvPoint(50, 50), &font, cvScalar(0, 0, 255, 0)); // release Storage cvReleaseMemStorage(&storage); cvReleaseMemStorage(&storage1); cvReleaseMemStorage(&storage2); cvReleaseMemStorage(&storage3); } } }
TÀI LIỆU THAM KHẢO 1. Giáo trình
[1] Trần Hoàn, Trích “ LUẬN VĂN THẠC SĨ”, Trường Đại học Bách khoa TPHCM [2] Abdullah Hasan AlSaadi et al. ,“Analysis of different background subtraction methods applied on Drone imagery under various weather conditions in the UAE region”, Tawazun Technology and Innovation (TTI), Khalifa University, Abu Dhabi, United Arab Emirates.
[3] S. Kolkur et al. ,” Human Skin Detection Using RGB, HSV and YCbCr Color Models”, Department of Computer Engineering, Sardar Patel Institute of Technology, Andheri,Mumbai, India.
2. Trang web
[1] http://hano.cf/ (trang hướng dẫn cài đặt opencv + visual + tài liệu)
[3] https://funvision.blogspot.com/ (Trang hướng dẫn xử lý ảnh)
[4] https://docs.opencv.org/2.4/index.html (Trang web cấu trúc mẫu code C/C++) [5] https://abnerrjo.github.io/ (Trang web cấu trúc mẫu code C/C++)