diff --git a/Programs/Eye Tracker Using mediapipe library b/Programs/Eye Tracker Using mediapipe library new file mode 100644 index 0000000..e7ccf6e --- /dev/null +++ b/Programs/Eye Tracker Using mediapipe library @@ -0,0 +1,244 @@ + +import cv2 as cv +import mediapipe as mp +import time +import utils, math +import numpy as np +# variables +frame_counter =0 +CEF_COUNTER =0 +TOTAL_BLINKS =0 +# constants +CLOSED_EYES_FRAME =3 +FONTS =cv.FONT_HERSHEY_COMPLEX + +# face bounder indices +FACE_OVAL=[ 10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, 127, 162, 21, 54, 103,67, 109] + +# lips indices for Landmarks +LIPS=[ 61, 146, 91, 181, 84, 17, 314, 405, 321, 375,291, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95,185, 40, 39, 37,0 ,267 ,269 ,270 ,409, 415, 310, 311, 312, 13, 82, 81, 42, 183, 78 ] +LOWER_LIPS =[61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95] +UPPER_LIPS=[ 185, 40, 39, 37,0 ,267 ,269 ,270 ,409, 415, 310, 311, 312, 13, 82, 81, 42, 183, 78] +# Left eyes indices +LEFT_EYE =[ 362, 382, 381, 380, 374, 373, 390, 249, 263, 466, 388, 387, 386, 385,384, 398 ] +LEFT_EYEBROW =[ 336, 296, 334, 293, 300, 276, 283, 282, 295, 285 ] + +# right eyes indices +RIGHT_EYE=[ 33, 7, 163, 144, 145, 153, 154, 155, 133, 173, 157, 158, 159, 160, 161 , 246 ] +RIGHT_EYEBROW=[ 70, 63, 105, 66, 107, 55, 65, 52, 53, 46 ] + +map_face_mesh = mp.solutions.face_mesh +# camera object +camera = cv.VideoCapture(0) +# landmark detection function +def landmarksDetection(img, results, draw=False): + img_height, img_width= img.shape[:2] + # list[(x,y), (x,y)....] + mesh_coord = [(int(point.x * img_width), int(point.y * img_height)) for point in results.multi_face_landmarks[0].landmark] + if draw : + [cv.circle(img, p, 2, (0,255,0), -1) for p in mesh_coord] + + # returning the list of tuples for each landmarks + return mesh_coord + +# Euclaidean distance +def euclaideanDistance(point, point1): + x, y = point + x1, y1 = point1 + distance = math.sqrt((x1 - x)**2 + (y1 - y)**2) + return distance + +# Blinking Ratio +def blinkRatio(img, landmarks, right_indices, left_indices): + # Right eyes + # horizontal line + rh_right = landmarks[right_indices[0]] + rh_left = landmarks[right_indices[8]] + # vertical line + rv_top = landmarks[right_indices[12]] + rv_bottom = landmarks[right_indices[4]] + # draw lines on right eyes + # cv.line(img, rh_right, rh_left, utils.GREEN, 2) + # cv.line(img, rv_top, rv_bottom, utils.WHITE, 2) + + # LEFT_EYE + # horizontal line + lh_right = landmarks[left_indices[0]] + lh_left = landmarks[left_indices[8]] + + # vertical line + lv_top = landmarks[left_indices[12]] + lv_bottom = landmarks[left_indices[4]] + + rhDistance = euclaideanDistance(rh_right, rh_left) + rvDistance = euclaideanDistance(rv_top, rv_bottom) + + lvDistance = euclaideanDistance(lv_top, lv_bottom) + lhDistance = euclaideanDistance(lh_right, lh_left) + + reRatio = rhDistance/rvDistance + leRatio = lhDistance/lvDistance + + ratio = (reRatio+leRatio)/2 + return ratio + +# Eyes Extrctor function, +def eyesExtractor(img, right_eye_coords, left_eye_coords): + # converting color image to scale image + gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY) + + # getting the dimension of image + dim = gray.shape + + # creating mask from gray scale dim + mask = np.zeros(dim, dtype=np.uint8) + + # drawing Eyes Shape on mask with white color + cv.fillPoly(mask, [np.array(right_eye_coords, dtype=np.int32)], 255) + cv.fillPoly(mask, [np.array(left_eye_coords, dtype=np.int32)], 255) + + # showing the mask + # cv.imshow('mask', mask) + + # draw eyes image on mask, where white shape is + eyes = cv.bitwise_and(gray, gray, mask=mask) + # change black color to gray other than eys + # cv.imshow('eyes draw', eyes) + eyes[mask==0]=155 + + # getting minium and maximum x and y for right and left eyes + # For Right Eye + r_max_x = (max(right_eye_coords, key=lambda item: item[0]))[0] + r_min_x = (min(right_eye_coords, key=lambda item: item[0]))[0] + r_max_y = (max(right_eye_coords, key=lambda item : item[1]))[1] + r_min_y = (min(right_eye_coords, key=lambda item: item[1]))[1] + + # For LEFT Eye + l_max_x = (max(left_eye_coords, key=lambda item: item[0]))[0] + l_min_x = (min(left_eye_coords, key=lambda item: item[0]))[0] + l_max_y = (max(left_eye_coords, key=lambda item : item[1]))[1] + l_min_y = (min(left_eye_coords, key=lambda item: item[1]))[1] + + # croping the eyes from mask + cropped_right = eyes[r_min_y: r_max_y, r_min_x: r_max_x] + cropped_left = eyes[l_min_y: l_max_y, l_min_x: l_max_x] + + # returning the cropped eyes + return cropped_right, cropped_left + +# Eyes Postion Estimator +def positionEstimator(cropped_eye): + # getting height and width of eye + h, w =cropped_eye.shape + + # remove the noise from images + gaussain_blur = cv.GaussianBlur(cropped_eye, (9,9),0) + median_blur = cv.medianBlur(gaussain_blur, 3) + + # applying thrsholding to convert binary_image + ret, threshed_eye = cv.threshold(median_blur, 130, 255, cv.THRESH_BINARY) + + # create fixd part for eye with + piece = int(w/3) + + # slicing the eyes into three parts + right_piece = threshed_eye[0:h, 0:piece] + center_piece = threshed_eye[0:h, piece: piece+piece] + left_piece = threshed_eye[0:h, piece +piece:w] + + # calling pixel counter function + eye_position, color = pixelCounter(right_piece, center_piece, left_piece) + + return eye_position, color + +# creating pixel counter function +def pixelCounter(first_piece, second_piece, third_piece): + # counting black pixel in each part + right_part = np.sum(first_piece==0) + center_part = np.sum(second_piece==0) + left_part = np.sum(third_piece==0) + # creating list of these values + eye_parts = [right_part, center_part, left_part] + + # getting the index of max values in the list + max_index = eye_parts.index(max(eye_parts)) + pos_eye ='' + if max_index==0: + pos_eye="RIGHT" + color=[utils.BLACK, utils.GREEN] + elif max_index==1: + pos_eye = 'CENTER' + color = [utils.YELLOW, utils.PINK] + elif max_index ==2: + pos_eye = 'LEFT' + color = [utils.GRAY, utils.YELLOW] + else: + pos_eye="Closed" + color = [utils.GRAY, utils.YELLOW] + return pos_eye, color + + +with map_face_mesh.FaceMesh(min_detection_confidence =0.5, min_tracking_confidence=0.5) as face_mesh: + + # starting time here + start_time = time.time() + # starting Video loop here. + while True: + frame_counter +=1 # frame counter + ret, frame = camera.read() # getting frame from camera + if not ret: + break # no more frames break + # resizing frame + + frame = cv.resize(frame, None, fx=1.5, fy=1.5, interpolation=cv.INTER_CUBIC) + frame_height, frame_width= frame.shape[:2] + rgb_frame = cv.cvtColor(frame, cv.COLOR_RGB2BGR) + results = face_mesh.process(rgb_frame) + if results.multi_face_landmarks: + mesh_coords = landmarksDetection(frame, results, False) + ratio = blinkRatio(frame, mesh_coords, RIGHT_EYE, LEFT_EYE) + # cv.putText(frame, f'ratio {ratio}', (100, 100), FONTS, 1.0, utils.GREEN, 2) + utils.colorBackgroundText(frame, f'Ratio : {round(ratio,2)}', FONTS, 0.7, (30,100),2, utils.PINK, utils.YELLOW) + + if ratio >5.5: + CEF_COUNTER +=1 + # cv.putText(frame, 'Blink', (200, 50), FONTS, 1.3, utils.PINK, 2) + utils.colorBackgroundText(frame, f'Blink', FONTS, 1.7, (int(frame_height/2), 100), 2, utils.YELLOW, pad_x=6, pad_y=6, ) + + else: + if CEF_COUNTER>CLOSED_EYES_FRAME: + TOTAL_BLINKS +=1 + CEF_COUNTER =0 + # cv.putText(frame, f'Total Blinks: {TOTAL_BLINKS}', (100, 150), FONTS, 0.6, utils.GREEN, 2) + utils.colorBackgroundText(frame, f'Total Blinks: {TOTAL_BLINKS}', FONTS, 0.7, (30,150),2) + + cv.polylines(frame, [np.array([mesh_coords[p] for p in LEFT_EYE ], dtype=np.int32)], True, utils.GREEN, 1, cv.LINE_AA) + cv.polylines(frame, [np.array([mesh_coords[p] for p in RIGHT_EYE ], dtype=np.int32)], True, utils.GREEN, 1, cv.LINE_AA) + + # Blink Detector Counter Completed + right_coords = [mesh_coords[p] for p in RIGHT_EYE] + left_coords = [mesh_coords[p] for p in LEFT_EYE] + crop_right, crop_left = eyesExtractor(frame, right_coords, left_coords) + # cv.imshow('right', crop_right) + # cv.imshow('left', crop_left) + eye_position, color = positionEstimator(crop_right) + utils.colorBackgroundText(frame, f'R: {eye_position}', FONTS, 1.0, (40, 220), 2, color[0], color[1], 8, 8) + eye_position_left, color = positionEstimator(crop_left) + utils.colorBackgroundText(frame, f'L: {eye_position_left}', FONTS, 1.0, (40, 320), 2, color[0], color[1], 8, 8) + + + + + # calculating frame per seconds FPS + end_time = time.time()-start_time + fps = frame_counter/end_time + + frame =utils.textWithBackground(frame,f'FPS: {round(fps,1)}',FONTS, 1.0, (30, 50), bgOpacity=0.9, textThickness=2) + # writing image for thumbnail drawing shape + # cv.imwrite(f'img/frame_{frame_counter}.png', frame) + cv.imshow('frame', frame) + key = cv.waitKey(2) + if key==ord('q') or key ==ord('Q'): + break + cv.destroyAllWindows() + camera.release()