Skip to content

mediapipe_face_captor

mediapipe_face_captor

Wrapper for mediapipe face capture.

Example

import fast_cody as fcd
import time
captor = fcd.mediapipe_face_captor()
for i in range(1000):
    [R, info] = captor.query_rotation()
    captor.imshow()
    print(R)
captor.release()
Source code in src\fast_cody\mediapipe_face_captor.py 
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
class mediapipe_face_captor():
    """ Wrapper for mediapipe face capture.


    Example
    -------
    ```
    import fast_cody as fcd
    import time
    captor = fcd.mediapipe_face_captor()
    for i in range(1000):
        [R, info] = captor.query_rotation()
        captor.imshow()
        print(R)
    captor.release()
    ```
    """
    def __init__(self, R0=None, draw_landmarks=False):
        """
        Parameters
        ----------
        R0 : 3x3 float numpy array
            Initial rotation matrix used when face is not detected.
        draw_landmarks : bool
            Whether to draw landmarks on the image. Slows down the app if true
        """
        if R0 is None:
            R0 = np.identity(3)
        self.R0 = R0
        self.draw_landmarks = draw_landmarks

        self.mp_face_mesh = mp.solutions.face_mesh
        self.mp_drawing = mp.solutions.drawing_utils
        self.mp_drawing_styles = mp.solutions.drawing_styles
        self.mp_face_mesh = mp.solutions.face_mesh
        self.cap = cv2.VideoCapture(0)
        self.calibrated = False  # used to pick rest positions used for rotaiton fitting
        self.PX = None  # initial landmark config filled up after first frame of detected face
        self.filter = OneEuroFilter(R0)

        self.face_control_power = 10

        # dynamic member variables whose quantities are changed throughout app
        self.image = None
        self.multi_face_landmarks = None



    def query_rotation(s):
        """
        Returns
        -------
        R : 3x3 float numpy array
            Current rotation matrix.
        info : dict
            Dictionary containing 'image', 'multi_face_landmarks', and a 'calibrated' key.
        """
        R = s.R0
        with s.mp_face_mesh.FaceMesh(
            max_num_faces=1,
            refine_landmarks=True,
            min_detection_confidence=0.5,
            min_tracking_confidence=0.5) as face_mesh:

                if s.cap.isOpened():
                    success, image = s.cap.read()
                    if not success:
                        print("Ignoring empty camera frame.")
                        # If loading a video, use 'break' instead of 'continue'.
                        return
                    # To improve performance, optionally mark the image as not writeable to
                    # pass by reference.
                    image.flags.writeable = False
                    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
                    results = face_mesh.process(image)

                    # Draw the face mesh annotations on the image.
                    image.flags.writeable = True
                    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
                    s.image = image
                    s.multi_face_landmarks = results.multi_face_landmarks

                    if s.multi_face_landmarks:
                        P = face_landmarks_to_positions(s.multi_face_landmarks[0])
                        if not s.calibrated:
                            s.calibrated = True
                            s.PX = P - P.mean(axis=0)
                        K = P - P.mean(axis=0)
                        [R, S] = igl.polar_dec(K.T @ s.PX);
                        R = s.filter(R.T)

                info = {'calibrated': s.calibrated, 'landmarks':s.multi_face_landmarks, 'image':s.image}

                return R, info

    '''
    displays the image
    '''
    def imshow(s):
        """
        Displays the image on the screen. If self.draw_landmarks is True, also draws landmarks on the face.
        """
        if s.cap.isOpened():
            if s.draw_landmarks:
                if s.multi_face_landmarks:
                    for face_landmarks in s.multi_face_landmarks:
                        s.mp_drawing.draw_landmarks(
                            image=s.image,
                            landmark_list=face_landmarks,
                            connections=s.mp_face_mesh.FACEMESH_TESSELATION,
                            landmark_drawing_spec=None,
                            connection_drawing_spec=s.mp_drawing_styles
                            .get_default_face_mesh_tesselation_style())
                        s.mp_drawing.draw_landmarks(
                            image=s.image,
                            landmark_list=face_landmarks,
                            connections=s.mp_face_mesh.FACEMESH_CONTOURS,
                            landmark_drawing_spec=None,
                            connection_drawing_spec=s.mp_drawing_styles
                            .get_default_face_mesh_contours_style())
                        s.mp_drawing.draw_landmarks(
                            image=s.image,
                            landmark_list=face_landmarks,
                            connections=s.mp_face_mesh.FACEMESH_IRISES,
                            landmark_drawing_spec=None,
                            connection_drawing_spec=s.mp_drawing_styles
                            .get_default_face_mesh_iris_connections_style())


            cv2.imshow('MediaPipe Face Detection', cv2.flip(s.image, 1))
            cv2.waitKey(1)
            # cv2.waitKey()
        # if (record):
        #     cv2.imwrite(results_dir + "/camera_stream/" + str(step).zfill(4) + ".png", image)
        #


    def release(self):
        """ Releases the camera.
        """
        self.cap.release()

__init__(R0=None, draw_landmarks=False)

Parameters:

Name Type Description Default
R0 3x3 float numpy array

Initial rotation matrix used when face is not detected.

 None
draw_landmarks bool

Whether to draw landmarks on the image. Slows down the app if true

 False
Source code in src\fast_cody\mediapipe_face_captor.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
def __init__(self, R0=None, draw_landmarks=False):
    """
    Parameters
    ----------
    R0 : 3x3 float numpy array
        Initial rotation matrix used when face is not detected.
    draw_landmarks : bool
        Whether to draw landmarks on the image. Slows down the app if true
    """
    if R0 is None:
        R0 = np.identity(3)
    self.R0 = R0
    self.draw_landmarks = draw_landmarks

    self.mp_face_mesh = mp.solutions.face_mesh
    self.mp_drawing = mp.solutions.drawing_utils
    self.mp_drawing_styles = mp.solutions.drawing_styles
    self.mp_face_mesh = mp.solutions.face_mesh
    self.cap = cv2.VideoCapture(0)
    self.calibrated = False  # used to pick rest positions used for rotaiton fitting
    self.PX = None  # initial landmark config filled up after first frame of detected face
    self.filter = OneEuroFilter(R0)

    self.face_control_power = 10

    # dynamic member variables whose quantities are changed throughout app
    self.image = None
    self.multi_face_landmarks = None

imshow(s)

Displays the image on the screen. If self.draw_landmarks is True, also draws landmarks on the face.

Source code in src\fast_cody\mediapipe_face_captor.py 
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def imshow(s):
    """
    Displays the image on the screen. If self.draw_landmarks is True, also draws landmarks on the face.
    """
    if s.cap.isOpened():
        if s.draw_landmarks:
            if s.multi_face_landmarks:
                for face_landmarks in s.multi_face_landmarks:
                    s.mp_drawing.draw_landmarks(
                        image=s.image,
                        landmark_list=face_landmarks,
                        connections=s.mp_face_mesh.FACEMESH_TESSELATION,
                        landmark_drawing_spec=None,
                        connection_drawing_spec=s.mp_drawing_styles
                        .get_default_face_mesh_tesselation_style())
                    s.mp_drawing.draw_landmarks(
                        image=s.image,
                        landmark_list=face_landmarks,
                        connections=s.mp_face_mesh.FACEMESH_CONTOURS,
                        landmark_drawing_spec=None,
                        connection_drawing_spec=s.mp_drawing_styles
                        .get_default_face_mesh_contours_style())
                    s.mp_drawing.draw_landmarks(
                        image=s.image,
                        landmark_list=face_landmarks,
                        connections=s.mp_face_mesh.FACEMESH_IRISES,
                        landmark_drawing_spec=None,
                        connection_drawing_spec=s.mp_drawing_styles
                        .get_default_face_mesh_iris_connections_style())


        cv2.imshow('MediaPipe Face Detection', cv2.flip(s.image, 1))
        cv2.waitKey(1)

query_rotation(s)

Returns:

Name Type Description
R 3x3 float numpy array

Current rotation matrix.
info dict

Dictionary containing 'image', 'multi_face_landmarks', and a 'calibrated' key.
Source code in src\fast_cody\mediapipe_face_captor.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def query_rotation(s):
    """
    Returns
    -------
    R : 3x3 float numpy array
        Current rotation matrix.
    info : dict
        Dictionary containing 'image', 'multi_face_landmarks', and a 'calibrated' key.
    """
    R = s.R0
    with s.mp_face_mesh.FaceMesh(
        max_num_faces=1,
        refine_landmarks=True,
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5) as face_mesh:

            if s.cap.isOpened():
                success, image = s.cap.read()
                if not success:
                    print("Ignoring empty camera frame.")
                    # If loading a video, use 'break' instead of 'continue'.
                    return
                # To improve performance, optionally mark the image as not writeable to
                # pass by reference.
                image.flags.writeable = False
                image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
                results = face_mesh.process(image)

                # Draw the face mesh annotations on the image.
                image.flags.writeable = True
                image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
                s.image = image
                s.multi_face_landmarks = results.multi_face_landmarks

                if s.multi_face_landmarks:
                    P = face_landmarks_to_positions(s.multi_face_landmarks[0])
                    if not s.calibrated:
                        s.calibrated = True
                        s.PX = P - P.mean(axis=0)
                    K = P - P.mean(axis=0)
                    [R, S] = igl.polar_dec(K.T @ s.PX);
                    R = s.filter(R.T)

            info = {'calibrated': s.calibrated, 'landmarks':s.multi_face_landmarks, 'image':s.image}

            return R, info

release()

Releases the camera.

Source code in src\fast_cody\mediapipe_face_captor.py 
167
168
169
170
def release(self):
    """ Releases the camera.
    """
    self.cap.release()