如何使Mediapipe的手部检测适用于戴手套的手。

huangapple go评论90阅读模式
英文:

How to Make the Hand Detection of Mediapipe Work on Hand with Gloves

问题

我尝试让Mediapipe的手部检测在实时环境中识别戴着蓝色手套的手,但它没有正常工作。如果手套的颜色与肤色相似,检测仍然可以正常工作。因此,我尝试进行一些预处理,将帧中的蓝色像素更改为肉色。结果是它不能准确地检测手部 - 有时可以,但之后会消失。

请帮助我修复这个问题。我在某处读到这个手部检测可以通过将手套的颜色更改为肤色并保留手的阴影来在实时环境中识别蓝色手套,但我不知道如何正确做。如果您能帮助我,我将非常感激 T-T。

def findHands(self, img, draw=True, flipType=True):
 imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

 blue  = [0,0,255]
 nude=[225, 190, 160]

 #生成掩码
 Bmask = np.all(imgRGB == blue, axis=-1)

 #将蓝色像素替换为肉色
 imgRGB[Bmask] = nude

 img_res=imgRGB

 #将帧发送到Mediapipe        
 self.results = self.hands.process(img_res)
英文:

I was trying to make the hand detection of mediapipe to work on hands with blue gloves in real time. But it was not working properly. The detection can still work on gloves if the color is similar to skintone. So, I tried to do some pre-processing wherein I was changing the blue pixels found on the frame into nude color. The result was it cannot detect the hand accurately - sometimes it can but it would later disappear.

Please help me fix this. I have read somewhere that this hand detection can work on blue gloves in real time by changing the color of the glove to skin tone while retaining the shading of the hand. But I have no idea on how to do that properly. I would deeply appreciate it if you can help me T-T.

def findHands(self, img, draw=True, flipType=True):
 imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

 blue  = [0,0,255]
 nude=[225, 190, 160]

 #Make mask 
 Bmask = np.all(imgRGB == blue, axis=-1)

 #Replace blue pixel into nude
 imgRGB[Bmask] = nude

 img_res=imgRGB

 #Send frame to mediapipe        
 self.results = self.hands.process(img_res)

答案1

得分: 0

最简单的解决方案是省略cv2.cvtColor(),这将导致图像通道交换,从而使blue看起来像orange。以下是一个示例(适用于mediapipe==0.9.0):

import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

with mp_hands.Hands(
    static_image_mode=True,
    max_num_hands=2,
    min_detection_confidence=0.5) as hands:
        image = cv2.imread("gloves.jpg")
        results = hands.process(image)

        if results.multi_hand_landmarks:
            image_height, image_width, _ = image.shape
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(
                    image,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())
        else:
            print("no hands were found")
                
        cv2.imshow("result", image)
        cv2.waitKey(0)

这应该足以让MediaPipe识别手部:

如何使Mediapipe的手部检测适用于戴手套的手。

或者,您还可以使用HSV模式中的色调偏移进行颜色更改的微调(改编自此处)。这将是一个“更好”的解决方案,因为它仅更改色调,而不交换通道。

编辑: 我创建了一个简单的色调调整“GUI样式”脚本,可帮助找到合适的色调偏移:

import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

window_title = 'Recognition result'
trackbar_title = 'Hue offset'

class HueHelper:
    def __init__(self):
        self.mp_hands = mp_hands.Hands(static_image_mode=True,
                                        max_num_hands=2,
                                        min_detection_confidence=0.5)
        self.img_bgr = None

    def apply_hue_offset(self,image, hue_offset):# 0 is no change; 0<=huechange<=180
        # convert img to hsv
        img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
        h = img_hsv[:,:,0]
        s = img_hsv[:,:,1]
        v = img_hsv[:,:,2]
        # shift the hue
        # cv2 will clip automatically to avoid color wrap-around
        img_hsv = cv2.add(h, hue_offset)
        # combine new hue with s and v
        img_hsv = cv2.merge([img_hsv,s,v])
        # convert from HSV to BGR
        return cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR)

    def on_trackbar_change(self, trackbar_hue_offset):
        img_bgr_modified = self.recognize(self.img_bgr.copy(), trackbar_hue_offset)
        cv2.imshow(window_title, img_bgr_modified)

    def recognize(self, img_bgr, hue_offset):
        img_bgr = self.apply_hue_offset(img_bgr, hue_offset)
        img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
        results = self.mp_hands.process(img_rgb)

        if results.multi_hand_landmarks:
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(
                    img_bgr,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())
        else:
            print('no hands were found')
        return img_bgr
    
    def run(self, img_path):
        self.img_bgr = cv2.imread(img_path)
        if self.img_bgr is None: print('Image was not found!')
        self.on_trackbar_change(0)
        # Hue range is 0-179: https://docs.opencv.org/4.x/df/d9d/tutorial_py_colorspaces.html
        cv2.createTrackbar(trackbar_title, window_title, 0, 179, self.on_trackbar_change)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

if __name__ == '__main__':
    h = HueHelper()
    h.run('gloves.jpg')

再次演示:

如何使Mediapipe的手部检测适用于戴手套的手。

英文:

The simplest solution is to omit the cv2.cvtColor(), which will cause the image channels to swap, effectively making blue look like orange. Here is an example (for mediapipe==0.9.0):

import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands

with mp_hands.Hands(
    static_image_mode=True,
    max_num_hands=2,
    min_detection_confidence=0.5) as hands:
        image = cv2.imread(&quot;gloves.jpg&quot;)
        results = hands.process(image)

        if results.multi_hand_landmarks:
            image_height, image_width, _ = image.shape
            for hand_landmarks in results.multi_hand_landmarks:
                mp_drawing.draw_landmarks(
                    image,
                    hand_landmarks,
                    mp_hands.HAND_CONNECTIONS,
                    mp_drawing_styles.get_default_hand_landmarks_style(),
                    mp_drawing_styles.get_default_hand_connections_style())
        else:
            print(&quot;no hands were found&quot;)
                
        cv2.imshow(&quot;result&quot;, image)
        cv2.waitKey(0)

This should be sufficient for MediaPipe to recognize the hands:

如何使Mediapipe的手部检测适用于戴手套的手。

Alternatively, you can also make a fine-tuning of the color change using Hue offset in HSV schema (adapted from here). This would be a "better" solution since it only changes the hue but does not swap the channels.

Edit: I've created a simple Hue-tunning "GUI-like" script that helps to find suitable Hue-offset:

import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands
window_title = &#39;Recognition result&#39;
trackbar_title = &#39;Hue offset&#39;
class HueHelper:
def __init__(self):
self.mp_hands = mp_hands.Hands(static_image_mode=True,
max_num_hands=2,
min_detection_confidence=0.5)
self.img_bgr = None
def apply_hue_offset(self,image, hue_offset):# 0 is no change; 0&lt;=huechange&lt;=180
# convert img to hsv
img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
h = img_hsv[:,:,0]
s = img_hsv[:,:,1]
v = img_hsv[:,:,2]
# shift the hue
# cv2 will clip automatically to avoid color wrap-around
img_hsv = cv2.add(h, hue_offset)
# combine new hue with s and v
img_hsv = cv2.merge([img_hsv,s,v])
# convert from HSV to BGR
return cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR)
def on_trackbar_change(self, trackbar_hue_offset):
img_bgr_modified = self.recognize(self.img_bgr.copy(), trackbar_hue_offset)
cv2.imshow(window_title, img_bgr_modified)
def recognize(self, img_bgr, hue_offset):
img_bgr = self.apply_hue_offset(img_bgr, hue_offset)
img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
results = self.mp_hands.process(img_rgb)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(
img_bgr,
hand_landmarks,
mp_hands.HAND_CONNECTIONS,
mp_drawing_styles.get_default_hand_landmarks_style(),
mp_drawing_styles.get_default_hand_connections_style())
else:
print(&#39;no hands were found&#39;)
return img_bgr
def run(self, img_path):
self.img_bgr = cv2.imread(img_path)
if self.img_bgr is None: print(&#39;Image was not found!&#39;)
self.on_trackbar_change(0)
# Hue range is 0-179: https://docs.opencv.org/4.x/df/d9d/tutorial_py_colorspaces.html
cv2.createTrackbar(trackbar_title, window_title, 0, 179, self.on_trackbar_change)
cv2.waitKey(0)
cv2.destroyAllWindows()
if __name__ == &#39;__main__&#39;:
h = HueHelper()
h.run(&#39;gloves.jpg&#39;)

Again, a demo:

如何使Mediapipe的手部检测适用于戴手套的手。

huangapple
  • 本文由 发表于 2023年5月25日 01:16:21
  • 转载请务必保留本文链接:https://go.coder-hub.com/76325975.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定