英文:
How to Make the Hand Detection of Mediapipe Work on Hand with Gloves
问题
我尝试让Mediapipe的手部检测在实时环境中识别戴着蓝色手套的手,但它没有正常工作。如果手套的颜色与肤色相似,检测仍然可以正常工作。因此,我尝试进行一些预处理,将帧中的蓝色像素更改为肉色。结果是它不能准确地检测手部 - 有时可以,但之后会消失。
请帮助我修复这个问题。我在某处读到这个手部检测可以通过将手套的颜色更改为肤色并保留手的阴影来在实时环境中识别蓝色手套,但我不知道如何正确做。如果您能帮助我,我将非常感激 T-T。
def findHands(self, img, draw=True, flipType=True):
imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
blue = [0,0,255]
nude=[225, 190, 160]
#生成掩码
Bmask = np.all(imgRGB == blue, axis=-1)
#将蓝色像素替换为肉色
imgRGB[Bmask] = nude
img_res=imgRGB
#将帧发送到Mediapipe
self.results = self.hands.process(img_res)
英文:
I was trying to make the hand detection of mediapipe to work on hands with blue gloves in real time. But it was not working properly. The detection can still work on gloves if the color is similar to skintone. So, I tried to do some pre-processing wherein I was changing the blue pixels found on the frame into nude color. The result was it cannot detect the hand accurately - sometimes it can but it would later disappear.
Please help me fix this. I have read somewhere that this hand detection can work on blue gloves in real time by changing the color of the glove to skin tone while retaining the shading of the hand. But I have no idea on how to do that properly. I would deeply appreciate it if you can help me T-T.
def findHands(self, img, draw=True, flipType=True):
imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
blue = [0,0,255]
nude=[225, 190, 160]
#Make mask
Bmask = np.all(imgRGB == blue, axis=-1)
#Replace blue pixel into nude
imgRGB[Bmask] = nude
img_res=imgRGB
#Send frame to mediapipe
self.results = self.hands.process(img_res)
答案1
得分: 0
最简单的解决方案是省略cv2.cvtColor()
,这将导致图像通道交换,从而使blue
看起来像orange
。以下是一个示例(适用于mediapipe==0.9.0):
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands
with mp_hands.Hands(
static_image_mode=True,
max_num_hands=2,
min_detection_confidence=0.5) as hands:
image = cv2.imread("gloves.jpg")
results = hands.process(image)
if results.multi_hand_landmarks:
image_height, image_width, _ = image.shape
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(
image,
hand_landmarks,
mp_hands.HAND_CONNECTIONS,
mp_drawing_styles.get_default_hand_landmarks_style(),
mp_drawing_styles.get_default_hand_connections_style())
else:
print("no hands were found")
cv2.imshow("result", image)
cv2.waitKey(0)
这应该足以让MediaPipe识别手部:
或者,您还可以使用HSV模式中的色调偏移进行颜色更改的微调(改编自此处)。这将是一个“更好”的解决方案,因为它仅更改色调,而不交换通道。
编辑: 我创建了一个简单的色调调整“GUI样式”脚本,可帮助找到合适的色调偏移:
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands
window_title = 'Recognition result'
trackbar_title = 'Hue offset'
class HueHelper:
def __init__(self):
self.mp_hands = mp_hands.Hands(static_image_mode=True,
max_num_hands=2,
min_detection_confidence=0.5)
self.img_bgr = None
def apply_hue_offset(self,image, hue_offset):# 0 is no change; 0<=huechange<=180
# convert img to hsv
img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
h = img_hsv[:,:,0]
s = img_hsv[:,:,1]
v = img_hsv[:,:,2]
# shift the hue
# cv2 will clip automatically to avoid color wrap-around
img_hsv = cv2.add(h, hue_offset)
# combine new hue with s and v
img_hsv = cv2.merge([img_hsv,s,v])
# convert from HSV to BGR
return cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR)
def on_trackbar_change(self, trackbar_hue_offset):
img_bgr_modified = self.recognize(self.img_bgr.copy(), trackbar_hue_offset)
cv2.imshow(window_title, img_bgr_modified)
def recognize(self, img_bgr, hue_offset):
img_bgr = self.apply_hue_offset(img_bgr, hue_offset)
img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
results = self.mp_hands.process(img_rgb)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(
img_bgr,
hand_landmarks,
mp_hands.HAND_CONNECTIONS,
mp_drawing_styles.get_default_hand_landmarks_style(),
mp_drawing_styles.get_default_hand_connections_style())
else:
print('no hands were found')
return img_bgr
def run(self, img_path):
self.img_bgr = cv2.imread(img_path)
if self.img_bgr is None: print('Image was not found!')
self.on_trackbar_change(0)
# Hue range is 0-179: https://docs.opencv.org/4.x/df/d9d/tutorial_py_colorspaces.html
cv2.createTrackbar(trackbar_title, window_title, 0, 179, self.on_trackbar_change)
cv2.waitKey(0)
cv2.destroyAllWindows()
if __name__ == '__main__':
h = HueHelper()
h.run('gloves.jpg')
再次演示:
英文:
The simplest solution is to omit the cv2.cvtColor()
, which will cause the image channels to swap, effectively making blue
look like orange
. Here is an example (for mediapipe==0.9.0):
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands
with mp_hands.Hands(
static_image_mode=True,
max_num_hands=2,
min_detection_confidence=0.5) as hands:
image = cv2.imread("gloves.jpg")
results = hands.process(image)
if results.multi_hand_landmarks:
image_height, image_width, _ = image.shape
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(
image,
hand_landmarks,
mp_hands.HAND_CONNECTIONS,
mp_drawing_styles.get_default_hand_landmarks_style(),
mp_drawing_styles.get_default_hand_connections_style())
else:
print("no hands were found")
cv2.imshow("result", image)
cv2.waitKey(0)
This should be sufficient for MediaPipe to recognize the hands:
Alternatively, you can also make a fine-tuning of the color change using Hue offset in HSV schema (adapted from here). This would be a "better" solution since it only changes the hue but does not swap the channels.
Edit: I've created a simple Hue-tunning "GUI-like" script that helps to find suitable Hue-offset:
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_hands = mp.solutions.hands
window_title = 'Recognition result'
trackbar_title = 'Hue offset'
class HueHelper:
def __init__(self):
self.mp_hands = mp_hands.Hands(static_image_mode=True,
max_num_hands=2,
min_detection_confidence=0.5)
self.img_bgr = None
def apply_hue_offset(self,image, hue_offset):# 0 is no change; 0<=huechange<=180
# convert img to hsv
img_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
h = img_hsv[:,:,0]
s = img_hsv[:,:,1]
v = img_hsv[:,:,2]
# shift the hue
# cv2 will clip automatically to avoid color wrap-around
img_hsv = cv2.add(h, hue_offset)
# combine new hue with s and v
img_hsv = cv2.merge([img_hsv,s,v])
# convert from HSV to BGR
return cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR)
def on_trackbar_change(self, trackbar_hue_offset):
img_bgr_modified = self.recognize(self.img_bgr.copy(), trackbar_hue_offset)
cv2.imshow(window_title, img_bgr_modified)
def recognize(self, img_bgr, hue_offset):
img_bgr = self.apply_hue_offset(img_bgr, hue_offset)
img_rgb = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2RGB)
results = self.mp_hands.process(img_rgb)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(
img_bgr,
hand_landmarks,
mp_hands.HAND_CONNECTIONS,
mp_drawing_styles.get_default_hand_landmarks_style(),
mp_drawing_styles.get_default_hand_connections_style())
else:
print('no hands were found')
return img_bgr
def run(self, img_path):
self.img_bgr = cv2.imread(img_path)
if self.img_bgr is None: print('Image was not found!')
self.on_trackbar_change(0)
# Hue range is 0-179: https://docs.opencv.org/4.x/df/d9d/tutorial_py_colorspaces.html
cv2.createTrackbar(trackbar_title, window_title, 0, 179, self.on_trackbar_change)
cv2.waitKey(0)
cv2.destroyAllWindows()
if __name__ == '__main__':
h = HueHelper()
h.run('gloves.jpg')
Again, a demo:
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论