OpenCV를 이용한 지렁이게임 (hand tracking)

opencv, cvzone 라이브러리를 이용한 지렁이 게임 입니다.
코드를 과정 별로 설명한 글입니다. 최종 코드를 보고 싶으시면 가장 아래를 참고 바랍니다.

1. 영상


2. 라이브러리 및 카메라 setup
해당 라이브러리들을 설치하고 잘 실행이 되는지 확인을 위해, 다음 코드를 실행합니다.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import cvzone
from cvzone.HandTrackingModule import HandDetector
import cv2

cap = cv2.VideoCapture(0) 
cap.set(3, 1280) 
cap.set(4, 720)
detector = HandDetector(detectionCon = 0.8, maxHands = 1) 

cap = cv2.VideoCapture(0) # connect camera
cap.set(3, 1280) # set the width and height of the image
cap.set(4, 720)
detector = HandDetector(detectionCon=0.8, maxHands=1) # set the threshold and the number of hands to detect


while True:
    success, img = cap.read()
    img = cv2.flip(img, 1)
    hands, img = detector.findHands(img, flipType=False)
    
    cv2.imshow("Image", img)
    
    key = cv2.waitKey(1)
    if key == 27: # key 'esc'
        cv2.destroyAllWindows()
        cap.release() # disconnect and turn off the camera
        print('break')
        break

저는 cvzone를 import 중 에러가 발생했습니다. 검색해보니 제 python이 3.9 버전이었는데 cvzone이 python 3.6, 3.8 버전에서 작동한다고 합니다. python downgrade를 시도 바랍니다.

위 코드가 잘 작동한다면 다음과 같이 보입니다.


3. 클래스 생성 및 필요한 변수, 함수들 구상
게임으로서 기능을 하기 위해서 클래스를 생성하고 필요한 변수, 함수들을 구상합니다. 변수, 함수들을 선언만 하고 작동만 되게 빈 함수들을 생성합니다.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
import math
import random
import cvzone
import cv2
import numpy as np
from cvzone.HandTrackingModule import HandDetector
import time

class SnakeGameClass:
    def __init__(self):
        self.backSize = (1280, 720)
        self.numTile = 25 # tile number of the game: ex) 25 means 25 x 25 tiles game
        
        self.lastFinger = [None, None] # the last location of the index finger
        self.points = [(self.numTile//2, self.numTile//2)] # all the locations of the snake
                                                            # the initial location of the snake is center
        self.length = 1 # The length of the snake
        self.foodPoint = (0,0) # The location of the food
        self.score = 0
        self.gameOver = False
        self.gameStart = False
        
        self.direction = '' # left, right, up, down = 'l', 'r', 'u', 'd'
        
        self.snakePeriod = 0.25 # (sec) the period of the snake moving
    
    def start(self):
        cap = cv2.VideoCapture(0)
        cap.set(3, 1280)
        cap.set(4, 720)
        detector = HandDetector(detectionCon=0.8, maxHands=1)
        
        # run this loop consistently and get capture every loop
        while True:
            success, img = cap.read()
            img = cv2.flip(img, 1)
            hands, img = detector.findHands(img, flipType=False)
            
            if hands:
                lmList = hands[0]['lmList']
                self.lastFinger = lmList[8][0:2]
                self.updateDirection()
                self.gameStart = True
            
            if self.isTimeToMoveSnake():
                self.update() # update the class variables for moving of snake
            
            img = self.displayGUI(img) # update the GUI
            cv2.imshow("Image", img)
            
            key = cv2.waitKey(1)
            if key == ord('r'): # key r to reset the game
                self.resetGame()
                print('reset')
            elif key == 27: # key 'esc' to break the game
                cv2.destroyAllWindows()
                cap.release() # disconnect and turn off the camera
                print('break')
                break
                
    def update(self):
        # if it is time to move the snake
        # change the class variables about moving
        return 0

    def randomFood(self):
        # return the new location of food
        # (tuple) (i, j)
        # i,j : 0 ~ (numTile -1)
        return (0,0)

    def updateDirection(self):
        # using the last node of self.points which means the head of the snake
        # and using the lastFinger position
        # change self.direction to one of these : 'l', 'r', 'u', 'd' (left, right, up, down)
        return 0

    def indexToPixel(self, index):
        # we need to change tile index to pixel index for decision the direction
        # parameter index : the location of the last node (head of the snake)
        # ex) (0, 0) ~ (24, 24)
        # return : the location represented with pixel
        # ex) (600, 600)
        x_new, y_new = (0, 0)
        return (x_new, y_new)

    def isTimeToMoveSnake(self):
        # if it is time to move of the snake : return 1
        # or return 0
        return True

    def resetGame(self):
        # if you press 'r' button, this method will be executed
        # reset the game
        return 0

    def displayGUI(self, imgMain):
        # display the GUI
        # change the img to include GUI and return img

        return imgMain

    def drawSquare(self, imgMain, position, color, fill=False):
        # position = not the position of the pixel, the position of the tile
        # left, top is (0, 0), right top is (i, 0), right bottom is (i, j)
        # color = (b, g, r), 0 ~ 255
        # fill = determine if want to fill or not
        return imgMain
    
game = SnakeGameClass()
game.start()

실행 후 결과는 1과 동일합니다.

4. 최종 코드
해당 함수들을 채워 넣고 다음 최종 코드를 완성합니다.
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
import math
import random
import cvzone
import cv2
import numpy as np
from cvzone.HandTrackingModule import HandDetector
import time

class SnakeGameClass:
    def __init__(self):
        self.backSize = (1280, 720)
        self.gameSize = (500, 500)
        self.numTile = 25
        self.margin = 30
        
        self.lastFinger = [None, None] # the last location of the index finger
        self.points = [(self.numTile//2, self.numTile//2)] # all the locations of the snake
        self.length = 1 # The length of the snake
        self.foodPoint = self.randomFood()
        self.score = 0
        self.gameOver = False
        self.gameStart = False
        
        self.direction = '' # left, right, up, down = 'l', 'r', 'u', 'd'
        
        self.previousTime = time.time()
        self.currentTime = time.time()
        self.snakePeriod = 0.25 # (sec) the period of the snake moving

    def start(self):
        cap = cv2.VideoCapture(0)
        cap.set(3, 1280)
        cap.set(4, 720)
        detector = HandDetector(detectionCon=0.8, maxHands=1)
        
        # run this loop consistently and get capture every loop
        while True:
            success, img = cap.read()
            img = cv2.flip(img, 1)
            hands, img = detector.findHands(img, flipType=False)
            
            if hands:
                lmList = hands[0]['lmList']
                self.lastFinger = lmList[8][0:2]
                self.updateDirection()
                self.gameStart = True
            
            if self.isTimeToMoveSnake():
                self.update() # update the class variables for moving of snake
                
            img = self.displayGUI(img) # update the GUI
            cv2.imshow("Image", img)
            
            key = cv2.waitKey(1)
            if key == ord('r'): # key r to reset the game
                self.resetGame()
                print('reset')
            elif key == 27: # key 'esc' to break the game
                cv2.destroyAllWindows()
                cap.release() # disconnect and turn off the camera
                print('break')
                break
                
    def update(self):
        # if it is time to move the snake
        # change the class variables about moving
        if (self.gameOver == True) or (self.gameStart == False):
            return 0
        
        hx, hy = self.points[-1] # position of the head
        if self.direction == 'l': #left
            if hx - 1 < 0: # when it collided to the wall
                self.gameOver = True
                return 0
            elif (hx - 1, hy) in self.points:
                self.gameOver = True
                return 0
            elif (hx - 1, hy) == self.foodPoint: # when head ate food
                self.whenAteFood()
                return 0
            else:
                self.points.append((hx-1,hy))
                del self.points[0]
                
        elif self.direction == 'r': #right
            if hx + 1 >= self.numTile: # when it collided to the wall
                self.gameOver = True
                return 0
            elif (hx + 1, hy) in self.points:
                self.gameOver = True
                return 0
            elif (hx + 1, hy) == self.foodPoint: # when head ate food
                self.whenAteFood()
                return 0
            else:
                self.points.append((hx+1,hy))
                del self.points[0]
                
        elif self.direction == 'u': #left
            if hy - 1 < 0: # when it collided to the wall
                self.gameOver = True
                return 0
            elif (hx, hy-1) in self.points:
                self.gameOver = True
                return 0
            elif (hx, hy-1) == self.foodPoint: # when head ate food
                self.whenAteFood()
                return 0
            else:
                self.points.append((hx,hy-1))
                del self.points[0]
                
        else:
            if hy + 1 >= self.numTile: # when it collided to the wall
                self.gameOver = True
                return 0
            elif (hx, hy + 1) in self.points:
                self.gameOver = True
                return 0
            elif (hx, hy + 1) == self.foodPoint: # when head ate food
                self.whenAteFood()
                return 0
            else:
                self.points.append((hx,hy + 1))
                del self.points[0]
                
        return 0    
    
    def whenAteFood(self):
        self.points.append(self.foodPoint)
        self.length += 1
        self.foodPoint = self.randomFood()
        self.score += 1
        return 0
    
    def randomFood(self):
        # return the new location of food
        # (tuple) (i, j)
        # i,j : 0 ~ (numTile -1)
        foodSpace = []
        for i in range(self.numTile):
            for j in range(self.numTile):
                foodSpace.append((i,j))
        
        for item in self.points:
            foodSpace.remove(item)
        
        index = random.randrange(len(foodSpace))
        return foodSpace[index]
        
    def updateDirection(self):
        # using the last node of self.points and the lastFinger position
        # determine the direction 
        # return one of these : 'l', 'r', 'u', d'
        x_finger, y_finger = self.lastFinger
        x_head, y_head = self.indexToPixel(self.points[-1])
        
        dx = x_finger - x_head
        dy = y_finger - y_head
        
        if abs(dx) >= abs(dy):
            if dx >= 0:
                self.direction = 'r'
            else:
                self.direction = 'l'
        else:
            if dy >= 0:
                self.direction = 'd'
            else:
                self.direction = 'u'
        return 0
    
    def indexToPixel(self, index):
        # we need to change tile index to pixel index for decision the direction
        # parameter index : the location of the last node (head of the snake)
        # ex) (0, 0) ~ (24, 24)
        # return : the location represented with pixel
        # ex) (600, 600)
        i, j = index
        a, _ = self.backSize
        c, _ = self.gameSize
        e = self.margin
        f = c // self.numTile
        x_new = a/2 - c/2 + f*(i+1/2) + 1/2
        y_new = 2*e + f*(j+1/2) + 1/2
        
        return (x_new, y_new)
        
    def isTimeToMoveSnake(self):
        # if it is time to move of the snake : return 1
        # or return 0
        self.currentTime = time.time()
        if self.currentTime > self.previousTime + self.snakePeriod:
            self.previousTime += self.snakePeriod
            return True
        return False
    
    def resetGame(self):
        # if you press 'r' button, this method will be executed
        # reset the game
        self.lastFinger = [None, None] # the last location of the index finger
        self.points = [(self.numTile//2, self.numTile//2)] # all the locations of the snake
        self.length = 1 # The length of the snake
        self.foodPoint = self.randomFood()
        self.score = 0
        self.gameOver = False
        self.gameStart = False
        
        return 0

    def displayGUI(self, imgMain):
        # display the GUI
        # change the img to include GUI and return img
        
        marginColor = (0, 255, 255)
        marginThick = 3 # px
        
        textColor = (0, 0, 0)
        backColor = (0, 255, 255)
        fontScale = 2
        fontThick = 2
        
        snakeColor = (255, 0, 0)
        headColor = (255, 255, 0)
        
        foodColor = (0, 255, 255)
        
        imgMain = cv2.rectangle(imgMain,
                                (self.backSize[0]//2-self.gameSize[0]//2-self.margin+1, 1*self.margin+1),
                                (self.backSize[0]//2+self.gameSize[0]//2+self.margin, 3*self.margin+self.gameSize[1]),
                                marginColor, marginThick)
        
        if self.gameOver:
            cvzone.putTextRect(imgMain, "Game Over", [300, 400],
                               scale=5, thickness=3, offset=20)
            cvzone.putTextRect(imgMain, f'Your Score:{self.score}', [300, 500],
                               scale=5, thickness=3, offset=20)
            cvzone.putTextRect(imgMain, "Press 'r' to restart", [300, 600],
                               scale=5, thickness=3, offset=20)
        elif self.gameStart == False:
            imgMain,_ = cvzone.putTextRect(imgMain, "Show your hand for start",
                              (self.backSize[0]//2-self.gameSize[0]//2+1, self.margin+self.gameSize[1]),
                              fontScale, fontThick, textColor, backColor)
            return imgMain
        
        for i, point in enumerate(self.points):
            if i != len(self.points)-1: # if is not the head
                imgMain = self.drawSquare(imgMain, point, snakeColor)
            else:
                imgMain = self.drawSquare(imgMain, point, headColor, True)
                
        imgMain = self.drawSquare(imgMain, self.foodPoint, foodColor, True)
        
        
        
        return imgMain
    
    def drawSquare(self, imgMain, position, color, fill=False):
        # position = not the position of the pixel, the position of the array
        # left, top is (0, 0), right top is (i, 0)
        # color = (b, g, r)
        # fill = determine if want to fill or not
        
        if fill == True:
            thickness = -1 # fill the rectangle
        else:
            thickness = 1 # 1px
        
        i, j = position
        n = self.gameSize[0] // self.numTile # number of pixels of each rectangle
    
        imgMain = cv2.rectangle(imgMain, (self.backSize[0]//2-self.gameSize[0]//2+n*i+1, 2*self.margin+n*j+1),
                                (self.backSize[0]//2-self.gameSize[0]//2+n*(i+1), 2*self.margin+n*(j+1)),
                                color, thickness)
        return imgMain
    
game = SnakeGameClass()
game.start()

여기서 설명이 가장 필요한 부분은 line 184 (tile index를 pixel index로 변환하는 과정)입니다. updateDirection()에서 검지 손가락 끝의 위치와 현재 뱀의 머리의 위치를 이용하여 방향을 결정합니다. 그런데 검지 위치의 단위는 pixel이고 뱀 머리 위치의 단위는 tile index이기 때문에 tile index를 pixel로 단위 환산하는 과정이 필요합니다.

계산식은 아래 이미지를 참고 바랍니다.
-
카메라 이미지 속에 게임 화면, 그리고 칸(snake의 몸 마디)가 있는 그림입니다.
(a, b) = backSize
c = gameSize
f = c / numTile
e = margin


혹시 문제가 있거나 도움이 필요하시면 메일이나 댓글 바랍니다.
감사합니다.

댓글

댓글 쓰기

이 블로그의 인기 게시물

Snake game using openCV (hand tracking)

이미지
The snake game using openCV, cvzone libraries. Here I will explain the steps for the code. If you need a final code, please check the bottom of the blog. 1. Video 2. Setup the libraries and camera After installing the libraries, you can check if it works using the below code. 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 import cvzone from cvzone.HandTrackingModule import HandDetector import cv2 cap = cv2 . VideoCapture( 0 ) cap . set( 3 , 1280 ) cap . set( 4 , 720 ) detector = HandDetector(detectionCon = 0.8 , maxHands = 1 ) cap = cv2 . VideoCapture( 0 ) # connect camera cap . set( 3 , 1280 ) # set the width and height of the image cap . set( 4 , 720 ) detector = HandDetector(detectionCon = 0.8 , maxHands = 1 ) # set the threshold and the number of hands to detect while True : success, img = cap . read() img = cv2 . flip(img, 1 ) hands, img = detector . findHands(img, flipType = False ) cv2 . imshow(...
자세한 내용 보기