实验三、机器视觉实验

一、实验简介

实验主要包含基于 python 和 OpenCV 的图像获取、处理和应用。通过对摄像头驱动的了解、颜色识别方法及流程的学习，开启视觉图像智能处理及实际应用实践训练。实验中，结合硬件平台的控制技术可实现目标物体的自主追踪。

二、实验目的

  1. 运用OpenCV使小车实现颜色识别和云台追踪
  2. 运用OpenCV使小车实现二维码运动识别
  3. 运用OpenCV使小车实现人脸识别
  4. 运用OpenCV使小车实现手势识别

三、实验步骤

1. 颜色识别

实验步骤

先调用函数cv2.VideoCapture读取摄像头，然后将读取图像格式转换为.jpeg并显示。颜色识别需要先利用cv2.cvtColor(input_image,flag)函数将颜色空间转换为 HSV ，后进行腐蚀。膨胀等操作，最后使用inRange()函数按照所选取颜色的上下界范围识别图像中的颜色范围，并用白色进行替换。

实验代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
import ctypes
import inspect
import threading
import time

import cv2
import ipywidgets.widgets as widgets
import numpy as np
import traitlets


# 转换格式
def bgr8_to_jpeg(value, quality=75):
    return bytes(cv2.imencode('.jpg', value)[1])


# 进程中止函数
def _async_raise(tid, exctype):
    """raises the exception, performs cleanup if needed"""
    tid = ctypes.c_long(tid)
    if not inspect.isclass(exctype):
        exctype = type(exctype)
    res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
    if res == 0:
        raise ValueError("invalid thread id")
    elif res != 1:
        # """if it returns a number greater than one, you're in trouble,
        # and you should call it again with exc=NULL to revert the effect"""
        ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)


def stop_thread(thread):
    _async_raise(thread.ident, SystemExit)


origin_widget = widgets.Image(format='jpeg', width=320, height=240)
mask_widget = widgets.Image(format='jpeg', width=320, height=240)
result_widget = widgets.Image(format='jpeg', width=320, height=240)
image_container = widgets.HBox([origin_widget, mask_widget, result_widget])
display(image_container)

cap = cv2.VideoCapture(0)
cap.set(3, 640)
cap.set(4, 480)
cap.set(5, 120)  # 设置帧率
cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('F', 'L', 'V', '1'))

# 默认选择红色的，想识别其他请注释下面红色区间代码，放开后面其他区间代码段
# 红色区间
# color_lower = np.array([0, 43, 46])
# color_upper = np.array([10, 255, 255])

# #绿色区间
color_lower = np.array([35, 43, 46])
color_upper = np.array([77, 255, 255])


# #蓝色区间
# color_lower=np.array([100, 43, 46])
# color_upper = np.array([124, 255, 255])

# #黄色区间
# color_lower = np.array([26, 43, 46])
# color_upper = np.array([34, 255, 255])

# #橙色区间
# color_lower = np.array([11, 43, 46])
# color_upper = np.array([25, 255, 255])


def Color_Recongnize():
    while (1):
        ret, frame = cap.read()
        origin_widget.value = bgr8_to_jpeg(frame)
        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
        mask = cv2.inRange(hsv, color_lower, color_upper)
        mask_widget.value = bgr8_to_jpeg(mask)
        res = cv2.bitwise_and(frame, frame, mask=mask)
        result_widget.value = bgr8_to_jpeg(res)
        time.sleep(0.01)

    cap.release()


thread1 = threading.Thread(target=Color_Recongnize)
thread1.setDaemon(True)
thread1.start()

2. 二维码运动识别

实验步骤

先读取摄像头并将其转换为.jpeg格式，后使用cv2.rectangle函数提取二维码边界框位置，向终端打印条形码数据和条形码类型，最后通过定义的运动函数输出运动指令。

实验代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
import cv2
import ipywidgets.widgets as widgets
import numpy as np
import pyzbar.pyzbar as pyzbar
from PIL import Image

image_widget = widgets.Image(format='jpeg', width=320, height=240)
display(image_widget)  # 显示摄像头组件

# bgr8转jpeg格式
import cv2


def bgr8_to_jpeg(value, quality=75):
    return bytes(cv2.imencode('.jpg', value)[1])


def decodeDisplay(image):
    barcodes = pyzbar.decode(image)
    for barcode in barcodes:
        # 提取二维码的边界框的位置
        # 画出图像中条形码的边界框
        (x, y, w, h) = barcode.rect
        cv2.rectangle(image, (x, y), (x + w, y + h), (225, 225, 225), 2)

        # 提取二维码数据为字节对象，所以如果我们想在输出图像上
        # 画出来，就需要先将它转换成字符串
        barcodeData = barcode.data.decode("utf-8")
        barcodeType = barcode.type

        # 绘出图像上条形码的数据和条形码类型
        text = "{} ({})".format(barcodeData, barcodeType)
        cv2.putText(image, text, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (225, 225, 225), 2)

        # 向终端打印条形码数据和条形码类型
        print("[INFO] Found {} barcode: {}".format(barcodeType, barcodeData))
    return image


def detect():
    camera = cv2.VideoCapture(0)
    camera.set(3, 320)
    camera.set(4, 240)
    camera.set(5, 120)  # 设置帧率
    # fourcc = cv2.VideoWriter_fourcc(*"MPEG")
    camera.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
    camera.set(cv2.CAP_PROP_BRIGHTNESS, 40)  # 设置亮度 -64 - 64  0.0
    camera.set(cv2.CAP_PROP_CONTRAST, 50)  # 设置对比度 -64 - 64  2.0
    camera.set(cv2.CAP_PROP_EXPOSURE, 156)  # 设置曝光值 1.0 - 5000  156.0
    ret, frame = camera.read()
    image_widget.value = bgr8_to_jpeg(frame)
    while True:
        # 读取当前帧
        ret, frame = camera.read()
        # 转为灰度图像
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        im = decodeDisplay(gray)

        cv2.waitKey(5)
        image_widget.value = bgr8_to_jpeg(im)
        # 如果按键q则跳出本次循环
        if cv2.waitKey(10) & 0xFF == ord('q'):
            break
    camera.release()
    cv2.destroyAllWindows()


while 1:
    detect()

3. 人脸识别

实验步骤

将人脸检测所需分类器和 OpenCV 检测中的 API 函数相结合实现人脸检测，结合 PID 控制舵机，实现人脸追踪。

实验代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
import cv2
import ipywidgets.widgets as widgets
from Raspblock import Raspblock

robot = Raspblock()
import cv2

face_haar = cv2.CascadeClassifier('haarcascade_profileface.xml')

image_widget = widgets.Image(format='jpeg', width=320, height=240)
display(image_widget)

# bgr8转jpeg格式
import cv2


def bgr8_to_jpeg(value, quality=75):
    return bytes(cv2.imencode('.jpg', value)[1])


import ipywidgets as widgets

XServo_P = widgets.FloatSlider(
    value=1.1,
    min=0,
    max=10.0,
    step=0.1,
    description='XServo-P:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='.1f',
)

XServo_I = widgets.FloatSlider(
    value=0.2,
    min=0,
    max=10.0,
    step=0.1,
    description='XServo-I:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='.1f',
)

XServo_D = widgets.FloatSlider(
    value=0.8,
    min=0,
    max=10.0,
    step=0.1,
    description='XServer-D:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='.1f',
)

YServo_P = widgets.FloatSlider(
    value=0.8,
    min=0,
    max=10.0,
    step=0.1,
    description='YServo-P:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='.1f',
)

YServo_I = widgets.FloatSlider(
    value=0.2,
    min=0,
    max=10.0,
    step=0.1,
    description='YServo-I:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='.1f',
)

YServo_D = widgets.FloatSlider(
    value=0.8,
    min=0,
    max=10.0,
    step=0.1,
    description='YServer-D:',
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=True,
    readout_format='.1f',
)
display(XServo_P, XServo_I, XServo_D, YServo_P, YServo_I, YServo_D)

global face_x, face_y, face_w, face_h
face_x = face_y = face_w = face_h = 0
global target_valuex
target_valuex = 2048
global target_valuey
target_valuey = 2048

import PID

# xservo_pid = PID.PositionalPID(1.1, 0.2, 0.8)
# yservo_pid = PID.PositionalPID(0.8, 0.2, 0.8)

xservo_pid = PID.PositionalPID(XServo_P.value, XServo_I.value, XServo_D.value)
yservo_pid = PID.PositionalPID(YServo_P.value, YServo_I.value, YServo_D.value)

image = cv2.VideoCapture(0)
image.set(3, 320)
image.set(4, 240)

while 1:

    ret, frame = image.read()
    try:
        image_widget.value = bgr8_to_jpeg(frame)
    except:
        continue
    # 把图像转为黑白图像
    gray_img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    faces = face_haar.detectMultiScale(gray_img, 1.1, 3)

    xservo_pid = PID.PositionalPID(XServo_P.value, XServo_I.value, XServo_D.value)
    yservo_pid = PID.PositionalPID(YServo_P.value, YServo_I.value, YServo_D.value)

    if len(faces) > 0:
        (face_x, face_y, face_w, face_h) = faces[0]
        # cv2.rectangle(frame,(face_x+10,face_y),(face_x+face_w-10,face_y+face_h+20),(0,255,0),2)
        cv2.rectangle(frame, (face_x, face_y), (face_x + face_w, face_y + face_h), (0, 255, 0), 2)
        try:
            image_widget.value = bgr8_to_jpeg(frame)
        except:
            continue

        # Proportion-Integration-Differentiation算法
        # 输入X轴方向参数PID控制输入
        xservo_pid.SystemOutput = face_x + face_w / 2
        xservo_pid.SetStepSignal(150)
        xservo_pid.SetInertiaTime(0.01, 0.1)
        target_valuex = int(1500 + xservo_pid.SystemOutput)

        # 输入Y轴方向参数PID控制输入
        yservo_pid.SystemOutput = face_y + face_h / 2
        yservo_pid.SetStepSignal(120)
        yservo_pid.SetInertiaTime(0.01, 0.1)
        target_valuey = int(1500 - yservo_pid.SystemOutput)

        # 将云台转动至PID调校位置
        robot.Servo_control(target_valuex, target_valuey)

4. 手势识别

实验步骤

通过手势识别 API 接口来进行手势识别，并将识别结果返回。后将识别结果发送到语音播报程序进行语音播报。

实验代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import cv2
import time
import demjson
import pygame
from aip import AipBodyAnalysis
from aip import AipSpeech
from PIL import Image, ImageDraw, ImageFont
import numpy
import ipywidgets.widgets as widgets


# bgr8转jpeg格式
def bgr8_to_jpeg(value, quality=75):
    return bytes(cv2.imencode('.jpg', value)[1])


# 具体手势请看官方提供 https://ai.baidu.com/ai-doc/BODY/4k3cpywrv
hand = {'One': '1', 'Five': '5', 'Fist': '拳头', 'Ok': 'OK',
        'Prayer': '祈祷', 'Congratulation': '作揖', 'Honour': '作别',
        'Heart_single': '比心心', 'Thumb_up': '点赞', 'Thumb_down': 'Diss',
        'ILY': '我爱你', 'Palm_up': '掌心向上', 'Heart_1': '双手比心',
        'Heart_2': '双手比心', 'Heart_3': '双手比心', 'Two': '2',
        'Three': '3', 'Four': '4', 'Six': '6', 'Seven': '7',
        'Eight': '8', 'Nine': '9', 'Rock': 'Rock', 'Insult': '中指', 'Face': '脸'}

""" 人体分析 APPID AK SK """
APP_ID = ''
API_KEY = ''
SECRET_KEY = ''

SpeechAPP_ID = ''
SpeechAPI_KEY = ''
SpeechSECRET_KEY = ''

Speechclient = AipSpeech(SpeechAPP_ID, SpeechAPI_KEY, SpeechSECRET_KEY)

# camera = PiCamera()
client = AipBodyAnalysis(APP_ID, API_KEY, SECRET_KEY)

g_camera = cv2.VideoCapture(0)
g_camera.set(3, 640)
g_camera.set(4, 480)
g_camera.set(5, 120)  # 设置帧率
g_camera.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
g_camera.set(cv2.CAP_PROP_BRIGHTNESS, 40)  # 设置亮度 -64 - 64  0.0
g_camera.set(cv2.CAP_PROP_CONTRAST, 50)  # 设置对比度 -64 - 64  2.0
g_camera.set(cv2.CAP_PROP_EXPOSURE, 156)  # 设置曝光值 1.0 - 5000  156.0

ret, frame = g_camera.read()

image_widget = widgets.Image(format='jpeg', width=600, height=500)  # 设置摄像头显示组件
display(image_widget)
image_widget.value = bgr8_to_jpeg(frame)


def cv2ImgAddText(img, text, left, top, textColor=(0, 255, 0), textSize=20):
    if (isinstance(img, numpy.ndarray)):  # 判断是否OpenCV图片类型
        img = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
    # 创建一个可以在给定图像上绘图的对象
    draw = ImageDraw.Draw(img)
    # 字体的格式
    fontStyle = ImageFont.truetype(
        "simhei.ttf", textSize, encoding="utf-8")
    # 绘制文本
    draw.text((left, top), text, textColor, font=fontStyle)
    # 转换回OpenCV格式
    return cv2.cvtColor(numpy.asarray(img), cv2.COLOR_RGB2BGR)


while True:
    """1.拍照 """
    retval, frame = g_camera.read()

    ret, frame = g_camera.read()

    # image = get_file_content('./image.jpg')

    """ 2.调用手势识别 """
    raw = str(client.gesture(image_widget.value))
    text = demjson.decode(raw)
    try:
        res = text['result'][0]['classname']
    except:
        print('识别结果：什么也没识别到哦~')
        # 1 dst 2 文字内容 3 坐标 4 5 字体大小 6 color 7 粗细 8 line type
        #       cv2.putText(frame, '未识别', (250,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,200), 2, cv2.LINE_AA) #只能显示英文
        img = cv2ImgAddText(frame, "未识别", 250, 30, (0, 0, 255), 30)
    else:
        print('识别结果：' + hand[res])
        # cv2.putText(frame, hand[res], (250,30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,0), 2, cv2.LINE_AA)
        img = cv2ImgAddText(frame, hand[res], 250, 30, (0, 255, 0), 30)
        result = Speechclient.synthesis(hand[res], 'zh', 1, {'spd': 3, 'vol': 5, 'per': 3})
        open('audio.mp3', 'wb').write(result)
        # 利用树莓派自带的pygame
        pygame.mixer.init()
        pygame.mixer.music.load('./audio.mp3')
        pygame.mixer.music.play()
        time.sleep(1)
    image_widget.value = bgr8_to_jpeg(img)

四、思考分析

  1. 实验总结与收获
    能够熟练运用OpenCV中的函数来进行程序功能的实现。

  2. 实验失败/待改进的分析
    需要更加熟悉各个函数的名称和作用。

课程报告
#河北工业大学 #系统设计与开发
实验三、机器视觉实验
https://flowerdown.org/posts/20221014-162914.html
作者
Unrealfeather
发布于
2022年10月14日
许可协议