配置环境：
cuda11.3 +cudnn+zed sdk+opencv
安装yolov4：
下载darknet
终端输入如下命令：
git clone https://github.com/AlexeyAB/darknet.git
使用make在Linx编译：
在make之前可以在查看Makefile文件
如上所示要使用Makefike在yolo中启用zed，只需要启用GPU和ZED_CAMERA，同时为了提高计算速度还应该启用CUDNN
我的Makefile文件设置成如下图所示：
注意：
- ARCH的值要根据显卡的算力来设置，我的设置为
ARCH= -gencode arch=compute_86,code=[sm_86,compute_86]- NVCC需要指定到cuda的位置
NVCC=/usr/local/cuda-11.3/bin/nvcc在上述设置好后进行编译
cd darknet
make
如上，编译成功
编译成功之后，可以进行一个测试，首先需要下载权重，权重下载地址：
https://drive.google.com/open?id=1cewMfusmPjYWbrnuJRuKhPMwRe_b9PaT
下载完成后终端运行如下命令;
./darknet detect cfg/yolov4.cfg yolov4.weights data/dog.jpg
测试成功
注：也可使用cmake编译，见附录。
使用ZED识别并测距离
在C++中使用yolo
注意如果在C++中使用需要先将Makefile中的LIBSO，LIBSO=1
它可以建立一个库链接到darknet.so以及二进制的可运行文件usblib
之后终端运行如下命令：
LD_LIBRARY_PATH=./:$LD_LIBRARY_PATH ./uselib data/coco.names cfg/yolov4.cfg yolov4.weights zed_camera
识别成功
在Python中使用yolo
需要先安装 ZED Python
下载zed-yolo-master.zip以及权重文件
https://github.com/stereolabs/zed-yolo
wget https://pjreddie.com/media/files/yolov3-tiny.weights
编译
cd libdarknet
make 运行应用程序：
python3 darknet_zed.py也可以更改参数：
python3 darknet_zed.py -c <config> -w <weight> -m <meta> -t <threshold> -s <svo_file>
例如：
python3 darknet_zed.py -c cfg/yolov3.cfg -w yolov3.weights -m cfg/coco.data -t 0.5
注：
- 要将darknet_zed.py下面这一行设置为绝对路径链接到libdarknet.so
else:
lib = CDLL("/home/huang/program/zed-yolo-master/libdarknet/libdarknet.so", RTLD_GLOBAL)- 在Python中使用yolo 使用的cudnn版本为cudnn7.x ，不然会报错：
由于我cudnn的版本为cudnn8.2.1
#查看cudnn版本方法
cat /usr/local/cuda/include/cudnn_version.h | grep CUDNN_MAJOR -A 2我在makefile中关闭了GPU，只用CPU进行测试，很卡
*******************************************************************************************
在用C++时发现yolov4可以使用cudnn8.2.1
尝试发现
如果将上述的libdarknet删掉，替换成yolov4的，并将darknet改称libdarknet，即可使用GPU CUDNN
#yolov4下载
git clone https://github.com/AlexeyAB/darknet.git设置Makefile如下后make
cd ~/program/zed-yolo-master/zed_python_sample
python3 darknet_zed.py
识别成功
附录一：
darknet_zed.py
#!python3
import os
import sys
import time
import logging
import random
from random import randint
import math
import statistics
import getopt
from ctypes import *
import numpy as np
import cv2
import pyzed.sl as sl
# Get the top-level logger object
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
def sample(probs):
s = sum(probs)
probs = [a/s for a in probs]
r = random.uniform(0, 1)
for i in range(len(probs)):
r = r - probs[i]
if r <= 0:
return i
return len(probs)-1
def c_array(ctype, values):
arr = (ctype*len(values))()
arr[:] = values
return arr
class BOX(Structure):
_fields_ = [("x", c_float),
("y", c_float),
("w", c_float),
("h", c_float)]
class DETECTION(Structure):
_fields_ = [("bbox", BOX),
("classes", c_int),
("prob", POINTER(c_float)),
("mask", POINTER(c_float)),
("objectness", c_float),
("sort_class", c_int),
("uc", POINTER(c_float)),
("points", c_int),
("embeddings", POINTER(c_float)),
("embedding_size", c_int),
("sim", c_float),
("track_id", c_int)]
class IMAGE(Structure):
_fields_ = [("w", c_int),
("h", c_int),
("c", c_int),
("data", POINTER(c_float))]
class METADATA(Structure):
_fields_ = [("classes", c_int),
("names", POINTER(c_char_p))]
#lib = CDLL("/home/pjreddie/documents/darknet/libdarknet.so", RTLD_GLOBAL)
#lib = CDLL("darknet.so", RTLD_GLOBAL)
hasGPU = True
if os.name == "nt":
cwd = os.path.dirname(__file__)
os.environ['PATH'] = cwd + ';' + os.environ['PATH']
winGPUdll = os.path.join(cwd, "yolo_cpp_dll.dll")
winNoGPUdll = os.path.join(cwd, "yolo_cpp_dll_nogpu.dll")
envKeys = list()
for k, v in os.environ.items():
envKeys.append(k)
try:
try:
tmp = os.environ["FORCE_CPU"].lower()
if tmp in ["1", "true", "yes", "on"]:
raise ValueError("ForceCPU")
else:
log.info("Flag value '"+tmp+"' not forcing CPU mode")
except KeyError:
# We never set the flag
if 'CUDA_VISIBLE_DEVICES' in envKeys:
if int(os.environ['CUDA_VISIBLE_DEVICES']) < 0:
raise ValueError("ForceCPU")
try:
global DARKNET_FORCE_CPU
if DARKNET_FORCE_CPU:
raise ValueError("ForceCPU")
except NameError:
pass
# log.info(os.environ.keys())
# log.warning("FORCE_CPU flag undefined, proceeding with GPU")
if not os.path.exists(winGPUdll):
raise ValueError("NoDLL")
lib = CDLL(winGPUdll, RTLD_GLOBAL)
except (KeyError, ValueError):
hasGPU = False
if os.path.exists(winNoGPUdll):
lib = CDLL(winNoGPUdll, RTLD_GLOBAL)
log.warning("Notice: CPU-only mode")
else:
# Try the other way, in case no_gpu was
# compile but not renamed
lib = CDLL(winGPUdll, RTLD_GLOBAL)
log.warning("Environment variables indicated a CPU run, but we didn't find `" +
winNoGPUdll+"`. Trying a GPU run anyway.")
else:
lib = CDLL("/home/huang/program/zed-yolo-master/libdarknet/libdarknet.so", RTLD_GLOBAL)
#设置成绝对路径
lib.network_width.argtypes = [c_void_p]
lib.network_width.restype = c_int
lib.network_height.argtypes = [c_void_p]
lib.network_height.restype = c_int
predict = lib.network_predict
predict.argtypes = [c_void_p, POINTER(c_float)]
predict.restype = POINTER(c_float)
if hasGPU:
set_gpu = lib.cuda_set_device
set_gpu.argtypes = [c_int]
make_image = lib.make_image
make_image.argtypes = [c_int, c_int, c_int]
make_image.restype = IMAGE
get_network_boxes = lib.get_network_boxes
get_network_boxes.argtypes = [c_void_p, c_int, c_int, c_float, c_float, POINTER(
c_int), c_int, POINTER(c_int), c_int]
get_network_boxes.restype = POINTER(DETECTION)
make_network_boxes = lib.make_network_boxes
make_network_boxes.argtypes = [c_void_p]
make_network_boxes.restype = POINTER(DETECTION)
free_detections = lib.free_detections
free_detections.argtypes = [POINTER(DETECTION), c_int]
free_ptrs = lib.free_ptrs
free_ptrs.argtypes = [POINTER(c_void_p), c_int]
network_predict = lib.network_predict
network_predict.argtypes = [c_void_p, POINTER(c_float)]
reset_rnn = lib.reset_rnn
reset_rnn.argtypes = [c_void_p]
load_net = lib.load_network
load_net.argtypes = [c_char_p, c_char_p, c_int]
load_net.restype = c_void_p
load_net_custom = lib.load_network_custom
load_net_custom.argtypes = [c_char_p, c_char_p, c_int, c_int]
load_net_custom.restype = c_void_p
do_nms_obj = lib.do_nms_obj
do_nms_obj.argtypes = [POINTER(DETECTION), c_int, c_int, c_float]
do_nms_sort = lib.do_nms_sort
do_nms_sort.argtypes = [POINTER(DETECTION), c_int, c_int, c_float]
free_image = lib.free_image
free_image.argtypes = [IMAGE]
letterbox_image = lib.letterbox_image
letterbox_image.argtypes = [IMAGE, c_int, c_int]
letterbox_image.restype = IMAGE
load_meta = lib.get_metadata
lib.get_metadata.argtypes = [c_char_p]
lib.get_metadata.restype = METADATA
load_image = lib.load_image_color
load_image.argtypes = [c_char_p, c_int, c_int]
load_image.restype = IMAGE
rgbgr_image = lib.rgbgr_image
rgbgr_image.argtypes = [IMAGE]
predict_image = lib.network_predict_image
predict_image.argtypes = [c_void_p, IMAGE]
predict_image.restype = POINTER(c_float)
def array_to_image(arr):
import numpy as np
# need to return old values to avoid python freeing memory
arr = arr.transpose(2, 0, 1)
c = arr.shape[0]
h = arr.shape[1]
w = arr.shape[2]
arr = np.ascontiguousarray(arr.flat, dtype=np.float32) / 255.0
data = arr.ctypes.data_as(POINTER(c_float))
im = IMAGE(w, h, c, data)
return im, arr
def classify(net, meta, im):
out = predict_image(net, im)
res = []
for i in range(meta.classes):
if altNames is None:
name_tag = meta.names[i]
else:
name_tag = altNames[i]
res.append((name_tag, out[i]))
res = sorted(res, key=lambda x: -x[1])
return res
def detect(net, meta, image, thresh=.5, hier_thresh=.5, nms=.45, debug=False):
"""
Performs the detection
"""
custom_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
custom_image = cv2.resize(custom_image, (lib.network_width(
net), lib.network_height(net)), interpolation=cv2.INTER_LINEAR)
im, arr = array_to_image(custom_image)
num = c_int(0)
pnum = pointer(num)
predict_image(net, im)
dets = get_network_boxes(
net, image.shape[1], image.shape[0], thresh, hier_thresh, None, 0, pnum, 0)
num = pnum[0]
if nms:
do_nms_sort(dets, num, meta.classes, nms)
res = []
if debug:
log.debug("about to range")
for j in range(num):
for i in range(meta.classes):
if dets[j].prob[i] > 0:
b = dets[j].bbox
if altNames is None:
name_tag = meta.names[i]
else:
name_tag = altNames[i]
res.append((name_tag, dets[j].prob[i], (b.x, b.y, b.w, b.h), i))
res = sorted(res, key=lambda x: -x[1])
free_detections(dets, num)
return res
netMain = None
metaMain = None
altNames = None
def get_object_depth(depth, bounds):
'''
Calculates the median x, y, z position of top slice(area_div) of point cloud
in camera frame.
Arguments:
depth: Point cloud data of whole frame.
bounds: Bounding box for object in pixels.
bounds[0]: x-center
bounds[1]: y-center
bounds[2]: width of bounding box.
bounds[3]: height of bounding box.
Return:
x, y, z: Location of object in meters.
'''
area_div = 2
x_vect = []
y_vect = []
z_vect = []
for j in range(int(bounds[0] - area_div), int(bounds[0] + area_div)):
for i in range(int(bounds[1] - area_div), int(bounds[1] + area_div)):
z = depth[i, j, 2]
if not np.isnan(z) and not np.isinf(z):
x_vect.append(depth[i, j, 0])
y_vect.append(depth[i, j, 1])
z_vect.append(z)
try:
x_median = statistics.median(x_vect)
y_median = statistics.median(y_vect)
z_median = statistics.median(z_vect)
except Exception:
x_median = -1
y_median = -1
z_median = -1
pass
return x_median, y_median, z_median
def generate_color(meta_path):
'''
Generate random colors for the number of classes mentioned in data file.
Arguments:
meta_path: Path to .data file.
Return:
color_array: RGB color codes for each class.
'''
random.seed(42)
with open(meta_path, 'r') as f:
content = f.readlines()
class_num = int(content[0].split("=")[1])
color_array = []
for x in range(0, class_num):
color_array.append((randint(0, 255), randint(0, 255), randint(0, 255)))
return color_array
def main(argv):
thresh = 0.25
darknet_path="../libdarknet/"
config_path = darknet_path + "cfg/yolov3-tiny.cfg"
weight_path = "yolov3-tiny.weights"
meta_path = "coco.data"
svo_path = None
zed_id = 0
help_str = 'darknet_zed.py -c <config> -w <weight> -m <meta> -t <threshold> -s <svo_file> -z <zed_id>'
try:
opts, args = getopt.getopt(
argv, "hc:w:m:t:s:z:", ["config=", "weight=", "meta=", "threshold=", "svo_file=", "zed_id="])
except getopt.GetoptError:
log.exception(help_str)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
log.info(help_str)
sys.exit()
elif opt in ("-c", "--config"):
config_path = arg
elif opt in ("-w", "--weight"):
weight_path = arg
elif opt in ("-m", "--meta"):
meta_path = arg
elif opt in ("-t", "--threshold"):
thresh = float(arg)
elif opt in ("-s", "--svo_file"):
svo_path = arg
elif opt in ("-z", "--zed_id"):
zed_id = int(arg)
input_type = sl.InputType()
if svo_path is not None:
log.info("SVO file : " + svo_path)
input_type.set_from_svo_file(svo_path)
else:
# Launch camera by id
input_type.set_from_camera_id(zed_id)
init = sl.InitParameters(input_t=input_type)
init.coordinate_units = sl.UNIT.METER
cam = sl.Camera()
if not cam.is_opened():
log.info("Opening ZED Camera...")
status = cam.open(init)
if status != sl.ERROR_CODE.SUCCESS:
log.error(repr(status))
exit()
runtime = sl.RuntimeParameters()
# Use STANDARD sensing mode
runtime.sensing_mode = sl.SENSING_MODE.STANDARD
mat = sl.Mat()
point_cloud_mat = sl.Mat()
# Import the global variables. This lets us instance Darknet once,
# then just call performDetect() again without instancing again
global metaMain, netMain, altNames # pylint: disable=W0603
assert 0 < thresh < 1, "Threshold should be a float between zero and one (non-inclusive)"
if not os.path.exists(config_path):
raise ValueError("Invalid config path `" +
os.path.abspath(config_path)+"`")
if not os.path.exists(weight_path):
raise ValueError("Invalid weight path `" +
os.path.abspath(weight_path)+"`")
if not os.path.exists(meta_path):
raise ValueError("Invalid data file path `" +
os.path.abspath(meta_path)+"`")
if netMain is None:
netMain = load_net_custom(config_path.encode(
"ascii"), weight_path.encode("ascii"), 0, 1) # batch size = 1
if metaMain is None:
metaMain = load_meta(meta_path.encode("ascii"))
if altNames is None:
# In thon 3, the metafile default access craps out on Windows (but not Linux)
# Read the names file and create a list to feed to detect
try:
with open(meta_path) as meta_fh:
meta_contents = meta_fh.read()
import re
match = re.search("names *= *(.*)$", meta_contents,
re.IGNORECASE | re.MULTILINE)
if match:
result = match.group(1)
else:
result = None
try:
if os.path.exists(result):
with open(result) as names_fh:
names_list = names_fh.read().strip().split("\n")
altNames = [x.strip() for x in names_list]
except TypeError:
pass
except Exception:
pass
color_array = generate_color(meta_path)
log.info("Running...")
key = ''
while key != 113: # for 'q' key
start_time = time.time() # start time of the loop
err = cam.grab(runtime)
if err == sl.ERROR_CODE.SUCCESS:
cam.retrieve_image(mat, sl.VIEW.LEFT)
image = mat.get_data()
cam.retrieve_measure(
point_cloud_mat, sl.MEASURE.XYZRGBA)
depth = point_cloud_mat.get_data()
# Do the detection
detections = detect(netMain, metaMain, image, thresh)
log.info(chr(27) + "[2J"+"**** " + str(len(detections)) + " Results ****")
for detection in detections:
label = detection[0]
confidence = detection[1]
pstring = label+": "+str(np.rint(100 * confidence))+"%"
log.info(pstring)
bounds = detection[2]
y_extent = int(bounds[3])
x_extent = int(bounds[2])
# Coordinates are around the center
x_coord = int(bounds[0] - bounds[2]/2)
y_coord = int(bounds[1] - bounds[3]/2)
#boundingBox = [[x_coord, y_coord], [x_coord, y_coord + y_extent], [x_coord + x_extent, y_coord + y_extent], [x_coord + x_extent, y_coord]]
thickness = 1
x, y, z = get_object_depth(depth, bounds)
distance = math.sqrt(x * x + y * y + z * z)
distance = "{:.2f}".format(distance)
cv2.rectangle(image, (x_coord - thickness, y_coord - thickness),
(x_coord + x_extent + thickness, y_coord + (18 + thickness*4)),
color_array[detection[3]], -1)
cv2.putText(image, label + " " + (str(distance) + " m"),
(x_coord + (thickness * 4), y_coord + (10 + thickness * 4)),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
cv2.rectangle(image, (x_coord - thickness, y_coord - thickness),
(x_coord + x_extent + thickness, y_coord + y_extent + thickness),
color_array[detection[3]], int(thickness*2))
cv2.imshow("ZED", image)
key = cv2.waitKey(5)
log.info("FPS: {}".format(1.0 / (time.time() - start_time)))
else:
key = cv2.waitKey(5)
cv2.destroyAllWindows()
cam.close()
log.info("\nFINISH")
if __name__ == "__main__":
main(sys.argv[1:])附录二：
使用cmake编译
它会自动查找可安装的依赖
git clone https://github.com/AlexeyAB/darknet
cd darknet
mkdir build_release
cd build_release
cmake ..
cmake --build . --target install --parallel 8
版权声明：本文为m0_62114628原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。