#! /usr/bin/python # -*- coding: utf-8 -*- import tensorflow as tf import colorsys, random, cv2 import numpy as np from tensorlayer.visualize import save_image def decode_tf(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=0, XYSCALE=[1, 1, 1]): batch_size = tf.shape(conv_output)[0] conv_output = tf.reshape(conv_output, (batch_size, output_size, output_size, 3, 5 + NUM_CLASS)) conv_raw_dxdy, conv_raw_dwdh, conv_raw_conf, conv_raw_prob = tf.split(conv_output, (2, 2, 1, NUM_CLASS), axis=-1) xy_grid = tf.meshgrid(tf.range(output_size), tf.range(output_size)) xy_grid = tf.expand_dims(tf.stack(xy_grid, axis=-1), axis=2) # [gx, gy, 1, 2] xy_grid = tf.tile(tf.expand_dims(xy_grid, axis=0), [batch_size, 1, 1, 3, 1]) xy_grid = tf.cast(xy_grid, tf.float32) pred_xy = ((tf.sigmoid(conv_raw_dxdy) * XYSCALE[i]) - 0.5 * (XYSCALE[i] - 1) + xy_grid) * \ STRIDES[i] pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i]) pred_xywh = tf.concat([pred_xy, pred_wh], axis=-1) pred_conf = tf.sigmoid(conv_raw_conf) pred_prob = tf.sigmoid(conv_raw_prob) pred_prob = pred_conf * pred_prob pred_prob = tf.reshape(pred_prob, (batch_size, -1, NUM_CLASS)) pred_xywh = tf.reshape(pred_xywh, (batch_size, -1, 4)) return pred_xywh, pred_prob def decode(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE=[1, 1, 1]): return decode_tf(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=i, XYSCALE=XYSCALE) def filter_boxes(box_xywh, scores, score_threshold=0.4, input_shape=tf.constant([416, 416])): scores_max = tf.math.reduce_max(scores, axis=-1) mask = scores_max >= score_threshold class_boxes = tf.boolean_mask(box_xywh, mask) pred_conf = tf.boolean_mask(scores, mask) class_boxes = tf.reshape(class_boxes, [tf.shape(scores)[0], -1, tf.shape(class_boxes)[-1]]) pred_conf = tf.reshape(pred_conf, [tf.shape(scores)[0], -1, tf.shape(pred_conf)[-1]]) box_xy, box_wh = tf.split(class_boxes, (2, 2), axis=-1) input_shape = tf.cast(input_shape, dtype=tf.float32) box_yx = box_xy[..., ::-1] box_hw = box_wh[..., ::-1] box_mins = (box_yx - (box_hw / 2.)) / input_shape box_maxes = (box_yx + (box_hw / 2.)) / input_shape boxes = tf.concat( [ box_mins[..., 0:1], # y_min box_mins[..., 1:2], # x_min box_maxes[..., 0:1], # y_max box_maxes[..., 1:2] # x_max ], axis=-1 ) # return tf.concat([boxes, pred_conf], axis=-1) return (boxes, pred_conf) def read_class_names(class_file_name): names = {} with open(class_file_name, 'r') as data: for ID, name in enumerate(data): names[ID] = name.strip('\n') return names def draw_bbox(image, bboxes, show_label=True): classes = read_class_names('model/coco.names') num_classes = len(classes) image_h, image_w, _ = image.shape hsv_tuples = [(1.0 * x / num_classes, 1., 1.) for x in range(num_classes)] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(0) random.shuffle(colors) random.seed(None) out_boxes, out_scores, out_classes, num_boxes = bboxes for i in range(num_boxes[0]): if int(out_classes[0][i]) < 0 or int(out_classes[0][i]) > num_classes: continue coor = out_boxes[0][i] coor[0] = int(coor[0] * image_h) coor[2] = int(coor[2] * image_h) coor[1] = int(coor[1] * image_w) coor[3] = int(coor[3] * image_w) fontScale = 0.5 score = out_scores[0][i] class_ind = int(out_classes[0][i]) bbox_color = colors[class_ind] bbox_thick = int(0.6 * (image_h + image_w) / 600) c1, c2 = (coor[1], coor[0]), (coor[3], coor[2]) cv2.rectangle(image, c1, c2, bbox_color, bbox_thick) if show_label: bbox_mess = '%s: %.2f' % (classes[class_ind], score) t_size = cv2.getTextSize(bbox_mess, 0, fontScale, thickness=bbox_thick // 2)[0] c3 = (c1[0] + t_size[0], c1[1] - t_size[1] - 3) cv2.rectangle(image, c1, (np.float32(c3[0]), np.float32(c3[1])), bbox_color, -1) #filled cv2.putText( image, bbox_mess, (c1[0], np.float32(c1[1] - 2)), cv2.FONT_HERSHEY_SIMPLEX, fontScale, (0, 0, 0), bbox_thick // 2, lineType=cv2.LINE_AA ) return image def get_anchors(anchors_path, tiny=False): anchors = np.array(anchors_path) if tiny: return anchors.reshape(2, 3, 2) else: return anchors.reshape(3, 3, 2) def decode_train(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=0, XYSCALE=[1, 1, 1]): conv_output = tf.reshape(conv_output, (tf.shape(conv_output)[0], output_size, output_size, 3, 5 + NUM_CLASS)) conv_raw_dxdy, conv_raw_dwdh, conv_raw_conf, conv_raw_prob = tf.split(conv_output, (2, 2, 1, NUM_CLASS), axis=-1) xy_grid = tf.meshgrid(tf.range(output_size), tf.range(output_size)) xy_grid = tf.expand_dims(tf.stack(xy_grid, axis=-1), axis=2) # [gx, gy, 1, 2] xy_grid = tf.tile(tf.expand_dims(xy_grid, axis=0), [tf.shape(conv_output)[0], 1, 1, 3, 1]) xy_grid = tf.cast(xy_grid, tf.float32) pred_xy = ((tf.sigmoid(conv_raw_dxdy) * XYSCALE[i]) - 0.5 * (XYSCALE[i] - 1) + xy_grid) * \ STRIDES[i] pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i]) pred_xywh = tf.concat([pred_xy, pred_wh], axis=-1) pred_conf = tf.sigmoid(conv_raw_conf) pred_prob = tf.sigmoid(conv_raw_prob) return tf.concat([pred_xywh, pred_conf, pred_prob], axis=-1) def yolo4_input_processing(original_image): image_data = cv2.resize(original_image, (416, 416)) image_data = image_data / 255. images_data = [] for i in range(1): images_data.append(image_data) images_data = np.asarray(images_data).astype(np.float32) batch_data = tf.constant(images_data) return batch_data def yolo4_output_processing(feature_maps): STRIDES = [8, 16, 32] ANCHORS = get_anchors([12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401]) NUM_CLASS = 80 XYSCALE = [1.2, 1.1, 1.05] iou_threshold = 0.45 score_threshold = 0.25 bbox_tensors = [] prob_tensors = [] score_thres = 0.2 for i, fm in enumerate(feature_maps): if i == 0: output_tensors = decode(fm, 416 // 8, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE) elif i == 1: output_tensors = decode(fm, 416 // 16, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE) else: output_tensors = decode(fm, 416 // 32, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE) bbox_tensors.append(output_tensors[0]) prob_tensors.append(output_tensors[1]) pred_bbox = tf.concat(bbox_tensors, axis=1) pred_prob = tf.concat(prob_tensors, axis=1) boxes, pred_conf = filter_boxes( pred_bbox, pred_prob, score_threshold=score_thres, input_shape=tf.constant([416, 416]) ) pred = {'concat': tf.concat([boxes, pred_conf], axis=-1)} for key, value in pred.items(): boxes = value[:, :, 0:4] pred_conf = value[:, :, 4:] boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression( boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)), scores=tf.reshape(pred_conf, (tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])), max_output_size_per_class=50, max_total_size=50, iou_threshold=iou_threshold, score_threshold=score_threshold ) output = [boxes.numpy(), scores.numpy(), classes.numpy(), valid_detections.numpy()] return output def result_to_json(image, pred_bbox): image_h, image_w, _ = image.shape out_boxes, out_scores, out_classes, num_boxes = pred_bbox class_names = {} json_result = [] with open('model/coco.names', 'r') as data: for ID, name in enumerate(data): class_names[ID] = name.strip('\n') nums_class = len(class_names) for i in range(num_boxes[0]): if int(out_classes[0][i]) < 0 or int(out_classes[0][i]) > nums_class: continue coor = out_boxes[0][i] coor[0] = int(coor[0] * image_h) coor[2] = int(coor[2] * image_h) coor[1] = int(coor[1] * image_w) coor[3] = int(coor[3] * image_w) score = float(out_scores[0][i]) class_ind = int(out_classes[0][i]) bbox = np.array([coor[1], coor[0], coor[3], coor[2]]).tolist() # [x1,y1,x2,y2] json_result.append({'image': None, 'category_id': class_ind, 'bbox': bbox, 'score': score}) return json_result def draw_boxes_and_labels_to_image_with_json(image, json_result, class_list, save_name=None): """Draw bboxes and class labels on image. Return the image with bboxes. Parameters ----------- image : numpy.array The RGB image [height, width, channel]. json_result : list of dict The object detection result with json format. classes_list : list of str For converting ID to string on image. save_name : None or str The name of image file (i.e. image.png), if None, not to save image. Returns ------- numpy.array The saved image. References ----------- - OpenCV rectangle and putText. - `scikit-image `__. """ image_h, image_w, _ = image.shape num_classes = len(class_list) hsv_tuples = [(1.0 * x / num_classes, 1., 1.) for x in range(num_classes)] colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors)) random.seed(0) random.shuffle(colors) random.seed(None) bbox_thick = int(0.6 * (image_h + image_w) / 600) fontScale = 0.5 for bbox_info in json_result: image_name = bbox_info['image'] category_id = bbox_info['category_id'] if category_id < 0 or category_id > num_classes: continue bbox = bbox_info['bbox'] # the order of coordinates is [x1, y2, x2, y2] score = bbox_info['score'] bbox_color = colors[category_id] c1, c2 = (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])) cv2.rectangle(image, c1, c2, bbox_color, bbox_thick) bbox_mess = '%s: %.2f' % (class_list[category_id], score) t_size = cv2.getTextSize(bbox_mess, 0, fontScale, thickness=bbox_thick // 2)[0] c3 = (c1[0] + t_size[0], c1[1] - t_size[1] - 3) cv2.rectangle(image, c1, (np.float32(c3[0]), np.float32(c3[1])), bbox_color, -1) cv2.putText( image, bbox_mess, (c1[0], np.float32(c1[1] - 2)), cv2.FONT_HERSHEY_SIMPLEX, fontScale, (0, 0, 0), bbox_thick // 2, lineType=cv2.LINE_AA ) if save_name is not None: save_image(image, save_name) return image