fuzz-scene-sim-test/FuzzScene/code/utils.py

import cv2
import math
import random
import time
import csv
import numpy as np
import pandas as pd
import tensorflow as tf
import skimage.transform as sktransform
import matplotlib.image as mpimg

from os import path
from collections import defaultdict
# add tensorflow.
from tensorflow.keras import backend as K
from tensorflow.keras.applications.imagenet_utils import preprocess_input
from tensorflow.keras.models import Model

from keras_preprocessing import image

#test
def get_start_point(orig_pixels,orig_shape,final_shape):
    final_pixels = [round(float(orig_pixels[0])*final_shape[0]/orig_shape[0]),round(float(orig_pixels[1])*final_shape[1]/orig_shape[1])]
    return final_pixels

def get_occl_size(orig_start,orig_end,orig_shape,final_shape):
    occl_size = [max(1,round((float(orig_end[0])-float(orig_start[0]))*final_shape[0]/orig_shape[0])),max(1,round((float(orig_end[1])-float(orig_start[1]))*final_shape[1]/orig_shape[1]))]
    return occl_size
def v3_preprocess(image, top_offset=.375, bottom_offset=.125,shape=(32,128)):
    """
    Applies preprocessing pipeline to an image: crops `top_offset` and `bottom_offset`
    portions of image, resizes to 32x128 px and scales pixel values to [0, 1].
    """
    top = int(top_offset * image.shape[0])
    bottom = int(bottom_offset * image.shape[0])
    if(bottom == 0):
        bottom = 1
    image = sktransform.resize(image[top:-bottom, :], shape)
    return image

def draw_arrow2(img, angle1,thickness = 1):
    pt1 = (int(img.shape[1] / 2), img.shape[0])
    pt2_angle1 = (int(img.shape[1] / 2 - img.shape[0] / 3 * math.sin(angle1)),
                  int(img.shape[0] - img.shape[0] / 3 * math.cos(angle1)))
    img = cv2.arrowedLine(img, pt1, pt2_angle1, (255, 0, 0), thickness)
    return img

def draw_arrow3(img, angle1,angle2):
    pt1 = (int(img.shape[1] / 2), img.shape[0])
    pt2_angle1 = (int(img.shape[1] / 2 - img.shape[0] / 3 * math.sin(angle1)),
                  int(img.shape[0] - img.shape[0] / 3 * math.cos(angle1)))
    pt2_angle2 = (int(img.shape[1] / 2 - img.shape[0] / 3 * math.sin(angle2)),
                  int(img.shape[0] - img.shape[0] / 3 * math.cos(angle2)))
    img = cv2.arrowedLine(img, pt1, pt2_angle1, (0, 0, 255), 5)
    img = cv2.arrowedLine(img, pt1, pt2_angle2, (0, 255, 0), 5)
    return img
def raw_draw_arrow3(img, angle1,angle2):
    pt1 = (int(img.shape[1] / 2), img.shape[0])
    pt2_angle1 = (int(img.shape[1] / 2 - img.shape[0] / 3 * math.sin(angle1)),
                  int(img.shape[0] - img.shape[0] / 3 * math.cos(angle1)))
    pt2_angle2 = (int(img.shape[1] / 2 - img.shape[0] / 3 * math.sin(angle2)),
                  int(img.shape[0] - img.shape[0] / 3 * math.cos(angle2)))
    img = cv2.arrowedLine(img, pt1, pt2_angle1, (1, 0, 0), 1)
    img = cv2.arrowedLine(img, pt1, pt2_angle2, (0, 1, 0), 1)
    return img


def draw_arrow(img, angle1, angle2, angle3):
    pt1 = (int(img.shape[1] / 2), img.shape[0])
    pt2_angle1 = (int(img.shape[1] / 2 - img.shape[0] / 3 * math.sin(angle1)),
                  int(img.shape[0] - img.shape[0] / 3 * math.cos(angle1)))
    pt2_angle2 = (int(img.shape[1] / 2 - img.shape[0] / 3 * math.sin(angle2)),
                  int(img.shape[0] - img.shape[0] / 3 * math.cos(angle2)))
    pt2_angle3 = (int(img.shape[1] / 2 - img.shape[0] / 3 * math.sin(angle3)),
                  int(img.shape[0] - img.shape[0] / 3 * math.cos(angle3)))
    img = cv2.arrowedLine(img, pt1, pt2_angle1, (0, 0, 255), 1)
    img = cv2.arrowedLine(img, pt1, pt2_angle2, (0, 255, 0), 1)
    img = cv2.arrowedLine(img, pt1, pt2_angle3, (255, 0, 0), 1)
    return img


def angle_diverged(angle1, angle2, angle3):
    if (abs(angle1 - angle2) > 0.2 or abs(angle1 - angle3) > 0.2 or abs(angle2 - angle3) > 0.2) and not (
                (angle1 > 0 and angle2 > 0 and angle3 > 0) or (
                                angle1 < 0 and angle2 < 0 and angle3 < 0)):
        return True
    return False

def angle_diverged2(angle1, angle2):
    if (abs(angle1 - angle2) > 0.1) and not (
                (angle1 > 0 and angle2 > 0 ) or (angle1 < 0 and angle2 < 0 )):
        return True
    return False

def angle_diverged3(angle1, angle2):
    if (abs(angle1 - angle2) > 0.1):
        return True
    return False


def preprocess_image(img_path, target_size=(100, 100)):
    img = image.load_img(img_path, target_size=target_size)
    input_img_data = image.img_to_array(img)
    input_img_data = np.expand_dims(input_img_data, axis=0)
    input_img_data = preprocess_input(input_img_data)
    return input_img_data

def preprocess_image_v3(img_path,top_offset,bottom_offset):
    orig_name = image.load_img(img_path)
    orig_name = image.img_to_array(orig_name)
    orig_name = orig_name.astype(np.uint8)
    gen_img = v3_preprocess(orig_name,top_offset = top_offset,bottom_offset = bottom_offset)
    gen_img = np.expand_dims(gen_img, axis=0)
    return gen_img

def preprocess_image_v2(img_path,top_offset,bottom_offset):
    orig_name = image.load_img(img_path)
    orig_name = image.img_to_array(orig_name)
    orig_name = orig_name.astype(np.uint8)
    gen_img = v3_preprocess(orig_name,top_offset = top_offset,bottom_offset = bottom_offset,shape=(66,200))
    gen_img = np.expand_dims(gen_img, axis=0)
    return gen_img
def deprocess_image(x,shape=(100,100,3)):
    tmp = x.copy()
    tmp = tmp.reshape(shape)
    # Remove zero-center by mean pixel
    tmp[:, :, 0] += 103.939
    tmp[:, :, 1] += 116.779
    tmp[:, :, 2] += 123.68
    # 'BGR'->'RGB'
    tmp = tmp[:, :, ::-1]
    tmp = np.clip(tmp, 0, 255).astype('uint8')
    return tmp


def atan_layer(x):
    return tf.multiply(tf.atan(x), 2)


def atan_layer_shape(input_shape):
    return input_shape


def normal_init(shape, dtype=None):
    return K.truncated_normal(shape, stddev=0.1)


def normalize(x):
    # utility function to normalize a tensor by its L2 norm
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)


def constraint_occl(gradients, start_point, rect_shape):
    new_grads = np.zeros_like(gradients)
    new_grads[:, start_point[0]:start_point[0] + rect_shape[0],
    start_point[1]:start_point[1] + rect_shape[1]] = gradients[:, start_point[0]:start_point[0] + rect_shape[0],
                                                     start_point[1]:start_point[1] + rect_shape[1]]
    return new_grads


def constraint_light(gradients):
    new_grads = np.ones_like(gradients)
    grad_mean = 500 * np.mean(gradients)
    return grad_mean * new_grads


def constraint_black(gradients, rect_shape=(10, 10)):
    start_point = (
        random.randint(0, gradients.shape[1] - rect_shape[0]), random.randint(0, gradients.shape[2] - rect_shape[1]))
    new_grads = np.zeros_like(gradients)
    patch = gradients[:, start_point[0]:start_point[0] + rect_shape[0], start_point[1]:start_point[1] + rect_shape[1]]
    if np.mean(patch) < 0:
        new_grads[:, start_point[0]:start_point[0] + rect_shape[0],
        start_point[1]:start_point[1] + rect_shape[1]] = -np.ones_like(patch)
    return new_grads


def init_coverage_tables(model1, model2, model3):
    model_layer_dict1 = defaultdict(bool)
    model_layer_dict2 = defaultdict(bool)
    model_layer_dict3 = defaultdict(bool)
    init_dict(model1, model_layer_dict1)
    init_dict(model2, model_layer_dict2)
    init_dict(model3, model_layer_dict3)
    return model_layer_dict1, model_layer_dict2, model_layer_dict3

def init_coverage_tables2(model1):
    model_layer_dict1 = defaultdict(bool)
    init_dict(model1, model_layer_dict1)
    return model_layer_dict1



def init_dict(model, model_layer_dict):
    for layer in model.layers:
        if 'flatten' in layer.name or 'input' in layer.name:
            continue
        for index in range(layer.output_shape[-1]):
            model_layer_dict[(layer.name, index)] = False


def neuron_to_cover(model_layer_dict):
    not_covered = [(layer_name, index) for (layer_name, index), v in model_layer_dict.items() if not v]
    if not_covered:
        layer_name, index = random.choice(not_covered)
    else:
        layer_name, index = random.choice(model_layer_dict.keys())
    return layer_name, index


def neuron_covered(model_layer_dict):
    covered_neurons = len([v for v in model_layer_dict.values() if v])
    total_neurons = len(model_layer_dict)
    return covered_neurons, total_neurons, covered_neurons / float(total_neurons)


def scale(intermediate_layer_output, rmax=1, rmin=0):
    X_std = (intermediate_layer_output - intermediate_layer_output.min()) / (
        intermediate_layer_output.max() - intermediate_layer_output.min())
    X_scaled = X_std * (rmax - rmin) + rmin
    return X_scaled


def update_coverage(input_data, model, model_layer_dict, threshold=0):
    layer_names = [layer.name for layer in model.layers if
                   'flatten' not in layer.name and 'input' not in layer.name]

    intermediate_layer_model = Model(inputs=model.input,
                                     outputs=[model.get_layer(layer_name).output for layer_name in layer_names])
    intermediate_layer_outputs = intermediate_layer_model.predict(input_data)

    for i, intermediate_layer_output in enumerate(intermediate_layer_outputs):
        scaled = scale(intermediate_layer_output[0])
        for num_neuron in range(scaled.shape[-1]):
            if np.mean(scaled[..., num_neuron]) > threshold and not model_layer_dict[(layer_names[i], num_neuron)]:
                model_layer_dict[(layer_names[i], num_neuron)] = True


def full_coverage(model_layer_dict):
    if False in model_layer_dict.values():
        return False
    return True

def fired(model, layer_name, index, input_data, threshold=0):
    intermediate_layer_model = Model(inputs=model.input, outputs=model.get_layer(layer_name).output)
    intermediate_layer_output = intermediate_layer_model.predict(input_data)[0]
    scaled = scale(intermediate_layer_output)
    if np.mean(scaled[..., index]) > threshold:
        return True
    return False


def diverged(predictions1, predictions2, predictions3, target):
    #     if predictions2 == predictions3 == target and predictions1 != target:
    if not predictions1 == predictions2 == predictions3:
        return True
    return False

def transform_occl(gradients,start_point,rect_shape,logo_data,order):
    new_grads = np.zeros((np.shape(gradients)[0],rect_shape[0],rect_shape[1],np.shape(gradients)[3]))
    new_grads = gradients[:, start_point[0]:start_point[0] + rect_shape[0],start_point[1]:start_point[1] + rect_shape[1]]
    for i in range(rect_shape[0]):
        for j in range(rect_shape[1]):          
            logo_shape = np.shape(logo_data)
            logo_data[order,round(logo_shape[1]*1.00*i/(rect_shape[0])),round(logo_shape[2]*1.00*j/(rect_shape[1])),:] = new_grads[0,i,j,:]
    return logo_data

#for pixel transform:  nearest interpolation
def transform_occl2(gradients,start_point,rect_shape,logo_data,order):
    new_grads = np.zeros((np.shape(gradients)[0],rect_shape[0],rect_shape[1],np.shape(gradients)[3]))
    new_grads = gradients[:, start_point[0]:start_point[0] + rect_shape[0],start_point[1]:start_point[1] + rect_shape[1]]
    logo_shape = np.shape(logo_data)
    for i in range(logo_shape[1]):
        for j in range(logo_shape[2]):          
            logo_data[order,i,j,:] = new_grads[0,round(rect_shape[0]*1.00*i/(logo_shape[1])-0.5),round(rect_shape[1]*1.00*j/(logo_shape[2])-0.5),:]
    return logo_data

#for pixel transform:  linear interpolation
def transform_occl3(gradients,start_point,rect_shape,logo_data,order):
    new_grads = np.zeros((np.shape(gradients)[0],rect_shape[0],rect_shape[1],np.shape(gradients)[3]))
    new_grads = gradients[:, start_point[0]:start_point[0] + rect_shape[0],start_point[1]:start_point[1] + rect_shape[1],:]
    logo_shape = np.shape(logo_data)
    #In this version (29/07/2018), we do not use own rescale code but opencv resize instead
    #print(np.shape(new_grads))
    logo_data[order] = cv2.resize(new_grads[0], dsize=(logo_shape[2], logo_shape[1]))
    '''
    for i in range(logo_shape[1]):
        for j in range(logo_shape[2]):
            y_approximation = rect_shape[0]*1.00*i/(logo_shape[1])
            x_approximation = rect_shape[1]*1.00*j/(logo_shape[2])
            y_offset =  y_approximation - int(y_approximation)
            x_offset = x_approximation -int(x_approximation)
            logo_data[order,i,j,:] = new_grads[0,int(y_approximation),int(x_approximation),:] * (1-x_offset)*(1-y_offset) + \
                                     new_grads[0,min(rect_shape[0]-1,int(y_approximation)+1),int(x_approximation),:] * (1-x_offset)*(y_offset) + \
                                     new_grads[0,int(y_approximation),min(rect_shape[1]-1,int(x_approximation)+1),:] * (x_offset)*(1-y_offset) + \
                                     new_grads[0,min(rect_shape[0]-1,int(y_approximation)+1),min(rect_shape[1]-1,int(x_approximation)+1),:] * (x_offset)*(y_offset)
    '''
    return logo_data

def transfrom_accurate(gradients,des_pixel,logo_data,order):
    des = np.array([[0.1, 0], [np.shape(logo_data[0])[1]-1, 0], [0, np.shape(logo_data[0])[0]-1], [np.shape(logo_data[0])[1]-1, np.shape(logo_data[0])[0]-1]],np.float32)
    transform = cv2.getPerspectiveTransform(des_pixel,des)
    logo_data[order] = cv2.warpPerspective(gradients,transform,( np.shape(logo_data[0])[1], np.shape(logo_data[0])[0]))
    return logo_data
def update_image(imgs,logo,start_point,occl_size):
    for i in range(len(imgs)):
        imgs[i][0,start_point[i][0]:start_point[i][0]+occl_size[i][0],start_point[i][1]:start_point[i][1]+occl_size[i][1],:] = cv2.resize(logo,(occl_size[i][1],occl_size[i][0]))[:min(occl_size[i][0],np.shape(imgs[i])[1]-start_point[i][0]),:min(occl_size[i][1],np.shape(imgs[i])[2]-start_point[i][1])]
    return imgs
#def accurate_logo_grad(gradients,des_pixels):

def accurate_update(imgs,logo,des_pixels):
    src = np.array([[0, 0], [np.shape(logo)[1]-1, 0], [0, np.shape(logo)[0]-1], [np.shape(logo)[1]-1, np.shape(logo)[0]-1]],np.float32)
    for i in range(len(imgs)):
        transform = cv2.getPerspectiveTransform(src,des_pixels[i])
        output = cv2.warpPerspective(logo,transform,( np.shape(imgs[i])[2], np.shape(imgs[i])[1]),flags=cv2.INTER_LINEAR )
        mask = output.astype(np.bool)
        mask = mask.astype(np.float32)
        back = cv2.multiply(1.0-mask,imgs[i][0])
        imgs[i][0] = np.array(cv2.add(back,np.array(output,dtype=np.float32)))
    return imgs

def control_bound(logo):
    np.clip(logo[:,:,0], -103.939, 255-103.939, out=logo[:,:,0])
    np.clip(logo[:,:,1], -116.779, 255-116.779, out=logo[:,:,1])
    np.clip(logo[:,:,2], -123.68, 255-123.68, out=logo[:,:,2])
    return logo

def raw_control_bound(logo):
    np.clip(logo, 0, 1, out=logo)
    return logo
def preprocess_color(color):
    color[0] = color[0] -103.939
    color[1] = color[1] -116.779
    color[2] = color[2] - 123.68
    return color

def total_diff(imgs,model1,angles_2):
    angles_diff = []
    count = 0
    for img in imgs:
        angles_diff.append(abs(model1.predict(img)[0]-angles_2[count]))
        count+=1
    return sum(angles_diff)

def gen_optimal(imgs,model1,angle3,start_points,occl_sizes):
    logo = np.zeros((480,640,3))
    result = {"pixel_value":np.zeros([10,3]),"diff":np.zeros(10)}
    for blue in range(0,256,51):
        for green in range(0,256,51):
            #print("[",blue,green,":] claculated, current result is,",result)
            for red in range(0,256,51):
                logo[:,:] = preprocess_color([blue,green,red])
                #imgs = accurate_update(imgs,logo,des_pixels)
                imgs = update_image(imgs,logo,start_points,occl_sizes)
                this_diff = total_diff(imgs,model1,angle3)
                this_diff = this_diff/len(imgs) * 180 / math.pi
                if(this_diff > result["diff"][0]):
                    result["pixel_value"][0] = np.array([blue,green,red])
                    result["diff"][0] =  this_diff
                    index = np.argsort(result["diff"])
                    result["diff"] = result["diff"][index]
                    result["pixel_value"] = result["pixel_value"][index]
    print(result)
    return preprocess_color(result["pixel_value"][-1])

def raw_gen_optimal(imgs,model1,angle3,start_points,occl_sizes):
    logo = np.zeros((480,640,3))
    result = {"pixel_value":np.zeros([10,3]),"diff":np.zeros(10)}
    for blue in range(0,256,51):
        for green in range(0,256,51):
            #print("[",blue,green,":] claculated, current result is,",result)
            for red in range(0,256,51):
                logo[:,:] = [blue/255.0,green/255.0,red/255.0]
                #imgs = accurate_update(imgs,logo,des_pixels)
                imgs = update_image(imgs,logo,start_points,occl_sizes)
                this_diff = total_diff(imgs,model1,angle3)
                this_diff = this_diff/len(imgs) * 180 / math.pi
                if(this_diff > result["diff"][0]):
                    result["pixel_value"][0] = np.array([blue/255.0,green/255.0,red/255.0])
                    result["diff"][0] =  this_diff
                    index = np.argsort(result["diff"])
                    result["diff"] = result["diff"][index]
                    result["pixel_value"] = result["pixel_value"][index]
    print(result)
    return result["pixel_value"][-1]

def read_input(imgs,des_pixels,occl_sizes,start_points,filelist,coordination,is_crop = False,is_all = False):
    if(is_crop):
        img_size = [600,800]
    else:
        img_size = [1080,1920]
    for f in sorted(filelist):
        orig_name = f
        gen_img = preprocess_image(orig_name)
        imgs.append(gen_img)
    with open(coordination) as f:
        if(is_all):
            spamreader = csv.reader(f)
        else:
            spamreader = csv.reader(f, delimiter=',', quotechar='|')
        for row in spamreader:
            if(len(row) != 9):
                print(len(row))
                continue
            tmp = np.array([[float(row[2])*100.0/img_size[1],float(row[1])*100.0/img_size[0]],[float(row[6])*100.0/img_size[1],float(row[5])*100.0/img_size[0]],[float(row[4])*100.0/img_size[1],float(row[3])*100.0/img_size[0]],[float(row[8])*100.0/img_size[1],float(row[7])*100.0/img_size[0]]],np.float32)
            des_pixels.append(tmp)
            start_points.append([round(float(row[1])*100.0/img_size[0]),round(float(row[2])*100.0/img_size[1])])
            occl_sizes.append([round((float(row[7])-float(row[1]))*100.0/img_size[0]),round((float(row[8])-float(row[2]))*100.0/img_size[1])])
    print(coordination + " read complete")
    return imgs,des_pixels,occl_sizes,start_points
def find_kth_max(array,k):
    tmp = array.flatten()
    tmp = abs(tmp)
    tmp.sort()
    return tmp[-k]