model_accuracy/server/lemon_master/scripts/localization/analysis_inter_inputs.py

import sys
import os
import numpy as np
import math
import pandas as pd
import warnings
import configparser
from itertools import combinations

warnings.filterwarnings("ignore")
os.environ["TF_CPP_MIN_LOG_LEVEL"] = '2' # 只显示 warning 和 Error
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = ""
if __name__ == '__main__':

    # get experiments configuration
    config_name = sys.argv[1]
    lemon_cfg = configparser.ConfigParser()
    lemon_cfg.read(f"./config/{config_name}")
    parameters = lemon_cfg['parameters']
    output_dir = parameters['output_dir']
    output_dir = output_dir[:-1] if output_dir.endswith("/") else output_dir
    threshold = parameters.getfloat('threshold')
    current_container = output_dir.rstrip("/").split("/")[-1]

    pre_backends = parameters['backend'].split(" ")
    backends_str = "-".join(pre_backends)
    backend_pairs = [f"{pair[0]}_{pair[1]}" for pair in combinations(pre_backends, 2)]

    compare_columns = ["original","mutant(max)"]
    """Get all exps"""
    exps = parameters['exps'].lstrip().rstrip().split(" ")
    exps.sort(key=lambda x:x)
    backends_exp_indntfrs = {bk_p:{exp_id:list() for exp_id in exps} for bk_p in backend_pairs}
    for exp in exps:
        print('#########{}#########'.format(exp))
        exp_dir = os.path.join(output_dir, exp)
        metrics_dir = os.path.join(exp_dir, "metrics_result")
        exp_metrics_path = os.path.join(metrics_dir, "{}_D_MAD_result.csv".format(exp))
        with open(exp_metrics_path, "r") as fr:
            lines = fr.readlines()[1:]
            for line in lines:
                line_split = line.split(",")
                # incon_idntfr like mobilenet.1.00.224-imagenet_origin0_theano_cntk_input1494
                incon_idntfr, incon_value = line_split[0], float(line_split[1])
                # print(incon_value)
                incon_idntfr_split = incon_idntfr.split("_")
                incon_tuple = (incon_idntfr, incon_value )
                backend_pair,input_key = "{}_{}".format(incon_idntfr_split[2],incon_idntfr_split[3]),incon_idntfr_split[-1]
                if not math.isnan(incon_value):
                    backends_exp_indntfrs[backend_pair][exp].append(incon_tuple)

    all_inputs_name = {bk_p: {exp_id: set() for exp_id in exps} for bk_p in backend_pairs}

    for bk_pair in backend_pairs:
        exp_idntfrs = backends_exp_indntfrs[bk_pair]
        for exp_id,incon_idntfrs in exp_idntfrs.items():
            for incon_tuple in incon_idntfrs:
                (incon_idntfr, incon_value) = incon_tuple
                incon_idntfr_split = incon_idntfr.split("_")
                input_k = incon_idntfr_split[-1]
                model_type = incon_idntfr_split[1]
                if incon_value >= threshold:
                    all_inputs_name[bk_pair][exp_id].add(input_k)

    for bk_pair in backend_pairs:
        exp_idntfrs = backends_exp_indntfrs[bk_pair]
        exp_analysis = {}
        for exp_id,incon_idntfrs in exp_idntfrs.items():
            exp_analysis[exp_id] = {input_name:[0,0] for input_name in all_inputs_name[bk_pair][exp_id]}
            for incon_tuple in incon_idntfrs:
                (incon_idntfr, incon_value) = incon_tuple
                incon_idntfr_split = incon_idntfr.split("_")
                input_k = incon_idntfr_split[-1]
                if input_k in exp_analysis[exp_id].keys():
                    model_type = incon_idntfr_split[1]
                    if model_type == "origin0":
                        exp_analysis[exp_id][input_k][0] = incon_value
                    else:
                        if incon_value > exp_analysis[exp_id][input_k][1]:
                            exp_analysis[exp_id][input_k][1] = incon_value

            input_keys = list(exp_analysis[exp_id].keys())
            input_keys.sort()
            data = np.zeros((len(input_keys), 2), dtype=np.float)
            tuples = []
            df = pd.DataFrame(data, columns=compare_columns,index=input_keys)

            for input_idx in input_keys:
                df.loc[input_idx,compare_columns[0]] = exp_analysis[exp_id][input_idx][0]
                df.loc[input_idx,compare_columns[1]] = exp_analysis[exp_id][input_idx][1]

            # print(df)
            save_dir = os.path.join(output_dir, exp_id,"{}_inter".format(exp_id))
            if not os.path.exists(save_dir):
                os.makedirs(save_dir)
            df.to_csv(os.path.join(save_dir,"{}.csv".format(bk_pair)))