添加了图片输出

server/README.md

@@ -6,7 +6,7 @@ python -u mutation_executor.py
 --mutate_num {} # int like 10 变异次数
 --config_name {} # str like demo.conf 配置文件
 
-example: python -u mutation_executor.py --exp fashion2 --root_dir /app/LEMON-master --output_dir /app/output --mutate_num 10 --config_name demo.conf
+example: python -u mutation_executor.py --exp fashion2 --root_dir /home/ysyy/mooctest/ma/model_accuracy/server/lemon_master --output_dir /home/ysyy/mooctest/ma/model_accuracy/server/output/task_test --mutate_num 1 --config_name demo.conf
 
 # 参数设置
 /LEMON-master/origin_model # 种子模型文件夹

server/app.py

@@ -1,4 +1,4 @@
-from flask import Flask, request
+from flask import Flask, request,send_file
 from tool import runtool
 import yaml
 import os
@@ -29,27 +29,60 @@ def get_task_list():
     
     task_list = []
     task_list = os.listdir(task_output_dir)
-    # 按照时间排序
-    task_list.sort()
-    
-    # 检查是否超过了最大任务数
-    max_task_num = app.config['max_task_num']
-    if len(task_list) > max_task_num:
-        # 删除最早的任务
-        task_list.sort()
-        for i in range(len(task_list) - max_task_num):
-            task_dir = os.path.join(task_output_dir, task_list[i])
-            os.system("rm -rf " + task_dir)
+    # 文件夹的名称是按照task_<task_num>_...来的, 按照task_num排序
     
-    task_list = os.listdir(task_output_dir)
-    task_list.sort()
-    return task_list
+    task_list_dict = {}
+    
+    for task_dir in task_list:
+        task_num = int(task_dir.split('_')[1])
+        task_list_dict[task_num] = task_dir
+    
+    task_num_list = list(task_list_dict.keys())
+    task_num_list.sort()
+    
+    return_list = []
+    for task_num in task_num_list:
+        return_list.append(task_list_dict[task_num])
+    return return_list
 
+def get_max_task_id():
+    
+    task_list = []
+    task_list = os.listdir(task_output_dir)
+    # 文件夹的名称是按照task_<task_id>_...来的，取出最大的task_id
+    task_id_to_task_dir = {}
+    task_id_list = []
+    for task_dir in task_list:
+        task_id = int(task_dir.split('_')[1])
+        task_id_list.append(task_id)
+        task_id_to_task_dir[task_id] = task_dir
+    
+    max_task_id =  max(task_id_list)
+    
+    # 检查是否超过了最大任务数，删除task_num最小的任务直到任务数小于最大任务数
+    print(len(task_list))
+    print(app.config['max_task_num'])
+    if len(task_list) > app.config['max_task_num']:
+        print("超过最大任务数，删除最早的任务直到任务数小于最大任务数")
+        task_id_list.sort()
+        over_num = len(task_list) - app.config['max_task_num']
+        for i in range(over_num):
+            task_id = task_id_list[i]
+            task_dir = task_id_to_task_dir[task_id]
+            task_dir_path = os.path.join(task_output_dir, task_dir)
+            print("删除任务:", task_dir_path)
+            os.system("rm -rf " + task_dir_path)
+        
+    
+    return max_task_id
+    
 def get_result(task_id:str):
     result_dir = os.path.join(task_output_dir, task_id)
     # 查看result_dir的下一级文件夹
     
-    result_model_dir = os.path.join(result_dir,os.listdir(result_dir)[0])
+    result_model_name = os.listdir(result_dir)[0]
+    
+    result_model_dir = os.path.join(result_dir,result_model_name)
     print(result_model_dir)
     
     mxnet_json = os.path.join(result_model_dir, "mxnet.json")
@@ -61,10 +94,20 @@ def get_result(task_id:str):
     tensorflow_json = os.path.join(result_model_dir, "tensorflow.json")
     with open(tensorflow_json, "r") as file2:
         data2 = json.load(file2)
-        
+    
+    img_root_relative_path = task_id + '/' + result_model_name + '/'
+    
+    
     combined_data = {
         "mxnet": data1,
-        "tensorflow": data2
+        "tensorflow": data2,
+        "img_path":{
+            "mxnet_train":img_root_relative_path + 'mxnet_train.jpg',
+            "tensorflow_train": img_root_relative_path + 'tensorflow_train.jpg',
+            "accuracy": img_root_relative_path + 'accuracy.jpg',
+            "losses": img_root_relative_path + 'losses.jpg',
+            "memory": img_root_relative_path + 'memory.jpg'
+        }
     }
     return combined_data
     
@@ -87,9 +130,9 @@ def run():
     exp = data['exp']
     mutate_num = data['mutate_num']
     
-    #创建task_id: task_2021-09-07T14:57:00_1
-    task_id = 'task_' + time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) + '_' + str(len(get_task_list())+1)
-    
+    #创建task_id: task_ task_num exp mutate_num
+    # task_id = 'task_' + time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime()) + '_' + str(len(get_task_list())+1)
+    task_id = 'task_' + str(get_max_task_id()+1) + '_' + exp + '_' + str(mutate_num)
     # 开启一个线程执行，直接返回请求
     # data = runtool(exp, mutate_num,task_id)
     t = threading.Thread(target=runtool, args=(exp, mutate_num,task_id))
@@ -112,7 +155,16 @@ def get_task_result():
     except Exception as e:
         return {'error:':'错误的task_id或任务未完成'}
     
+@app.route('/image/<path:filename>')
+def serve_image(filename):
+    # 指定图片所在的根目录
     
+    # 构建完整的图片路径
+    image_path = os.path.join(task_output_dir, filename)
+    
+    # 使用send_file()函数返回图片
+    return send_file(image_path, mimetype='image/jpeg')
+
 if __name__ == '__main__':
     # app.run()
     from waitress import serve

server/config.yml

@@ -1,4 +1,4 @@
 default:
   output_dir: server/lemon_master/lemon_outputs
   config_name: demo.conf
-  max_task_num: 9
+  max_task_num: 30

server/lemon_master/run/draw_result.py

@@ -0,0 +1,142 @@
+import json
+import matplotlib.pyplot as plt
+import argparse
+import sys
+# Load the data from the provided JSON file
+def draw_pdf(json_file_path,pdf_file_path,bk):
+    plt.switch_backend('Agg')
+
+    json_file_path = json_file_path
+
+    with open(json_file_path, 'r') as file:
+        data = json.load(file)
+
+    # Extracting the relevant data
+    losses = []
+    accuracy = []
+    memory = []
+
+    for entry in data:
+        result = entry['result']
+        losses.append(result['Losses'])
+        accuracy.append(result['Accuracy'])
+        memory.append(result['MemoryInfoList'])
+
+    # Plotting the data
+    plt.figure(figsize=(10, 6))
+
+    # Plot Losses
+    plt.plot(losses, label='Losses', marker='o')
+
+    # Plot Accuracy
+    plt.plot(accuracy, label='Accuracy', marker='x')
+
+    # Plot Memory Info List
+    plt.plot(memory, label='Memory', marker='s')
+
+    # Adding title and labels
+    plt.title(bk)
+    plt.xlabel('Iterations')
+    plt.ylabel('Values')
+    plt.legend()
+
+    # Display the plot
+    plt.grid(True)
+
+    pdf_file_path = pdf_file_path
+    plt.savefig(pdf_file_path)
+
+    plt.close()
+
+def draw_pdf_all(tensorflow_json_path,mxnet_json_path,losses_pdf_path, accuracy_pdf_path, memory_pdf_path):
+
+    with open(tensorflow_json_path, 'r') as file:
+        tensorflow_data = json.load(file)
+
+    with open(mxnet_json_path, 'r') as file:
+        mxnet_data = json.load(file)
+
+    # Extracting the relevant data
+    tf_losses = [entry['result']['Losses'] for entry in tensorflow_data]
+    tf_accuracy = [entry['result']['Accuracy'] for entry in tensorflow_data]
+    tf_memory = [entry['result']['MemoryInfoList'] for entry in tensorflow_data]
+
+    mx_losses = [entry['result']['Losses'] for entry in mxnet_data]
+    mx_accuracy = [entry['result']['Accuracy'] for entry in mxnet_data]
+    mx_memory = [entry['result']['MemoryInfoList'] for entry in mxnet_data]
+
+    # Creating plots for Losses, Accuracy, and Memory
+    # Separate the plots into individual figures and save each as a PDF
+
+    # Plot Losses
+    plt.figure(figsize=(10, 6))
+    plt.plot(tf_losses, label='TensorFlow Losses', marker='o')
+    plt.plot(mx_losses, label='MXNet Losses', marker='x')
+    plt.title('Losses Over Iterations')
+    plt.xlabel('Iterations')
+    plt.ylabel('Losses')
+    plt.legend()
+    plt.grid(True)
+    plt.tight_layout()
+
+    plt.savefig(losses_pdf_path)
+    plt.close()
+
+    # Plot Accuracy
+    plt.figure(figsize=(10, 6))
+    plt.plot(tf_accuracy, label='TensorFlow Accuracy', marker='o')
+    plt.plot(mx_accuracy, label='MXNet Accuracy', marker='x')
+    plt.title('Accuracy Over Iterations')
+    plt.xlabel('Iterations')
+    plt.ylabel('Accuracy')
+    plt.legend()
+    plt.grid(True)
+    plt.tight_layout()
+
+    plt.savefig(accuracy_pdf_path)
+    plt.close()
+
+    # Plot Memory
+    plt.figure(figsize=(10, 6))
+    plt.plot(tf_memory, label='TensorFlow Memory', marker='o')
+    plt.plot(mx_memory, label='MXNet Memory', marker='x')
+    plt.title('Memory Usage Over Iterations')
+    plt.xlabel('Iterations')
+    plt.ylabel('Memory Info List')
+    plt.legend()
+    plt.grid(True)
+    plt.tight_layout()
+
+    plt.savefig(memory_pdf_path)
+    plt.close()
+
+
+
+if __name__ == "__main__":
+
+    """Parser of command args"""
+    parse = argparse.ArgumentParser()
+    parse.add_argument("--backend", type=str, help="name of backends")
+    parse.add_argument("--model_path", type=str, help="redis db port")
+    flags, unparsed = parse.parse_known_args(sys.argv[1:])
+
+    bk = ['tensorflow', 'mxnet']
+    model_path = flags.model_path
+    for i in bk:
+        if 'svhn' in model_path or 'fashion2' in model_path:
+            txt_path = model_path.split("\\")[-1][:-5].split("mut_model")[0]+i+"_train.json"
+        else:
+            txt_path = model_path.split("\\")[-1][:-3].split("mut_model")[0]+i+"_train.json"
+
+        pdf_path = txt_path.replace(".json",".jpg")
+        draw_pdf(txt_path,pdf_path,i)
+
+    tensorflow_json_path = model_path.split("\\")[-1][:-5].split("mut_model")[0]+'tensorflow_train'+".json"
+    mxnet_json_path = model_path.split("\\")[-1][:-5].split("mut_model")[0]+'mxnet_train'+".json"
+
+    losses_pdf_path = model_path.split("\\")[-1][:-5].split("mut_model")[0]+"losses"+".jpg"
+    accuracy_pdf_path = model_path.split("\\")[-1][:-5].split("mut_model")[0]+"accuracy"+".jpg"
+    memory_pdf_path = model_path.split("\\")[-1][:-5].split("mut_model")[0]+"memory"+".jpg"
+    
+
+    draw_pdf_all(tensorflow_json_path,mxnet_json_path,losses_pdf_path, accuracy_pdf_path, memory_pdf_path)

server/lemon_master/run/model_to_txt.py

@@ -37,6 +37,25 @@ def save_log_txt(model, path, name, bk, x_train, y_train):
         }
     }
 
+
+    iterations_data = []
+    for i in range(len(history_loss.losses)):
+        iterations_data.append({
+            "Iterations": i + 1,
+            "result": {
+                "Losses": float(history_loss.losses[i]),
+                "Accuracy": float(history_loss.acc[i]),
+                "MemoryInfoList": float(history_loss.mem_info_list[i])
+            }
+        })
+
+
+    iterations_data_path = path.replace(".json","_train.json")
+    with open(iterations_data_path, 'w') as json_file:
+        json.dump(iterations_data, json_file, indent=4)
+
+
+
     if os.path.exists(path):
         with open(path, 'r') as json_file:
             data = json.load(json_file)
@@ -140,4 +159,4 @@ if __name__ == "__main__":
     import keras
     print("Using {} as backend for states extraction| {} is wanted".format(K.backend(),bk))
     """Get model hidden output on selected_index data on specific backend"""
-    model_to_txt1(flags.model_path, bk)
+    model_to_txt1(flags.model_path, bk)

server/lemon_master/run/mutate_lemon.py

@@ -5,11 +5,9 @@ import keras
 import json
 import networkx as nx
 import sys
-
 # sys.path.append("../")
 import os
-
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
 
 from scripts.logger.lemon_logger import Logger
 from scripts.tools.mutator_selection_logic import Roulette, MCMC
@@ -34,7 +32,7 @@ import math
 lines = 0
 # np.random.seed(20200501)
 warnings.filterwarnings("ignore")
-os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = '2'
 os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
 os.environ["CUDA_VISIBLE_DEVICES"] = ""
 import psutil
@@ -78,11 +76,7 @@ def save_mutate_history(selector, invalid_history: dict, mutant_history: list):
         for op in invalid_history.keys():
             mtrs = selector.mutators[op]
             invalid_cnt = invalid_history[op]
-            fw.write(
-                "{},{},{},{}\n".format(
-                    op, mtrs.delta_bigger_than_zero, invalid_cnt, mtrs.total
-                )
-            )
+            fw.write("{},{},{},{}\n".format(op, mtrs.delta_bigger_than_zero, invalid_cnt, mtrs.total))
     with open(mutant_history_path, "w+") as fw:
         for mutant in mutant_history:
             fw.write("{}\n".format(mutant))
@@ -96,33 +90,27 @@ def is_nan_or_inf(t):
 
 
 def continue_checker(**run_stat):  # 判断算法是否满足退出条件
-    start_time = run_stat["start_time"]
-    time_limitation = run_stat["time_limit"]
-    cur_counters = run_stat["cur_counters"]
-    counters_limit = run_stat["counters_limit"]
-    s_mode = run_stat["stop_mode"]
+    start_time = run_stat['start_time']
+    time_limitation = run_stat['time_limit']
+    cur_counters = run_stat['cur_counters']
+    counters_limit = run_stat['counters_limit']
+    s_mode = run_stat['stop_mode']
 
     # if timing
     # 时间限制
-    if s_mode == "TIMING":
-        hours, minutes, seconds = utils.ToolUtils.get_HH_mm_ss(
-            datetime.datetime.now() - start_time
-        )
+    if s_mode == 'TIMING':
+        hours, minutes, seconds = utils.ToolUtils.get_HH_mm_ss(datetime.datetime.now() - start_time)
         total_minutes = hours * 60 + minutes
-        mutate_logger.info(
-            f"INFO: Mutation progress: {total_minutes}/{time_limitation} Minutes!"
-        )
+        mutate_logger.info(f"INFO: Mutation progress: {total_minutes}/{time_limitation} Minutes!")
         if total_minutes < time_limitation:
             return True
         else:
             return False
     # if counters
     # 次数限制,size(models)<N
-    elif s_mode == "COUNTER":
+    elif s_mode == 'COUNTER':
         if cur_counters < counters_limit:
-            mutate_logger.info(
-                "INFO: Mutation progress {}/{}".format(cur_counters + 1, counters_limit)
-            )
+            mutate_logger.info("INFO: Mutation progress {}/{}".format(cur_counters + 1, counters_limit))
             return True
         else:
             return False
@@ -142,9 +130,7 @@ def calc_inner_div(model):
     return len(longest_path) / len(graph)
 
 
-def _generate_and_predict(
-    res_dict, filename, mutate_num, mutate_ops, test_size, exp, backends
-):
+def _generate_and_predict(res_dict, filename, mutate_num, mutate_ops, test_size, exp, backends):
     # 主算法函数
     """
     Generate models using mutate operators and store them
@@ -153,31 +139,27 @@ def _generate_and_predict(
     mutate_op_invalid_history = {k: 0 for k in mutate_ops}
     mutant_history = []
     # get mutator selection strategy
-    if "svhn" in exp or "fashion2" in exp:
+    if 'svhn' in exp or 'fashion2' in exp:
         origin_model_name = "{}_origin0.hdf5".format(exp)
     else:
         origin_model_name = "{}_origin0.h5".format(exp)
     # 初始种子模型列表Ms初始时只有这一个模型
     root_dir = os.path.dirname(os.getcwd())
-
+    
     origin_save_path = os.path.join(mut_dir, origin_model_name)
-    mutator_selector_func, mutant_selector_func = get_selector_by_startegy_name(
-        mutator_strategy, mutant_strategy
-    )
+    mutator_selector_func, mutant_selector_func = get_selector_by_startegy_name(mutator_strategy, mutant_strategy)
     # [origin_model_name] means seed pool only contains initial model at beginning.
-    mutator_selector, mutant_selector = mutator_selector_func(
-        mutate_ops
-    ), mutant_selector_func([origin_model_name], capacity=mutate_num + 1)
+    mutator_selector, mutant_selector = mutator_selector_func(mutate_ops), mutant_selector_func([origin_model_name],
+                                                                                                capacity=mutate_num + 1)
     # MCMC,Roulette
     shutil.copy(src=filename, dst=origin_save_path)
-    origin_model_status, res_dict, accumulative_inconsistency, _ = get_model_prediction(
-        res_dict, origin_save_path, origin_model_name, exp, test_size, backends
-    )
+    origin_model_status, res_dict, accumulative_inconsistency, _ = get_model_prediction(res_dict,
+                                                                                        origin_save_path,
+                                                                                        origin_model_name, exp,
+                                                                                        test_size, backends)
 
     if not origin_model_status:
-        mutate_logger.error(
-            f"Origin model {exp} crashed on some backends! LEMON would skip it"
-        )
+        mutate_logger.error(f"Origin model {exp} crashed on some backends! LEMON would skip it")
         sys.exit(-1)
 
     last_used_mutator = None
@@ -186,39 +168,27 @@ def _generate_and_predict(
 
     start_time = datetime.datetime.now()
     order_inconsistency_dict = {}
-    run_stat = {
-        "start_time": start_time,
-        "time_limit": time_limit,
-        "cur_counters": mutant_counter,
-        "counters_limit": mutate_num,
-        "stop_mode": stop_mode,
-    }
+    run_stat = {'start_time': start_time, 'time_limit': time_limit, 'cur_counters': mutant_counter,
+                'counters_limit': mutate_num, 'stop_mode': stop_mode}
 
     # 满足限制条件就继续循环
     while continue_checker(**run_stat):
         global model_num
         if model_num == mutate_num:
             break
-        picked_seed = utils.ToolUtils.select_mutant(
-            mutant_selector
-        )  # 轮盘赌选择种子模型(伪代码3-14行)
-        selected_op = utils.ToolUtils.select_mutator(
-            mutator_selector, last_used_mutator=last_used_mutator
-        )  # 蒙特卡洛选择变异算子(伪代码15-20行)
+        picked_seed = utils.ToolUtils.select_mutant(mutant_selector)  # 轮盘赌选择种子模型(伪代码3-14行)
+        selected_op = utils.ToolUtils.select_mutator(mutator_selector,
+                                                     last_used_mutator=last_used_mutator)  # 蒙特卡洛选择变异算子(伪代码15-20行)
         mutate_op_history[selected_op] += 1
         last_used_mutator = selected_op
         mutator = mutator_selector.mutators[selected_op]  # 变异算子对象
         mutant = mutant_selector.mutants[picked_seed]  # 种子模型对象
 
-        if "svhn" in picked_seed or "fashion2" in picked_seed:
-            new_seed_name = "{}-{}{}.hdf5".format(
-                picked_seed[:-5], selected_op, mutate_op_history[selected_op]
-            )
+        if 'svhn' in picked_seed or 'fashion2' in picked_seed:
+            new_seed_name = "{}-{}{}.hdf5".format(picked_seed[:-5], selected_op, mutate_op_history[selected_op])
 
         else:
-            new_seed_name = "{}-{}{}.h5".format(
-                picked_seed[:-3], selected_op, mutate_op_history[selected_op]
-            )  # 生成新模型
+            new_seed_name = "{}-{}{}.h5".format(picked_seed[:-3], selected_op, mutate_op_history[selected_op])  # 生成新模型
         # seed name would not be duplicate
         if new_seed_name not in mutant_selector.mutants.keys():
             # 对应伪代码22行，因为种子模型是以当前选择的种子模型和变异算子命名的，所以重名就表示这个模型已经存在了
@@ -226,78 +196,54 @@ def _generate_and_predict(
             picked_seed_path = os.path.join(mut_dir, picked_seed)
             mutate_st = datetime.datetime.now()
 
-            model_mutation_generators = (
-                root_dir + "/scripts/mutation/model_mutation_generators.py"
-            )
-            mutate_status = os.system(
-                "{}/lemon/bin/python -u {} --model {} "
-                "--mutate_op {} --save_path {} --mutate_ratio {}".format(
-                    python_prefix,
-                    model_mutation_generators,
-                    picked_seed_path,
-                    selected_op,
-                    new_seed_path,
-                    flags.mutate_ratio,
-                )
-            )
+            model_mutation_generators = root_dir + "/scripts/mutation/model_mutation_generators.py"
+            mutate_status = os.system("{}/lemon/bin/python -u {} --model {} "
+                                      "--mutate_op {} --save_path {} --mutate_ratio {}".format(python_prefix,
+                                                                                               model_mutation_generators,
+                                                                                               picked_seed_path,
+                                                                                               selected_op,
+                                                                                               new_seed_path,
+                                                                                               flags.mutate_ratio))
             # 使用变异算子进行变异(伪代码21行)
 
             mutate_et = datetime.datetime.now()
             mutate_dt = mutate_et - mutate_st
             h, m, s = utils.ToolUtils.get_HH_mm_ss(mutate_dt)
-            mutate_logger.info(
-                "INFO:Mutate Time Used on {} : {}h, {}m, {}s".format(
-                    selected_op, h, m, s
-                )
-            )
+            mutate_logger.info("INFO:Mutate Time Used on {} : {}h, {}m, {}s".format(selected_op, h, m, s))
             # mutation status code is successful
 
             if mutate_status == 0:  # 变异执行完成
                 mutant.selected += 1
                 mutator.total += 1
                 # execute this model on all platforms
-                predict_status, res_dict, accumulative_inconsistency, model_outputs = (
-                    get_model_prediction(
-                        res_dict, new_seed_path, new_seed_name, exp, test_size, backends
-                    )
-                )
+                predict_status, res_dict, accumulative_inconsistency, model_outputs = \
+                    get_model_prediction(res_dict, new_seed_path, new_seed_name, exp, test_size, backends)
                 # 计算ACC(m)
 
                 if predict_status:
                     mutant_history.append(new_seed_name)
                     # 伪代码23-25行
-                    print("type:", type(model_outputs))
-                    print("model_outputs:", model_outputs)
+                    print('type:', type(model_outputs))
+                    print('model_outputs:',model_outputs)
 
-                    if utils.ModelUtils.is_valid_model(
-                        inputs_backends=model_outputs, backends_nums=len(backends)
-                    ):
+                    if utils.ModelUtils.is_valid_model(inputs_backends=model_outputs, backends_nums=len(backends)):
 
-                        delta = (
-                            accumulative_inconsistency - last_inconsistency
-                        )  # 也就是ACC(m)-ACC(s)
+                        delta = accumulative_inconsistency - last_inconsistency  # 也就是ACC(m)-ACC(s)
                         # 下面两个if好像没什么用，因为mutator字典里只有MCMC，mutant字典里只有ROULETTE
 
-                        if mutator_strategy == "MCMC":
-                            mutator.delta_bigger_than_zero = (
-                                mutator.delta_bigger_than_zero + 1
-                                if delta > 0
-                                else mutator.delta_bigger_than_zero
-                            )
+                        if mutator_strategy == 'MCMC':
+                            mutator.delta_bigger_than_zero = mutator.delta_bigger_than_zero + 1 \
+                                if delta > 0 else mutator.delta_bigger_than_zero
 
-                        if mutant_strategy == "ROULETTE" and delta > 0:
+                        if mutant_strategy == 'ROULETTE' and delta > 0:
                             # when size >= capacity:
                             # random_mutant & Roulette would drop one and add new one
                             if mutant_selector.is_full():
                                 mutant_selector.pop_one_mutant()
-                            mutant_selector.add_mutant(
-                                new_seed_name
-                            )  # 如果放大了不一致程度，即ACC(m)>=ACC(s),就加入到种子模型集合里
+                            mutant_selector.add_mutant(new_seed_name)  # 如果放大了不一致程度，即ACC(m)>=ACC(s),就加入到种子模型集合里
                             last_inconsistency = accumulative_inconsistency  # 29行
 
-                        mutate_logger.info(
-                            "SUCCESS:{} pass testing!".format(new_seed_name)
-                        )
+                        mutate_logger.info("SUCCESS:{} pass testing!".format(new_seed_name))
                         mutant_counter += 1
                     else:
                         mutate_op_invalid_history[selected_op] += 1
@@ -305,11 +251,7 @@ def _generate_and_predict(
                 else:
                     mutate_logger.error("Crashed or NaN model Found!")
             else:
-                mutate_logger.error(
-                    "Exception raised when mutate {} with {}".format(
-                        picked_seed, selected_op
-                    )
-                )
+                mutate_logger.error("Exception raised when mutate {} with {}".format(picked_seed, selected_op))
 
             mutate_logger.info("Mutated op used history:")
             mutate_logger.info(mutate_op_history)
@@ -317,7 +259,7 @@ def _generate_and_predict(
             mutate_logger.info("Invalid mutant generated history:")
             mutate_logger.info(mutate_op_invalid_history)
 
-        run_stat["cur_counters"] = mutant_counter
+        run_stat['cur_counters'] = mutant_counter
 
     save_mutate_history(mutator_selector, mutate_op_invalid_history, mutant_history)
 
@@ -359,16 +301,12 @@ def generate_metrics_result(res_dict, predict_output, model_idntfr):  # 计算AC
         for metrics_name, metrics_result_dict in res_dict.items():
             metrics_func = utils.MetricsUtils.get_metrics_by_name(metrics_name)  # 计算
             # metrics_results in list type
-            metrics_results = metrics_func(
-                prediction1, prediction2, y_test[: flags.test_size]
-            )
+            metrics_results = metrics_func(prediction1, prediction2, y_test[:flags.test_size])
             # 一共test_size个数据集，所以metrics_result是长度为test_size的预测结果列表
             # ACC -> float: The sum of all inputs under all backends
             accumulative_incons += sum(metrics_results)  # ACC=∑
             for input_idx, delta in enumerate(metrics_results):
-                delta_key = "{}_{}_{}_input{}".format(
-                    model_idntfr, bk_name1, bk_name2, input_idx
-                )
+                delta_key = "{}_{}_{}_input{}".format(model_idntfr, bk_name1, bk_name2, input_idx)
                 metrics_result_dict[delta_key] = delta
 
     mutate_logger.info(f"Accumulative Inconsistency: {accumulative_incons}")
@@ -387,9 +325,7 @@ def generate_theta(predict_output, backends):
     theta_res = {bk: 0 for bk in backends}
     for pair in predict_output.items():
         bk_name, prediction = pair
-        theta_res[bk_name] = utils.MetricsUtils.get_theta_mean(
-            prediction, y_test[: flags.test_size]
-        )
+        theta_res[bk_name] = utils.MetricsUtils.get_theta_mean(prediction, y_test[:flags.test_size])
     return theta_res
 
 
@@ -404,10 +340,8 @@ def get_model_prediction(res_dict, model_path, model_name, exp, test_size, backe
     """
 
     root_dir = model_path.split("origin_model")[0]
-
-    npy_path = (
-        root_dir + "res.npy"
-    )  # 保存模型预测结果的路径,patch_prediction_extractor.py中的44行改成一样的路径
+    
+    npy_path = root_dir + 'res.npy'  # 保存模型预测结果的路径,patch_prediction_extractor.py中的44行改成一样的路径
 
     predict_output = {b: [] for b in backends}
     model_idntfr = model_name[:-3]
@@ -416,18 +350,14 @@ def get_model_prediction(res_dict, model_path, model_name, exp, test_size, backe
         python_bin = f"{python_prefix}/{bk}/bin/python"
         predict_st = datetime.datetime.now()
         # 使用不同的库进行预测
-        pre_status_bk = os.system(
-            f"{python_bin} -u -m patch_prediction_extractor --backend {bk} "
-            f"--exp {exp} --test_size {test_size} --model {model_path} "
-            f"--config_name {flags.config_name}"
-        )
+        pre_status_bk = os.system(f"{python_bin} -u -m patch_prediction_extractor --backend {bk} "
+                                  f"--exp {exp} --test_size {test_size} --model {model_path} "
+                                  f"--config_name {flags.config_name}")
 
         predict_et = datetime.datetime.now()
         predict_td = predict_et - predict_st
         h, m, s = utils.ToolUtils.get_HH_mm_ss(predict_td)
-        mutate_logger.info(
-            "Prediction Time Used on {} : {}h, {}m, {}s".format(bk, h, m, s)
-        )
+        mutate_logger.info("Prediction Time Used on {} : {}h, {}m, {}s".format(bk, h, m, s))
 
         # If no exception is thrown,save prediction result
         if pre_status_bk == 0:  # 预测执行成功，保存结果
@@ -438,21 +368,16 @@ def get_model_prediction(res_dict, model_path, model_name, exp, test_size, backe
         # record the crashed backend
         else:
             all_backends_predict_status = False
-            mutate_logger.error(
-                "{} crash on backend {} when predicting ".format(model_name, bk)
-            )
+            mutate_logger.error("{} crash on backend {} when predicting ".format(model_name, bk))
 
     status = False
     accumulative_incons = None
 
     # run ok on all platforms
-    if (
-        all_backends_predict_status
-    ):  # 所有的库都执行成功且保存了结果，判断结果中是否有错误
+    if all_backends_predict_status:  # 所有的库都执行成功且保存了结果，判断结果中是否有错误
         predictions = list(predict_output.values())
-        res_dict, accumulative_incons = generate_metrics_result(
-            res_dict=res_dict, predict_output=predict_output, model_idntfr=model_idntfr
-        )
+        res_dict, accumulative_incons = generate_metrics_result(res_dict=res_dict, predict_output=predict_output,
+                                                                model_idntfr=model_idntfr)
         # 计算ACC(用于衡量预测结果的不一致程度)
         # gini_res = generate_gini_result(predict_output=predict_output, backends=backends)
         # theta = generate_theta(predict_output=predict_output, backends=backends)
@@ -477,21 +402,16 @@ def get_model_prediction(res_dict, model_path, model_name, exp, test_size, backe
 
             # has NaN on all backends --> not a NaN bug
             else:
-                nan_model_path = os.path.join(
-                    nan_dir, f"{model_idntfr}_NaN_on_all_backends.h5"
-                )
-                mutate_logger.error(
-                    "Error: Found one NaN Model on all libraries. move NAN model"
-                )
+                nan_model_path = os.path.join(nan_dir, f"{model_idntfr}_NaN_on_all_backends.h5")
+                mutate_logger.error("Error: Found one NaN Model on all libraries. move NAN model")
             shutil.move(model_path, nan_model_path)
 
         else:  # No NaN or INF on any backend
-            print(model_path)
+            # print(model_path)
             for bk in backends:
                 python_bin = f"{python_prefix}/{bk}/bin/python"
-                os.system(
-                    f"{python_bin} -u -m model_to_txt --backend {bk} --model_path {model_path} --root_dir {root_dir}"
-                )
+                os.system(f"{python_bin} -u -m model_to_txt --backend {bk} --model_path {model_path} --root_dir {root_dir}")
+            os.system(f"{python_bin} -u -m draw_result --backend {bk} --model_path {model_path}")
             # if 'svhn' in model_name or 'fashion2' in model_name:
             #     file_path = os.path.join(folder_path, model_path.split("\\")[-1][:-5] + '.json')
             # else:
@@ -505,9 +425,7 @@ def get_model_prediction(res_dict, model_path, model_name, exp, test_size, backe
             #     tar_set = set(sub_info['layer_type'])
 
             mutate_logger.info("Saving prediction")
-            with open(
-                "{}/prediction_{}.pkl".format(inner_output_dir, model_idntfr), "wb+"
-            ) as f:
+            with open("{}/prediction_{}.pkl".format(inner_output_dir, model_idntfr), "wb+") as f:
                 pickle.dump(predict_output, file=f)
             status = True
 
@@ -529,44 +447,15 @@ if __name__ == "__main__":
     It could make mutate_lemon.py run independently without relying on mutation_executor.py
     """
     parse = argparse.ArgumentParser()
-    parse.add_argument(
-        "--is_mutate",
-        type=ast.literal_eval,
-        default=False,
-        help="parameter to determine mutation option",
-    )
-    parse.add_argument(
-        "--mutate_op",
-        type=str,
-        nargs="+",
-        choices=[
-            "WS",
-            "GF",
-            "NEB",
-            "NAI",
-            "NS",
-            "ARem",
-            "ARep",
-            "LA",
-            "LC",
-            "LR",
-            "LS",
-            "MLA",
-        ],
-        help="parameter to determine mutation option",
-    )
-    parse.add_argument(
-        "--model", type=str, help="relative path of model file(from root dir)"
-    )
-    parse.add_argument(
-        "--output_dir", type=str, help="relative path of output dir(from root dir)"
-    )
-    parse.add_argument("--backends", type=str, nargs="+", help="list of backends")
-    parse.add_argument(
-        "--mutate_num",
-        type=int,
-        help="number of variant models generated by each mutation operator",
-    )
+    parse.add_argument("--is_mutate", type=ast.literal_eval, default=False,
+                       help="parameter to determine mutation option")
+    parse.add_argument("--mutate_op", type=str, nargs='+',
+                       choices=['WS', 'GF', 'NEB', 'NAI', 'NS', 'ARem', 'ARep', 'LA', 'LC', 'LR', 'LS', 'MLA']
+                       , help="parameter to determine mutation option")
+    parse.add_argument("--model", type=str, help="relative path of model file(from root dir)")
+    parse.add_argument("--output_dir", type=str, help="relative path of output dir(from root dir)")
+    parse.add_argument("--backends", type=str, nargs='+', help="list of backends")
+    parse.add_argument("--mutate_num", type=int, help="number of variant models generated by each mutation operator")
     parse.add_argument("--mutate_ratio", type=float, help="ratio of mutation")
     parse.add_argument("--exp", type=str, help="experiments identifiers")
     parse.add_argument("--test_size", type=int, help="amount of testing image")
@@ -576,13 +465,13 @@ if __name__ == "__main__":
     lemon_cfg = configparser.ConfigParser()
     # lemon_cfg.read(f".\config\{flags.config_name}")
     cfg_path = os.path.join(os.path.dirname(os.getcwd()), "config", flags.config_name)
-    lemon_cfg.read(cfg_path)
+    lemon_cfg.read(cfg_path) 
     # lemon_cfg.read(f"config/demo.conf")
-    time_limit = lemon_cfg["parameters"].getint("time_limit")
-    mutator_strategy = lemon_cfg["parameters"].get("mutator_strategy").upper()
-    mutant_strategy = lemon_cfg["parameters"].get("mutant_strategy").upper()
-    stop_mode = lemon_cfg["parameters"].get("stop_mode").upper()
-    alpha = lemon_cfg["parameters"].getfloat("alpha")
+    time_limit = lemon_cfg['parameters'].getint("time_limit")
+    mutator_strategy = lemon_cfg['parameters'].get("mutator_strategy").upper()
+    mutant_strategy = lemon_cfg['parameters'].get("mutant_strategy").upper()
+    stop_mode = lemon_cfg['parameters'].get("stop_mode").upper()
+    alpha = lemon_cfg['parameters'].getfloat("alpha")
 
     mutate_logger = Logger()
     #    pool = redis.ConnectionPool(host=lemon_cfg['redis']['host'], port=lemon_cfg['redis']['port'],
@@ -602,28 +491,17 @@ if __name__ == "__main__":
     metrics_result_dir = os.path.join(experiment_dir, "metrics_result")
 
     x, y = utils.DataUtils.get_data_by_exp(flags.exp)  # 从conf文件中读取数据并转换形式
-    x_test, y_test = x[: flags.test_size], y[: flags.test_size]
-    pool_size = lemon_cfg["parameters"].getint("pool_size")
-    python_prefix = lemon_cfg["parameters"]["python_prefix"].rstrip("\\")
+    x_test, y_test = x[:flags.test_size], y[:flags.test_size]
+    pool_size = lemon_cfg['parameters'].getint('pool_size')
+    python_prefix = lemon_cfg['parameters']['python_prefix'].rstrip("\\")
     try:  # 执行算法
-        metrics_list = lemon_cfg["parameters"]["metrics"].split(" ")  # D_MAD
+        metrics_list = lemon_cfg['parameters']['metrics'].split(" ")  # D_MAD
         lemon_results = {k: dict() for k in metrics_list}
-        lemon_results = _generate_and_predict(
-            lemon_results,
-            flags.model,
-            flags.mutate_num,
-            flags.mutate_op,
-            flags.test_size,
-            flags.exp,
-            flags.backends,
-        )
-        with open(
-            "{}/{}_lemon_results.pkl".format(experiment_dir, flags.exp), "wb+"
-        ) as f:
+        lemon_results = _generate_and_predict(lemon_results, flags.model, flags.mutate_num, flags.mutate_op,
+                                              flags.test_size, flags.exp, flags.backends)
+        with open("{}/{}_lemon_results.pkl".format(experiment_dir, flags.exp), "wb+") as f:
             pickle.dump(lemon_results, file=f)
-        utils.MetricsUtils.generate_result_by_metrics(
-            metrics_list, lemon_results, metrics_result_dir, flags.exp
-        )
+        utils.MetricsUtils.generate_result_by_metrics(metrics_list, lemon_results, metrics_result_dir, flags.exp)
 
     except Exception as e:
         mutate_logger.exception(sys.exc_info())
@@ -635,6 +513,6 @@ if __name__ == "__main__":
     endtime = datetime.datetime.now()
     time_delta = endtime - starttime
     h, m, s = utils.ToolUtils.get_HH_mm_ss(time_delta)
-    mutate_logger.info(
-        "Mutation process is done: Time used: {} hour,{} min,{} sec".format(h, m, s)
-    )
+    mutate_logger.info("Mutation process is done: Time used: {} hour,{} min,{} sec".format(h, m, s))
+
+

server/lemon_master/run/mutation_executor.py

@@ -150,6 +150,8 @@ if __name__ == "__main__":
 
     lemon_cfg = configparser.ConfigParser()
     cfg_path = os.path.join(flags_input.root_dir, "config", flags_input.config_name)
+    print(cfg_path)
+    
     lemon_cfg.read(cfg_path)
     parameters = lemon_cfg["parameters"]
     flags = argparse.Namespace(

server/output/task_1_fashion2_1/fashion2/metrics_result/fashion2_D_MAD_result.csv

@@ -0,0 +1,21 @@
+Mutation-Backend-Pair,Inconsistency Score
+fashion2_origin0.h_tensorflow_mxnet_input0,2.789349491649773e-05
+fashion2_origin0.h_tensorflow_mxnet_input1,2.0714646780106705e-06
+fashion2_origin0.h_tensorflow_mxnet_input2,1.3693011169380043e-06
+fashion2_origin0.h_tensorflow_mxnet_input3,3.507439600980433e-07
+fashion2_origin0.h_tensorflow_mxnet_input4,7.013139224909537e-07
+fashion2_origin0.h_tensorflow_mxnet_input5,0.0011009954614564776
+fashion2_origin0.h_tensorflow_mxnet_input6,1.8905183196693542e-06
+fashion2_origin0.h_tensorflow_mxnet_input7,8.76466219779104e-06
+fashion2_origin0.h_tensorflow_mxnet_input8,3.5046812172367936e-06
+fashion2_origin0.h_tensorflow_mxnet_input9,6.812222181906691e-07
+fashion2_origin0-WS1.h_tensorflow_mxnet_input0,1.1224511808904936e-06
+fashion2_origin0-WS1.h_tensorflow_mxnet_input1,3.5494170447236684e-07
+fashion2_origin0-WS1.h_tensorflow_mxnet_input2,1.7220250470018073e-07
+fashion2_origin0-WS1.h_tensorflow_mxnet_input3,3.008639396284707e-05
+fashion2_origin0-WS1.h_tensorflow_mxnet_input4,1.443948463020206e-06
+fashion2_origin0-WS1.h_tensorflow_mxnet_input5,1.452377631494528e-07
+fashion2_origin0-WS1.h_tensorflow_mxnet_input6,3.9023408504590407e-08
+fashion2_origin0-WS1.h_tensorflow_mxnet_input7,0.0
+fashion2_origin0-WS1.h_tensorflow_mxnet_input8,2.476682539054309e-06
+fashion2_origin0-WS1.h_tensorflow_mxnet_input9,7.502628704969538e-07

server/output/task_1_fashion2_1/fashion2/mutant_history.txt

@@ -0,0 +1 @@
+fashion2_origin0-WS1.hdf5

server/output/task_1_fashion2_1/fashion2/mutator_history.csv

@@ -0,0 +1,13 @@
+Name,Success,Invalid,Total
+WS,0,0,1
+GF,0,0,0
+NEB,0,0,0
+NAI,0,0,0
+NS,0,0,0
+ARem,0,0,0
+ARep,0,0,0
+LA,0,0,0
+LC,0,0,0
+LR,0,0,0
+LS,0,0,0
+MLA,0,0,0

server/output/task_1_fashion2_1/fashion2/mxnet.json

@@ -0,0 +1,20 @@
+[
+    {
+        "model": "fashion2",
+        "method": "origin0",
+        "result": {
+            "Losses": 0.2744102403521538,
+            "Accuracy": 0.9029728807508945,
+            "MemoryInfoList": 0.8383903503417969
+        }
+    },
+    {
+        "model": "fashion2",
+        "method": "origin0-WS1",
+        "result": {
+            "Losses": 0.34761693328619003,
+            "Accuracy": 0.877291165292263,
+            "MemoryInfoList": 0.8219783306121826
+        }
+    }
+]

server/output/task_1_fashion2_1/fashion2/mxnet_train.json

@@ -0,0 +1,130 @@
+[
+    {
+        "Iterations": 1,
+        "result": {
+            "Losses": 0.4872690439224243,
+            "Accuracy": 0.8408203125,
+            "MemoryInfoList": 0.4745445251464844
+        }
+    },
+    {
+        "Iterations": 2,
+        "result": {
+            "Losses": 0.3547583818435669,
+            "Accuracy": 0.875,
+            "MemoryInfoList": 0.5845527648925781
+        }
+    },
+    {
+        "Iterations": 3,
+        "result": {
+            "Losses": 0.3730500340461731,
+            "Accuracy": 0.869140625,
+            "MemoryInfoList": 0.6955375671386719
+        }
+    },
+    {
+        "Iterations": 4,
+        "result": {
+            "Losses": 0.38311153650283813,
+            "Accuracy": 0.8623046875,
+            "MemoryInfoList": 0.7729949951171875
+        }
+    },
+    {
+        "Iterations": 5,
+        "result": {
+            "Losses": 0.38041412830352783,
+            "Accuracy": 0.87109375,
+            "MemoryInfoList": 0.7646446228027344
+        }
+    },
+    {
+        "Iterations": 6,
+        "result": {
+            "Losses": 0.356524258852005,
+            "Accuracy": 0.8662109375,
+            "MemoryInfoList": 0.8172721862792969
+        }
+    },
+    {
+        "Iterations": 7,
+        "result": {
+            "Losses": 0.3416406512260437,
+            "Accuracy": 0.876953125,
+            "MemoryInfoList": 0.8653450012207031
+        }
+    },
+    {
+        "Iterations": 8,
+        "result": {
+            "Losses": 0.3707755208015442,
+            "Accuracy": 0.8725961446762085,
+            "MemoryInfoList": 0.8653450012207031
+        }
+    },
+    {
+        "Iterations": 9,
+        "result": {
+            "Losses": 0.3097502589225769,
+            "Accuracy": 0.892578125,
+            "MemoryInfoList": 0.8866806030273438
+        }
+    },
+    {
+        "Iterations": 10,
+        "result": {
+            "Losses": 0.2972621023654938,
+            "Accuracy": 0.89453125,
+            "MemoryInfoList": 0.9267616271972656
+        }
+    },
+    {
+        "Iterations": 11,
+        "result": {
+            "Losses": 0.31856751441955566,
+            "Accuracy": 0.8896484375,
+            "MemoryInfoList": 0.8817214965820312
+        }
+    },
+    {
+        "Iterations": 12,
+        "result": {
+            "Losses": 0.32103055715560913,
+            "Accuracy": 0.884765625,
+            "MemoryInfoList": 0.9151649475097656
+        }
+    },
+    {
+        "Iterations": 13,
+        "result": {
+            "Losses": 0.3285468816757202,
+            "Accuracy": 0.8818359375,
+            "MemoryInfoList": 0.942626953125
+        }
+    },
+    {
+        "Iterations": 14,
+        "result": {
+            "Losses": 0.3207525610923767,
+            "Accuracy": 0.8818359375,
+            "MemoryInfoList": 0.9426307678222656
+        }
+    },
+    {
+        "Iterations": 15,
+        "result": {
+            "Losses": 0.3216632008552551,
+            "Accuracy": 0.88671875,
+            "MemoryInfoList": 0.8962364196777344
+        }
+    },
+    {
+        "Iterations": 16,
+        "result": {
+            "Losses": 0.29675430059432983,
+            "Accuracy": 0.890625,
+            "MemoryInfoList": 0.9195938110351562
+        }
+    }
+]

server/output/task_1_fashion2_1/fashion2/tensorflow.json

@@ -0,0 +1,20 @@
+[
+    {
+        "model": "fashion2",
+        "method": "origin0",
+        "result": {
+            "Losses": 0.2782079726457596,
+            "Accuracy": 0.90216064453125,
+            "MemoryInfoList": 0.796128511428833
+        }
+    },
+    {
+        "model": "fashion2",
+        "method": "origin0-WS1",
+        "result": {
+            "Losses": 0.35277533531188965,
+            "Accuracy": 0.874417819082737,
+            "MemoryInfoList": 0.7188582420349121
+        }
+    }
+]

server/output/task_1_fashion2_1/fashion2/tensorflow_train.json

@@ -0,0 +1,130 @@
+[
+    {
+        "Iterations": 1,
+        "result": {
+            "Losses": 0.46252918243408203,
+            "Accuracy": 0.849609375,
+            "MemoryInfoList": 0.39239501953125
+        }
+    },
+    {
+        "Iterations": 2,
+        "result": {
+            "Losses": 0.4296347498893738,
+            "Accuracy": 0.8447265625,
+            "MemoryInfoList": 0.499359130859375
+        }
+    },
+    {
+        "Iterations": 3,
+        "result": {
+            "Losses": 0.420149028301239,
+            "Accuracy": 0.865234375,
+            "MemoryInfoList": 0.5355758666992188
+        }
+    },
+    {
+        "Iterations": 4,
+        "result": {
+            "Losses": 0.3606413006782532,
+            "Accuracy": 0.8671875,
+            "MemoryInfoList": 0.6154556274414062
+        }
+    },
+    {
+        "Iterations": 5,
+        "result": {
+            "Losses": 0.3635426163673401,
+            "Accuracy": 0.8603515625,
+            "MemoryInfoList": 0.6601371765136719
+        }
+    },
+    {
+        "Iterations": 6,
+        "result": {
+            "Losses": 0.3755783438682556,
+            "Accuracy": 0.865234375,
+            "MemoryInfoList": 0.6851425170898438
+        }
+    },
+    {
+        "Iterations": 7,
+        "result": {
+            "Losses": 0.3496626019477844,
+            "Accuracy": 0.8759765625,
+            "MemoryInfoList": 0.7299652099609375
+        }
+    },
+    {
+        "Iterations": 8,
+        "result": {
+            "Losses": 0.3218604624271393,
+            "Accuracy": 0.8786057829856873,
+            "MemoryInfoList": 0.7072334289550781
+        }
+    },
+    {
+        "Iterations": 9,
+        "result": {
+            "Losses": 0.33442234992980957,
+            "Accuracy": 0.8818359375,
+            "MemoryInfoList": 0.7475395202636719
+        }
+    },
+    {
+        "Iterations": 10,
+        "result": {
+            "Losses": 0.33098429441452026,
+            "Accuracy": 0.8837890625,
+            "MemoryInfoList": 0.7941474914550781
+        }
+    },
+    {
+        "Iterations": 11,
+        "result": {
+            "Losses": 0.34452107548713684,
+            "Accuracy": 0.875,
+            "MemoryInfoList": 0.8432769775390625
+        }
+    },
+    {
+        "Iterations": 12,
+        "result": {
+            "Losses": 0.3103902041912079,
+            "Accuracy": 0.888671875,
+            "MemoryInfoList": 0.8433609008789062
+        }
+    },
+    {
+        "Iterations": 13,
+        "result": {
+            "Losses": 0.30054712295532227,
+            "Accuracy": 0.89453125,
+            "MemoryInfoList": 0.8606986999511719
+        }
+    },
+    {
+        "Iterations": 14,
+        "result": {
+            "Losses": 0.2906036078929901,
+            "Accuracy": 0.892578125,
+            "MemoryInfoList": 0.8672676086425781
+        }
+    },
+    {
+        "Iterations": 15,
+        "result": {
+            "Losses": 0.32827141880989075,
+            "Accuracy": 0.8935546875,
+            "MemoryInfoList": 0.8688278198242188
+        }
+    },
+    {
+        "Iterations": 16,
+        "result": {
+            "Losses": 0.3210670053958893,
+            "Accuracy": 0.8737980723381042,
+            "MemoryInfoList": 0.851348876953125
+        }
+    }
+]

server/output/task_2024-07-19T14:45:07_1/fashion2/mxnet.json

@@ -1,20 +0,0 @@
-[
-    {
-        "model": "fashion2",
-        "method": "origin0",
-        "result": {
-            "Losses": 0.2729402296245098,
-            "Accuracy": 0.903226412832737,
-            "MemoryInfoList": 0.9190187454223633
-        }
-    },
-    {
-        "model": "fashion2",
-        "method": "origin0-WS1",
-        "result": {
-            "Losses": 0.4089081399142742,
-            "Accuracy": 0.8602341897785664,
-            "MemoryInfoList": 0.811927080154419
-        }
-    }
-]

server/output/task_2024-07-19T14:45:07_1/fashion2/tensorflow.json

@@ -1,20 +0,0 @@
-[
-    {
-        "model": "fashion2",
-        "method": "origin0",
-        "result": {
-            "Losses": 0.27484310790896416,
-            "Accuracy": 0.904052734375,
-            "MemoryInfoList": 0.7509610652923584
-        }
-    },
-    {
-        "model": "fashion2",
-        "method": "origin0-WS1",
-        "result": {
-            "Losses": 0.4064286407083273,
-            "Accuracy": 0.8588679395616055,
-            "MemoryInfoList": 0.70198655128479
-        }
-    }
-]

server/output/task_2_fashion2_1/fashion2/metrics_result/fashion2_D_MAD_result.csv

@@ -0,0 +1,21 @@
+Mutation-Backend-Pair,Inconsistency Score
+fashion2_origin0.h_tensorflow_mxnet_input0,2.789349491649773e-05
+fashion2_origin0.h_tensorflow_mxnet_input1,2.0714646780106705e-06
+fashion2_origin0.h_tensorflow_mxnet_input2,1.3693011169380043e-06
+fashion2_origin0.h_tensorflow_mxnet_input3,3.507439600980433e-07
+fashion2_origin0.h_tensorflow_mxnet_input4,7.013139224909537e-07
+fashion2_origin0.h_tensorflow_mxnet_input5,0.0011009954614564776
+fashion2_origin0.h_tensorflow_mxnet_input6,1.8905183196693542e-06
+fashion2_origin0.h_tensorflow_mxnet_input7,8.76466219779104e-06
+fashion2_origin0.h_tensorflow_mxnet_input8,3.5046812172367936e-06
+fashion2_origin0.h_tensorflow_mxnet_input9,6.812222181906691e-07
+fashion2_origin0-LR1.h_tensorflow_mxnet_input0,3.4806015491994913e-07
+fashion2_origin0-LR1.h_tensorflow_mxnet_input1,2.343457936149207e-06
+fashion2_origin0-LR1.h_tensorflow_mxnet_input2,7.252479576891346e-07
+fashion2_origin0-LR1.h_tensorflow_mxnet_input3,0.09363756328821182
+fashion2_origin0-LR1.h_tensorflow_mxnet_input4,1.1790518783527659e-07
+fashion2_origin0-LR1.h_tensorflow_mxnet_input5,0.017461484298110008
+fashion2_origin0-LR1.h_tensorflow_mxnet_input6,6.805962129874388e-06
+fashion2_origin0-LR1.h_tensorflow_mxnet_input7,5.853631137142656e-06
+fashion2_origin0-LR1.h_tensorflow_mxnet_input8,8.788790000835434e-05
+fashion2_origin0-LR1.h_tensorflow_mxnet_input9,1.294190496992087e-07

server/output/task_2_fashion2_1/fashion2/mutant_history.txt

@@ -0,0 +1 @@
+fashion2_origin0-LR1.hdf5

server/output/task_2_fashion2_1/fashion2/mutator_history.csv

@@ -0,0 +1,13 @@
+Name,Success,Invalid,Total
+WS,0,0,0
+GF,0,0,0
+NEB,0,0,0
+NAI,0,0,0
+NS,0,0,0
+ARem,0,0,0
+ARep,0,0,0
+LA,0,0,0
+LC,0,0,0
+LR,1,0,1
+LS,0,0,0
+MLA,0,0,0

server/output/task_2_fashion2_1/fashion2/mxnet.json

@@ -0,0 +1,20 @@
+[
+    {
+        "model": "fashion2",
+        "method": "origin0",
+        "result": {
+            "Losses": 0.27659945096820593,
+            "Accuracy": 0.9029071517288685,
+            "MemoryInfoList": 0.835566520690918
+        }
+    },
+    {
+        "model": "fashion2",
+        "method": "origin0-LR1",
+        "result": {
+            "Losses": 0.3440984021872282,
+            "Accuracy": 0.8804274350404739,
+            "MemoryInfoList": 0.7985403537750244
+        }
+    }
+]

server/output/task_2_fashion2_1/fashion2/mxnet_train.json

@@ -0,0 +1,130 @@
+[
+    {
+        "Iterations": 1,
+        "result": {
+            "Losses": 0.4482508897781372,
+            "Accuracy": 0.837890625,
+            "MemoryInfoList": 0.4694023132324219
+        }
+    },
+    {
+        "Iterations": 2,
+        "result": {
+            "Losses": 0.38157153129577637,
+            "Accuracy": 0.8740234375,
+            "MemoryInfoList": 0.5575714111328125
+        }
+    },
+    {
+        "Iterations": 3,
+        "result": {
+            "Losses": 0.3769081234931946,
+            "Accuracy": 0.8662109375,
+            "MemoryInfoList": 0.6399078369140625
+        }
+    },
+    {
+        "Iterations": 4,
+        "result": {
+            "Losses": 0.4163675010204315,
+            "Accuracy": 0.8671875,
+            "MemoryInfoList": 0.689849853515625
+        }
+    },
+    {
+        "Iterations": 5,
+        "result": {
+            "Losses": 0.39336487650871277,
+            "Accuracy": 0.8779296875,
+            "MemoryInfoList": 0.71978759765625
+        }
+    },
+    {
+        "Iterations": 6,
+        "result": {
+            "Losses": 0.3816114068031311,
+            "Accuracy": 0.8681640625,
+            "MemoryInfoList": 0.7420616149902344
+        }
+    },
+    {
+        "Iterations": 7,
+        "result": {
+            "Losses": 0.31748074293136597,
+            "Accuracy": 0.8857421875,
+            "MemoryInfoList": 0.7750282287597656
+        }
+    },
+    {
+        "Iterations": 8,
+        "result": {
+            "Losses": 0.3084408938884735,
+            "Accuracy": 0.8810096383094788,
+            "MemoryInfoList": 0.8363761901855469
+        }
+    },
+    {
+        "Iterations": 9,
+        "result": {
+            "Losses": 0.34968775510787964,
+            "Accuracy": 0.8818359375,
+            "MemoryInfoList": 0.9132308959960938
+        }
+    },
+    {
+        "Iterations": 10,
+        "result": {
+            "Losses": 0.31936150789260864,
+            "Accuracy": 0.8837890625,
+            "MemoryInfoList": 0.9047775268554688
+        }
+    },
+    {
+        "Iterations": 11,
+        "result": {
+            "Losses": 0.29849982261657715,
+            "Accuracy": 0.9013671875,
+            "MemoryInfoList": 0.9065437316894531
+        }
+    },
+    {
+        "Iterations": 12,
+        "result": {
+            "Losses": 0.3444221615791321,
+            "Accuracy": 0.87890625,
+            "MemoryInfoList": 0.9080543518066406
+        }
+    },
+    {
+        "Iterations": 13,
+        "result": {
+            "Losses": 0.2600328326225281,
+            "Accuracy": 0.9072265625,
+            "MemoryInfoList": 0.9174995422363281
+        }
+    },
+    {
+        "Iterations": 14,
+        "result": {
+            "Losses": 0.3060879707336426,
+            "Accuracy": 0.8837890625,
+            "MemoryInfoList": 0.9175033569335938
+        }
+    },
+    {
+        "Iterations": 15,
+        "result": {
+            "Losses": 0.32010728120803833,
+            "Accuracy": 0.88671875,
+            "MemoryInfoList": 0.9293327331542969
+        }
+    },
+    {
+        "Iterations": 16,
+        "result": {
+            "Losses": 0.28337913751602173,
+            "Accuracy": 0.9050480723381042,
+            "MemoryInfoList": 0.9497184753417969
+        }
+    }
+]

server/output/task_2_fashion2_1/fashion2/tensorflow.json

@@ -0,0 +1,20 @@
+[
+    {
+        "model": "fashion2",
+        "method": "origin0",
+        "result": {
+            "Losses": 0.28040206525474787,
+            "Accuracy": 0.9014986492693424,
+            "MemoryInfoList": 0.7640633583068848
+        }
+    },
+    {
+        "model": "fashion2",
+        "method": "origin0-LR1",
+        "result": {
+            "Losses": 0.3380755726248026,
+            "Accuracy": 0.8809908367693424,
+            "MemoryInfoList": 0.8035497665405273
+        }
+    }
+]

server/output/task_2_fashion2_1/fashion2/tensorflow_train.json

@@ -0,0 +1,130 @@
+[
+    {
+        "Iterations": 1,
+        "result": {
+            "Losses": 0.48184114694595337,
+            "Accuracy": 0.8349609375,
+            "MemoryInfoList": 0.3923149108886719
+        }
+    },
+    {
+        "Iterations": 2,
+        "result": {
+            "Losses": 0.4115346074104309,
+            "Accuracy": 0.8720703125,
+            "MemoryInfoList": 0.4987945556640625
+        }
+    },
+    {
+        "Iterations": 3,
+        "result": {
+            "Losses": 0.3135305643081665,
+            "Accuracy": 0.8916015625,
+            "MemoryInfoList": 0.5897254943847656
+        }
+    },
+    {
+        "Iterations": 4,
+        "result": {
+            "Losses": 0.3325827419757843,
+            "Accuracy": 0.8740234375,
+            "MemoryInfoList": 0.6878318786621094
+        }
+    },
+    {
+        "Iterations": 5,
+        "result": {
+            "Losses": 0.4124763011932373,
+            "Accuracy": 0.861328125,
+            "MemoryInfoList": 0.7660140991210938
+        }
+    },
+    {
+        "Iterations": 6,
+        "result": {
+            "Losses": 0.361680805683136,
+            "Accuracy": 0.8837890625,
+            "MemoryInfoList": 0.7935562133789062
+        }
+    },
+    {
+        "Iterations": 7,
+        "result": {
+            "Losses": 0.34919673204421997,
+            "Accuracy": 0.876953125,
+            "MemoryInfoList": 0.6982421875
+        }
+    },
+    {
+        "Iterations": 8,
+        "result": {
+            "Losses": 0.33619052171707153,
+            "Accuracy": 0.8762019276618958,
+            "MemoryInfoList": 0.7669258117675781
+        }
+    },
+    {
+        "Iterations": 9,
+        "result": {
+            "Losses": 0.3225761651992798,
+            "Accuracy": 0.8935546875,
+            "MemoryInfoList": 0.8512115478515625
+        }
+    },
+    {
+        "Iterations": 10,
+        "result": {
+            "Losses": 0.31542497873306274,
+            "Accuracy": 0.8837890625,
+            "MemoryInfoList": 0.889129638671875
+        }
+    },
+    {
+        "Iterations": 11,
+        "result": {
+            "Losses": 0.3095233738422394,
+            "Accuracy": 0.8916015625,
+            "MemoryInfoList": 0.9327316284179688
+        }
+    },
+    {
+        "Iterations": 12,
+        "result": {
+            "Losses": 0.30772215127944946,
+            "Accuracy": 0.8876953125,
+            "MemoryInfoList": 0.9641952514648438
+        }
+    },
+    {
+        "Iterations": 13,
+        "result": {
+            "Losses": 0.2790094316005707,
+            "Accuracy": 0.8974609375,
+            "MemoryInfoList": 0.9669837951660156
+        }
+    },
+    {
+        "Iterations": 14,
+        "result": {
+            "Losses": 0.34252986311912537,
+            "Accuracy": 0.8828125,
+            "MemoryInfoList": 1.0014228820800781
+        }
+    },
+    {
+        "Iterations": 15,
+        "result": {
+            "Losses": 0.2947428524494171,
+            "Accuracy": 0.876953125,
+            "MemoryInfoList": 1.0404624938964844
+        }
+    },
+    {
+        "Iterations": 16,
+        "result": {
+            "Losses": 0.23864692449569702,
+            "Accuracy": 0.911057710647583,
+            "MemoryInfoList": 1.0172538757324219
+        }
+    }
+]