crowdsource-backend/src/main/java/edu/nju/service/DataService.java

package edu.nju.service;

import com.alibaba.fastjson.JSON;
import edu.nju.dao.*;
import edu.nju.entities.*;
import edu.nju.util.TransUtil;
import org.json.JSONArray;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.*;
import java.lang.reflect.Field;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * @Author JiaWei Xu
 * @Date 2020-12-25 10:45
 * @Email xjwhhh233@outlook.com
 */
@Service
public class DataService {
    private static final int BUFFER_SIZE = 2 * 1024;

    private static final String imageUrlPrefix="https://mooctest-site.oss-cn-shanghai.aliyuncs.com/xinchuang/image/";
    @Autowired
    ExamDao examDao;

    @Autowired
    BugDao bugDao;

    @Autowired
    ReportDao reportDao;

    @Autowired
    TestCaseDao testCaseDao;

    @Autowired
    CaseToBugDao caseToBugDao;

    @Autowired
    BugScoreDao bugScoreDao;

    @Autowired
    BugMirrorDao bugMirrorDao;

    @Autowired
    BugHistoryDao bugHistoryDao;

    @Autowired
    BugDetailDao bugDetailDao;

    private String cpSerialNum = "cp_ent_dev";

    /**
     * 根据caseId获取对应bug信息
     *
     * @param caseId
     * @return
     */
    public List<BugDetail> getBugDetailByCaseId(String caseId) {
        List<BugDetail> bugDetailList = new ArrayList<>();
        Exam crowdCase = examDao.findById(caseId);
        if (crowdCase != null) {
            List<Report> reportList = reportDao.findByCaseId(caseId);
            for (Report report : reportList) {
                String reportId = report.getId();
                List<TestCase> testCaseList = testCaseDao.findByReport(reportId);
                for (TestCase testCase : testCaseList) {
                    String testCaseId = testCase.getId();
                    CaseToBug caseToBug = caseToBugDao.findById(testCaseId);
                    if (caseToBug != null) {
                        List<String> bugIdList = caseToBug.getBug_id();
                        for (String bugId : bugIdList) {
                            BugDetail bugDetail = new BugDetail();
                            bugDetail.setId(bugId);
                            //bug基本属性
                            Bug bug = bugDao.findByid(bugId);
                            if (bug != null) {
                                bugDetail.setBugCategory(bug.getBug_category());
                                bugDetail.setSeverity(TransUtil.severityTransFromInt(bug.getSeverity()));
                                bugDetail.setRecurrent(TransUtil.recurrentTransFromInt(bug.getRecurrent()));
                                bugDetail.setBugCreateTime(TransUtil.formatTimeMillis(bug.getCreate_time_millis()));
                                bugDetail.setBugPage(bug.getBug_page());
                                bugDetail.setTitle(bug.getTitle());
                                bugDetail.setBugDescription(bug.getDescription());
                                bugDetail.setImgUrl(bug.getImg_url());
                            }
                            //bugScore属性
                            BugScore bugScore = bugScoreDao.findById(bugId);
                            if (bugScore != null) {
                                bugDetail.setScore(bugScore.getGrade());
                            }
                            //bugMirror属性
                            BugMirror bugMirror = bugMirrorDao.findById(bugId);
                            if (bugMirror != null) {
                                Set<String> goodWorkerIdSet = new HashSet<>();
                                Set<String> badWorkerIdSet = new HashSet<>();
                                Set<String> goodReportIdSet = bugMirror.getGood();
                                Set<String> badReportIdSet = bugMirror.getBad();
                                int goodNum = 0;
                                int badNum = 0;
                                for (String goodReportId : goodReportIdSet) {
                                    Report goodReport = reportDao.findById(goodReportId);
                                    if (goodReport != null) {
                                        goodNum++;
                                        goodWorkerIdSet.add(goodReport.getWorker_id());
                                    }
                                }
                                for (String badReportId : badReportIdSet) {
                                    Report badReport = reportDao.findById(badReportId);
                                    if (badReport != null) {
                                        badNum++;
                                        badWorkerIdSet.add(badReport.getWorker_id());
                                    }
                                }
                                bugDetail.setGoodNum(goodNum);
                                bugDetail.setBadNum(badNum);
                                bugDetail.setGoodWorkerId(goodWorkerIdSet);
                                bugDetail.setBadWorkerId(badWorkerIdSet);
                            }
                            //bugHistory属性
                            BugHistory bugHistory = bugHistoryDao.findByid(bugId);
                            if (bugHistory != null) {
                                bugDetail.setParent(bugHistory.getParent());
                                bugDetail.setChildren(bugHistory.getChildren());
                                bugDetail.setRoot(bugHistory.getRoot());
                            }
                            //testCase属性
                            bugDetail.setTestCaseId(testCase.getId());
                            bugDetail.setTestCaseName(testCase.getName());
                            bugDetail.setTestCaseFront(testCase.getFront());
                            bugDetail.setTestCaseBehind(testCase.getBehind());
                            bugDetail.setTestCaseDescription(testCase.getDescription());
                            bugDetail.setTestCaseCreateTime(TransUtil.formatTimeMillis(testCase.getCreate_time_millis()));
                            //report属性
                            bugDetail.setReportId(report.getId());
                            bugDetail.setReportName(report.getName());
                            bugDetail.setScriptLocation(report.getScript_location());
                            bugDetail.setReportLocation(report.getReport_location());
                            bugDetail.setLogLocation(report.getLog_location());
                            bugDetail.setDeviceModel(report.getDevice_model());
                            bugDetail.setDeviceBrand(report.getDevice_brand());
                            bugDetail.setDeviceOs(report.getDevice_os());
                            //worker属性
                            bugDetail.setWorkerId(report.getWorker_id());
                            //众测任务属性
                            bugDetail.setCaseAppName(crowdCase.getName());
                            bugDetail.setCasePaperType(crowdCase.getPaper_type());
                            bugDetail.setCaseTestType(crowdCase.getTest_type());
                            bugDetail.setCaseDescription(crowdCase.getDescription());
                            bugDetail.setCaseRequireDoc("");
                            bugDetail.setCaseTakeId(report.getCase_take_id());
                            //cp序列号
                            bugDetail.setCpSerialNum(cpSerialNum);
                            bugDetailList.add(bugDetail);
                        }
                    }
                }
            }
        }
        bugDetailToFile(bugDetailList, caseId);
        return bugDetailList;
    }

    public List<BugDetail> saveBugDetailFromOss(String jsonFilePath, String originalCaseId,String cpSerialNum) {
        try {
            //从oss下载json文件
            URL url = new URL(jsonFilePath);
            URLConnection urlConnection = url.openConnection();
            HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection;
            httpURLConnection.setConnectTimeout(1000 * 5);
            httpURLConnection.setRequestProperty("Charset", "UTF-8");
            httpURLConnection.connect();
            BufferedInputStream bufferedInputStream = new BufferedInputStream(httpURLConnection.getInputStream());
            ByteArrayOutputStream buf = new ByteArrayOutputStream();
            int result = bufferedInputStream.read();
            while (result != -1) {
                buf.write((byte) result);
                result = bufferedInputStream.read();
            }
            String json = buf.toString();
            //转为bugDetail
            List<BugDetail> bugDetailList = JSON.parseArray(json, BugDetail.class);
            for (BugDetail bugDetail : bugDetailList) {
                bugDetail.setOriginalCaseId(originalCaseId);
                //修改图片文件路径为oss路径
                String imageUrl=bugDetail.getImgUrl();
                String[] imageUrlArray=imageUrl.split(",");
                StringBuilder stringBuilder=new StringBuilder();
                for(String imageUrlStr:imageUrlArray){
                    String[] filePath=imageUrlStr.split("/");
                    String fileName=filePath[filePath.length-1];
                    String newImageUrl=imageUrlPrefix+originalCaseId+"/"+cpSerialNum+"/"+fileName;
                    stringBuilder.append(newImageUrl).append(",");
                }
                bugDetail.setImgUrl(stringBuilder.toString());
                bugDetailDao.save(bugDetail);
            }
            return bugDetailList;
        } catch (IOException e) {
            e.printStackTrace();
        }
        return new ArrayList<>();
    }

    private void bugDetailToFile(List<BugDetail> bugDetailList, String caseId) {
        String[] titles = {"bug_id", "bug_category", "severity", "recurrent", "bug_create_time", "bug_page", "title",
                "description", "img_url",
                "score", "parent", "children", "root", "good_num", "good_worker_id", "bad_num", "bad_worker_id",
                "test_case_id", "test_case_name", "test_case_front", "test_case_behind", "test_case_description", "test_case_create_time",
                "report_id", "report_name", "report_create_time", "script_location", "report_location", "log_location", "device_model", "device_brand", "device_os",
                "worker_id",
                "case_app_name", "case_paper_type", "case_test_type", "case_description", "case_require_doc",
                "case_take_id", "originalCaseId", "cpSerialNum"};
        //导出文件
        exportCsv(titles, bugDetailList, caseId);
        exportJson(bugDetailList, caseId);
        //导出图片zip包
        String sourceFile = "/data/image/" + caseId;
        try {
            FileOutputStream fileOutputStream = new FileOutputStream(new File("/data/imageZip/" + caseId + ".zip"));
            toZip(sourceFile, fileOutputStream, false);
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }


    }

    private void exportJson(List<BugDetail> bugDetailList, String caseId) {
        try {
            File file = new File("data/output/" + caseId + "/" + caseId + ".json");
            JSONArray jsonArray = new JSONArray(bugDetailList);
            Writer write = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8);
            write.write(jsonArray.toString());
            write.flush();
            write.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }


    private <T> void exportCsv(String[] titles, List<T> list, String caseId) {
        try {
            File file = new File("data/output/" + caseId + "/" + caseId + ".csv");
            //构建输出流，同时指定编码
            OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8);
            //csv文件是逗号分隔，除第一个外，每次写入一个单元格数据后需要输入逗号
            for (String title : titles) {
                ow.write(title);
                ow.write(",");
            }
            //写完文件头后换行
            ow.write("\r\n");
            //写内容
            for (Object obj : list) {
                //利用反射获取所有字段
                Field[] fields = obj.getClass().getDeclaredFields();
                for (Field field : fields) {
                    //设置字段可见性
                    field.setAccessible(true);
                    //防止某个field没有赋值
                    if (field.get(obj) == null) {
                        ow.write("");
                    } else {
                        //解决csv文件中对于逗号和双引号的转义问题
                        ow.write("\"" + field.get(obj).toString().replaceAll("\"", "\"\"") + "\"");
                    }
                    ow.write(",");
                }
                //写完一行换行
                ow.write("\r\n");
            }
            ow.flush();
            ow.close();
        } catch (IOException | IllegalAccessException e) {
            e.printStackTrace();
        }
    }

    private void toZip(String srcDir, OutputStream out, boolean keepDirStructure)
            throws RuntimeException {
        long start = System.currentTimeMillis();
        ZipOutputStream zos = null;
        try {
            zos = new ZipOutputStream(out);
            File sourceFile = new File(srcDir);
            compress(sourceFile, zos, sourceFile.getName(), keepDirStructure);
            long end = System.currentTimeMillis();
            System.out.println("压缩完成，耗时：" + (end - start) + " ms");
        } catch (Exception e) {
            throw new RuntimeException("zip error from ZipUtils", e);
        } finally {
            if (zos != null) {
                try {
                    zos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 递归压缩方法
     *
     * @param sourceFile       源文件
     * @param zos              zip输出流
     * @param name             压缩后的名称
     * @param keepDirStructure 是否保留原来的目录结构,true:保留目录结构;
     *                         false:所有文件跑到压缩包根目录下(注意：不保留目录结构可能会出现同名文件,会压缩失败)
     * @throws Exception
     */
    private void compress(File sourceFile, ZipOutputStream zos, String name,
                          boolean keepDirStructure) throws Exception {
        byte[] buf = new byte[BUFFER_SIZE];
        if (sourceFile.isFile()) {
            // 向zip输出流中添加一个zip实体，构造器中name为zip实体的文件的名字
            zos.putNextEntry(new ZipEntry(name));
            // copy文件到zip输出流中
            int len;
            FileInputStream in = new FileInputStream(sourceFile);
            while ((len = in.read(buf)) != -1) {
                zos.write(buf, 0, len);
            }
            // Complete the entry
            zos.closeEntry();
            in.close();
        } else {
            File[] listFiles = sourceFile.listFiles();
            if (listFiles == null || listFiles.length == 0) {
                // 需要保留原来的文件结构时,需要对空文件夹进行处理
                if (keepDirStructure) {
                    // 空文件夹的处理
                    zos.putNextEntry(new ZipEntry(name + "/"));
                    // 没有文件，不需要文件的copy
                    zos.closeEntry();
                }

            } else {
                for (File file : listFiles) {
                    // 判断是否需要保留原来的文件结构
                    if (keepDirStructure) {
                        // 注意：file.getName()前面需要带上父文件夹的名字加一斜杠,
                        // 不然最后压缩包中就不能保留原来的文件结构,即：所有文件都跑到压缩包根目录下了
                        compress(file, zos, name + "/" + file.getName(), true);
                    } else {
                        compress(file, zos, file.getName(), false);
                    }
                }
            }
        }
    }
}