crowdsource-backend/src/main/java/edu/nju/service/DataService.java

package edu.nju.service;

import com.alibaba.fastjson.JSON;
import com.aliyun.oss.OSS;
import edu.nju.dao.*;
import edu.nju.entities.*;
import edu.nju.util.OssAliyun;
import edu.nju.util.TransUtil;
import org.json.JSONArray;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import java.io.*;
import java.lang.reflect.Field;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

/**
 * @Author JiaWei Xu
 * @Date 2020-12-25 10:45
 * @Email xjwhhh233@outlook.com
 */
@Service
public class DataService {
    private static final int BUFFER_SIZE = 2048;
    @Autowired
    ExamDao examDao;

    @Autowired
    BugDao bugDao;

    @Autowired
    ReportDao reportDao;

    @Autowired
    TestCaseDao testCaseDao;

    @Autowired
    CaseToBugDao caseToBugDao;

    @Autowired
    BugScoreDao bugScoreDao;

    @Autowired
    BugMirrorDao bugMirrorDao;

    @Autowired
    BugHistoryDao bugHistoryDao;

    @Autowired
    BugDetailDao bugDetailDao;

    @Value("${cpSerialNum}")
    private String cpSerialNum;

    private static final String bucketName = "mooctest-site";
    private static final String imageUrlPrefix = "https://mooctest-site.oss-cn-shanghai.aliyuncs.com/xinchuang/image/";


    /**
     * 根据caseId获取对应bug信息
     *
     * @param caseId
     * @return
     */
    public List<BugDetail> getBugDetailByCaseId(String caseId) {
        List<BugDetail> bugDetailList = new ArrayList<>();
        Exam crowdCase = examDao.findById(caseId);
        if (crowdCase != null) {
            List<Report> reportList = reportDao.findByCaseId(caseId);
            for (Report report : reportList) {
                String reportId = report.getId();
                List<TestCase> testCaseList = testCaseDao.findByReport(reportId);
                for (TestCase testCase : testCaseList) {
                    String testCaseId = testCase.getId();
                    CaseToBug caseToBug = caseToBugDao.findById(testCaseId);
                    if (caseToBug != null) {
                        List<String> bugIdList = caseToBug.getBug_id();
                        for (String bugId : bugIdList) {
                            BugDetail bugDetail = new BugDetail();
                            bugDetail.setId(bugId);
                            //bug基本属性
                            Bug bug = bugDao.findByid(bugId);
                            if (bug != null) {
                                bugDetail.setBugCategory(bug.getBug_category());
                                bugDetail.setSeverity(TransUtil.severityTransFromInt(bug.getSeverity()));
                                bugDetail.setRecurrent(TransUtil.recurrentTransFromInt(bug.getRecurrent()));
                                bugDetail.setBugCreateTime(TransUtil.formatTimeMillis(bug.getCreate_time_millis()));
                                bugDetail.setBugPage(bug.getBug_page());
                                bugDetail.setTitle(bug.getTitle());
                                bugDetail.setBugDescription(bug.getDescription());
                                bugDetail.setImgUrl(bug.getImg_url());
                            }
                            //bugScore属性
                            BugScore bugScore = bugScoreDao.findById(bugId);
                            if (bugScore != null) {
                                bugDetail.setScore(bugScore.getGrade());
                            }
                            //bugMirror属性
                            BugMirror bugMirror = bugMirrorDao.findById(bugId);
                            if (bugMirror != null) {
                                Set<String> goodWorkerIdSet = new HashSet<>();
                                Set<String> badWorkerIdSet = new HashSet<>();
                                Set<String> goodReportIdSet = bugMirror.getGood();
                                Set<String> badReportIdSet = bugMirror.getBad();
                                int goodNum = 0;
                                int badNum = 0;
                                for (String goodReportId : goodReportIdSet) {
                                    Report goodReport = reportDao.findById(goodReportId);
                                    if (goodReport != null) {
                                        goodNum++;
                                        goodWorkerIdSet.add(goodReport.getWorker_id());
                                    }
                                }
                                for (String badReportId : badReportIdSet) {
                                    Report badReport = reportDao.findById(badReportId);
                                    if (badReport != null) {
                                        badNum++;
                                        badWorkerIdSet.add(badReport.getWorker_id());
                                    }
                                }
                                bugDetail.setGoodNum(goodNum);
                                bugDetail.setBadNum(badNum);
                                bugDetail.setGoodWorkerId(goodWorkerIdSet);
                                bugDetail.setBadWorkerId(badWorkerIdSet);
                            }
                            //bugHistory属性
                            BugHistory bugHistory = bugHistoryDao.findByid(bugId);
                            if (bugHistory != null) {
                                bugDetail.setParent(bugHistory.getParent());
                                bugDetail.setChildren(bugHistory.getChildren());
                                bugDetail.setRoot(bugHistory.getRoot());
                            }
                            //testCase属性
                            bugDetail.setTestCaseId(testCase.getId());
                            bugDetail.setTestCaseName(testCase.getName());
                            bugDetail.setTestCaseFront(testCase.getFront());
                            bugDetail.setTestCaseBehind(testCase.getBehind());
                            bugDetail.setTestCaseDescription(testCase.getDescription());
                            bugDetail.setTestCaseCreateTime(TransUtil.formatTimeMillis(testCase.getCreate_time_millis()));
                            //report属性
                            bugDetail.setReportId(report.getId());
                            bugDetail.setReportName(report.getName());
                            bugDetail.setScriptLocation(report.getScript_location());
                            bugDetail.setReportLocation(report.getReport_location());
                            bugDetail.setLogLocation(report.getLog_location());
                            bugDetail.setDeviceModel(report.getDevice_model());
                            bugDetail.setDeviceBrand(report.getDevice_brand());
                            bugDetail.setDeviceOs(report.getDevice_os());
                            //worker属性
                            bugDetail.setWorkerId(report.getWorker_id());
                            //众测任务属性
                            bugDetail.setCaseAppName(crowdCase.getName());
                            bugDetail.setCasePaperType(crowdCase.getPaper_type());
                            bugDetail.setCaseTestType(crowdCase.getTest_type());
                            bugDetail.setCaseDescription(crowdCase.getDescription());
                            bugDetail.setCaseRequireDoc("");
                            bugDetail.setCaseTakeId(report.getCase_take_id());
                            //cp序列号
                            bugDetail.setCpSerialNum(cpSerialNum);
                            bugDetailList.add(bugDetail);
                        }
                    }
                }
            }
        }
        bugDetailToFile(bugDetailList, caseId);
        return bugDetailList;
    }

    public List<BugDetail> saveBugDetailFromOss(MultipartFile sourceZipFile, MultipartFile sourceJsonFile, String originalCaseId, String cpSerialNum) {
        try {
//            //从oss下载json文件
//            URL url = new URL(jsonFilePath);
//            URLConnection urlConnection = url.openConnection();
//            HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection;
//            httpURLConnection.setConnectTimeout(1000 * 5);
//            httpURLConnection.setRequestProperty("Charset", "UTF-8");
//            httpURLConnection.connect();

            //读取文件流并保存在本地
            String zipFilePath="xinchuangdata/input/imageZip/"+originalCaseId+"/"+cpSerialNum+"/"+originalCaseId+".zip";
            File zipFile=new File(zipFilePath);
            if(!zipFile.getParentFile().exists()) { zipFile.getParentFile().mkdirs(); }
            if(!sourceZipFile.isEmpty()) { sourceZipFile.transferTo(zipFile); }
            String jsonFilePath="xinchuangdata/input/"+originalCaseId+"/"+cpSerialNum+"/"+originalCaseId+".json";
            File jsonFile=new File(jsonFilePath);
            if(!jsonFile.getParentFile().exists()) { jsonFile.getParentFile().mkdirs(); }
            if(!sourceJsonFile.isEmpty()) { sourceJsonFile.transferTo(zipFile); }
            //读取本地文件
            BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(jsonFile));
            ByteArrayOutputStream buf = new ByteArrayOutputStream();
            int result = bufferedInputStream.read();
            while (result != -1) {
                buf.write((byte) result);
                result = bufferedInputStream.read();
            }
            String json = buf.toString();
            //转为bugDetail
            List<BugDetail> bugDetailList = JSON.parseArray(json, BugDetail.class);
            for (BugDetail bugDetail : bugDetailList) {
                bugDetail.setOriginalCaseId(originalCaseId);
                //修改图片文件路径为oss路径
                String imageUrl = bugDetail.getImgUrl();
                String[] imageUrlArray = imageUrl.split(",");
                StringBuilder stringBuilder = new StringBuilder();
                for (String imageUrlStr : imageUrlArray) {
                    String[] filePath = imageUrlStr.split("/");
                    String fileName = filePath[filePath.length - 1];
                    String newImageUrl = imageUrlPrefix + originalCaseId + "/" + cpSerialNum + "/" + fileName;
                    stringBuilder.append(newImageUrl).append(",");
                }
                bugDetail.setImgUrl(stringBuilder.toString());
                bugDetailDao.save(bugDetail);
            }
            //解压图片文件，上传至oss
            String destPath="xinchuangdata/input/imageUnzip/"+originalCaseId+"/"+cpSerialNum;
            File unzipFile=new File(destPath);
            if(!unzipFile.getParentFile().exists()) { unzipFile.getParentFile().mkdirs(); }
            unZip(zipFile,destPath,originalCaseId,cpSerialNum);
            return bugDetailList;
        } catch (IOException e) {
            e.printStackTrace();
        }
        return new ArrayList<>();
    }

    private void bugDetailToFile(List<BugDetail> bugDetailList, String caseId) {
        String[] titles = {"bug_id", "bug_category", "severity", "recurrent", "bug_create_time", "bug_page", "title",
                "description", "img_url",
                "score", "parent", "children", "root", "good_num", "good_worker_id", "bad_num", "bad_worker_id",
                "test_case_id", "test_case_name", "test_case_front", "test_case_behind", "test_case_description", "test_case_create_time",
                "report_id", "report_name", "report_create_time", "script_location", "report_location", "log_location", "device_model", "device_brand", "device_os",
                "worker_id",
                "case_app_name", "case_paper_type", "case_test_type", "case_description", "case_require_doc",
                "case_take_id", "originalCaseId", "cpSerialNum"};
        File csvFile = exportCsv(titles, bugDetailList, caseId);
        File jsonFile = exportJson(bugDetailList, caseId);
//        uploadToOss(csvFile);
//        uploadToOss(jsonFile);
    }

    private File exportJson(List<BugDetail> bugDetailList, String caseId) {
        try {
            File file = new File("data/output/" + caseId + ".json");
            JSONArray jsonArray = new JSONArray(bugDetailList);
            Writer write = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8);
            write.write(jsonArray.toString());
            write.flush();
            write.close();
            return file;
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }


    private <T> File exportCsv(String[] titles, List<T> list, String caseId) {
        try {
            File file = new File("data/output/" + caseId + ".csv");
            //构建输出流，同时指定编码
            OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8);
            //csv文件是逗号分隔，除第一个外，每次写入一个单元格数据后需要输入逗号
            for (String title : titles) {
                ow.write(title);
                ow.write(",");
            }
            //写完文件头后换行
            ow.write("\r\n");
            //写内容
            for (Object obj : list) {
                //利用反射获取所有字段
                Field[] fields = obj.getClass().getDeclaredFields();
                for (Field field : fields) {
                    //设置字段可见性
                    field.setAccessible(true);
                    //防止某个field没有赋值
                    if (field.get(obj) == null) {
                        ow.write("");
                    } else {
                        //解决csv文件中对于逗号和双引号的转义问题
                        ow.write("\"" + field.get(obj).toString().replaceAll("\"", "\"\"") + "\"");
                    }
                    ow.write(",");
                }
                //写完一行换行
                ow.write("\r\n");
            }
            ow.flush();
            ow.close();
            return file;
        } catch (IOException | IllegalAccessException e) {
            e.printStackTrace();
        }
        return null;
    }

    private static void uploadToOss(String objectName,File file) {
        if (file != null) {
            OSS ossClient = OssAliyun.initShangHaiOss();
            OssAliyun.uploadFile(ossClient, bucketName, objectName, file);
        } else {
            System.out.println("file is null");
        }
    }

    /**
     * zip解压
     *
     * @param srcFile     zip源文件
     * @param destDirPath 解压后的目标文件夹
     * @throws RuntimeException 解压失败会抛出运行时异常
     */

    public static void unZip(File srcFile, String destDirPath,String originalCaseId,String fromCpSerialNum) throws RuntimeException {
        long start = System.currentTimeMillis();
        // 判断源文件是否存在
        if (!srcFile.exists()) {
            throw new RuntimeException(srcFile.getPath() + "所指文件不存在");
        }
        // 开始解压
        ZipFile zipFile = null;
        try {
            zipFile = new ZipFile(srcFile);
            Enumeration<?> entries = zipFile.entries();

            while (entries.hasMoreElements()) {
                ZipEntry entry = (ZipEntry) entries.nextElement();
                System.out.println("解压" + entry.getName());
                // 如果是文件夹，就创建个文件夹
                if (entry.isDirectory()) {
                    String dirPath = destDirPath + "/" + entry.getName();
                    File dir = new File(dirPath);
                    dir.mkdirs();
                } else {
                    // 如果是文件，就先创建一个文件，然后用io流把内容copy过去
                    File targetFile = new File(destDirPath + "/" + entry.getName());
                    // 保证这个文件的父文件夹必须要存在
                    if (!targetFile.getParentFile().exists()) {
                        targetFile.getParentFile().mkdirs();
                    }
                    targetFile.createNewFile();
                    // 将压缩文件内容写入到这个文件中
                    InputStream is = zipFile.getInputStream(entry);
                    FileOutputStream fos = new FileOutputStream(targetFile);
                    int len;
                    byte[] buf = new byte[BUFFER_SIZE];
                    while ((len = is.read(buf)) != -1) {
                        fos.write(buf, 0, len);
                    }
                    // 关流顺序，先打开的后关闭
                    fos.close();
                    is.close();
                    //图片文件上传至oss
                    String objectName = "xinchuang/image/"+originalCaseId+"/"+fromCpSerialNum+"/" + targetFile.getName();
                    uploadToOss(objectName,targetFile);
                }
            }
            long end = System.currentTimeMillis();
            System.out.println("解压完成，耗时：" + (end - start) + " ms");
        } catch (Exception e) {
            throw new RuntimeException("unzip error from ZipUtils", e);
        } finally {
            if (zipFile != null) {
                try {
                    zipFile.close();

                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

//    public static void main(String[] args){
//        File a=new File("/Users/xujiawei/crowd/crowdsource-backend/data/a.zip");
//        unZip(a,"/Users/xujiawei/crowd/crowdsource-backend/data/test","1","test");
//    }
}