crowdsource-backend/src/main/java/edu/nju/service/OssFileService.java

package edu.nju.service;

import edu.nju.entities.BugDetail;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.web.multipart.MultipartFile;

import java.util.List;
import com.alibaba.fastjson.JSON;
import com.aliyun.oss.OSS;
import edu.nju.dao.*;
import edu.nju.entities.*;
import edu.nju.util.OssAliyun;
import edu.nju.util.TransUtil;
import org.json.JSONArray;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.*;
import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

/**
 * @Author JiaWei Xu
 * @Date 2021-01-04 15:53
 * @Email xjwhhh233@outlook.com
 */
@Service
@ConditionalOnExpression("${useOss}==true")
public class OssFileService implements FileService {
    private static final int BUFFER_SIZE = 2048;
    @Autowired
    ExamDao examDao;

    @Autowired
    BugDao bugDao;

    @Autowired
    ReportDao reportDao;

    @Autowired
    TestCaseDao testCaseDao;

    @Autowired
    CaseToBugDao caseToBugDao;

    @Autowired
    BugScoreDao bugScoreDao;

    @Autowired
    BugMirrorDao bugMirrorDao;

    @Autowired
    BugHistoryDao bugHistoryDao;

    @Autowired
    BugDetailDao bugDetailDao;

    @Value("${cpSerialNum}")
    private String cpSerialNum;

    @Value("${oss.bucketName}")
    private String bucketName;

    @Value("${oss.imageUrlPrefix}")
    private String ossImageUrlPrefix;

    String localPrefix ="";

    @Override
    public void uploadImage() {

    }

    @Override
    public List<BugDetail> importBugInfo(MultipartFile sourceZipFile, MultipartFile sourceJsonFile, String originalCaseId, String cpSerialNum) {
        return saveBugDetail(sourceZipFile,sourceJsonFile,originalCaseId,cpSerialNum);
    }

    @Override
    public List<BugDetail> exportBugInfo(String caseId) {
        List<BugDetail> bugDetailList = getBugDetailListByCaseId(caseId);
        bugDetailToFile(bugDetailList,caseId);
        return bugDetailList;
    }


    /**
     * 根据caseId获取对应bug信息
     *
     * @param caseId
     * @return
     */
    public List<BugDetail> getBugDetailListByCaseId(String caseId) {
        List<BugDetail> bugDetailList = new ArrayList<>();
        Exam crowdCase = examDao.findById(caseId);
        if (crowdCase != null) {
            List<Report> reportList = reportDao.findByCaseId(caseId);
            for (Report report : reportList) {
                String reportId = report.getId();
                List<TestCase> testCaseList = testCaseDao.findByReport(reportId);
                for (TestCase testCase : testCaseList) {
                    String testCaseId = testCase.getId();
                    CaseToBug caseToBug = caseToBugDao.findById(testCaseId);
                    if (caseToBug != null) {
                        List<String> bugIdList = caseToBug.getBug_id();
                        for (String bugId : bugIdList) {
                            BugDetail bugDetail = new BugDetail();
                            bugDetail.setId(bugId);
                            //bug基本属性
                            Bug bug = bugDao.findByid(bugId);
                            if (bug != null) {
                                bugDetail.setBugCategory(bug.getBug_category());
                                bugDetail.setSeverity(TransUtil.severityTransFromInt(bug.getSeverity()));
                                bugDetail.setRecurrent(TransUtil.recurrentTransFromInt(bug.getRecurrent()));
                                bugDetail.setBugCreateTime(TransUtil.formatTimeMillis(bug.getCreate_time_millis()));
                                bugDetail.setBugPage(bug.getBug_page());
                                bugDetail.setTitle(bug.getTitle());
                                bugDetail.setBugDescription(bug.getDescription());
                                bugDetail.setImgUrl(bug.getImg_url());
                            }
                            //bugScore属性
                            BugScore bugScore = bugScoreDao.findById(bugId);
                            if (bugScore != null) {
                                bugDetail.setScore(bugScore.getGrade());
                            }
                            //bugMirror属性
                            BugMirror bugMirror = bugMirrorDao.findById(bugId);
                            if (bugMirror != null) {
                                Set<String> goodWorkerIdSet = new HashSet<>();
                                Set<String> badWorkerIdSet = new HashSet<>();
                                Set<String> goodReportIdSet = bugMirror.getGood();
                                Set<String> badReportIdSet = bugMirror.getBad();
                                int goodNum = 0;
                                int badNum = 0;
                                for (String goodReportId : goodReportIdSet) {
                                    Report goodReport = reportDao.findById(goodReportId);
                                    if (goodReport != null) {
                                        goodNum++;
                                        goodWorkerIdSet.add(goodReport.getWorker_id());
                                    }
                                }
                                for (String badReportId : badReportIdSet) {
                                    Report badReport = reportDao.findById(badReportId);
                                    if (badReport != null) {
                                        badNum++;
                                        badWorkerIdSet.add(badReport.getWorker_id());
                                    }
                                }
                                bugDetail.setGoodNum(goodNum);
                                bugDetail.setBadNum(badNum);
                                bugDetail.setGoodWorkerId(goodWorkerIdSet);
                                bugDetail.setBadWorkerId(badWorkerIdSet);
                            }
                            //bugHistory属性
                            BugHistory bugHistory = bugHistoryDao.findByid(bugId);
                            if (bugHistory != null) {
                                bugDetail.setParent(bugHistory.getParent());
                                bugDetail.setChildren(bugHistory.getChildren());
                                bugDetail.setRoot(bugHistory.getRoot());
                            }
                            //testCase属性
                            bugDetail.setTestCaseId(testCase.getId());
                            bugDetail.setTestCaseName(testCase.getName());
                            bugDetail.setTestCaseFront(testCase.getFront());
                            bugDetail.setTestCaseBehind(testCase.getBehind());
                            bugDetail.setTestCaseDescription(testCase.getDescription());
                            bugDetail.setTestCaseCreateTime(TransUtil.formatTimeMillis(testCase.getCreate_time_millis()));
                            //report属性
                            bugDetail.setReportId(report.getId());
                            bugDetail.setReportName(report.getName());
                            bugDetail.setScriptLocation(report.getScript_location());
                            bugDetail.setReportLocation(report.getReport_location());
                            bugDetail.setLogLocation(report.getLog_location());
                            bugDetail.setDeviceModel(report.getDevice_model());
                            bugDetail.setDeviceBrand(report.getDevice_brand());
                            bugDetail.setDeviceOs(report.getDevice_os());
                            //worker属性
                            bugDetail.setWorkerId(report.getWorker_id());
                            //众测任务属性
                            bugDetail.setCaseAppName(crowdCase.getName());
                            bugDetail.setCasePaperType(crowdCase.getPaper_type());
                            bugDetail.setCaseTestType(crowdCase.getTest_type());
                            bugDetail.setCaseDescription(crowdCase.getDescription());
                            bugDetail.setCaseRequireDoc("");
                            bugDetail.setCaseTakeId(report.getCase_take_id());
                            //cp序列号
                            bugDetail.setCpSerialNum(cpSerialNum);
                            bugDetailList.add(bugDetail);
                        }
                    }
                }
            }
        }
        return bugDetailList;
    }

    public List<BugDetail> saveBugDetail(MultipartFile sourceZipFile, MultipartFile sourceJsonFile, String originalCaseId, String cpSerialNum) {
        try {
            //读取文件流并保存在本地

            String zipFilePath= localPrefix +"/xinchuangdata/input/imageZip/"+originalCaseId+"/"+cpSerialNum+"/"+originalCaseId+".zip";
            File zipFile=new File(zipFilePath);
            if(!zipFile.getParentFile().exists()) { zipFile.getParentFile().mkdirs(); }
            if(!sourceZipFile.isEmpty()) { sourceZipFile.transferTo(zipFile); }
            String jsonFilePath= localPrefix +"/xinchuangdata/input/"+originalCaseId+"/"+cpSerialNum+"/"+originalCaseId+".json";
            File jsonFile=new File(jsonFilePath);
            if(!jsonFile.getParentFile().exists()) { jsonFile.getParentFile().mkdirs(); }
            if(!sourceJsonFile.isEmpty()) { sourceJsonFile.transferTo(jsonFile); }
            //读取本地文件
            BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(jsonFile));
            ByteArrayOutputStream buf = new ByteArrayOutputStream();
            int result = bufferedInputStream.read();
            while (result != -1) {
                buf.write((byte) result);
                result = bufferedInputStream.read();
            }
            String json = buf.toString();
            //转为bugDetail
            List<BugDetail> bugDetailList = JSON.parseArray(json, BugDetail.class);
            for (BugDetail bugDetail : bugDetailList) {
                bugDetail.setOriginalCaseId(originalCaseId);
                //修改图片文件路径为oss路径
                String imageUrl = bugDetail.getImgUrl();
                String[] imageUrlArray = imageUrl.split(",");
                StringBuilder stringBuilder = new StringBuilder();
                for (String imageUrlStr : imageUrlArray) {
                    if(!"".equals(imageUrl)) {
                        String[] filePath = imageUrlStr.split("/");
                        String fileName = filePath[filePath.length - 1];
                        String newImageUrl = ossImageUrlPrefix + originalCaseId + "/" + cpSerialNum + "/" + fileName;
                        stringBuilder.append(newImageUrl).append(",");
                    }
                }
                bugDetail.setImgUrl(stringBuilder.toString());
                bugDetailDao.save(bugDetail);
            }
            //解压图片文件，上传至oss
            String destPath= localPrefix +"/xinchuangdata/input/imageUnzip/"+originalCaseId+"/"+cpSerialNum;
            File unzipFile=new File(destPath);
            if(!unzipFile.getParentFile().exists()) { unzipFile.getParentFile().mkdirs(); }
            unZip(zipFile,destPath,originalCaseId,cpSerialNum);
            return bugDetailList;
        } catch (IOException e) {
            e.printStackTrace();
        }
        return new ArrayList<>();
    }

    private void bugDetailToFile(List<BugDetail> bugDetailList, String caseId) {
        String[] titles = {"bug_id", "bug_category", "severity", "recurrent", "bug_create_time", "bug_page", "title",
                "description", "img_url",
                "score", "parent", "children", "root", "good_num", "good_worker_id", "bad_num", "bad_worker_id",
                "test_case_id", "test_case_name", "test_case_front", "test_case_behind", "test_case_description", "test_case_create_time",
                "report_id", "report_name", "report_create_time", "script_location", "report_location", "log_location", "device_model", "device_brand", "device_os",
                "worker_id",
                "case_app_name", "case_paper_type", "case_test_type", "case_description", "case_require_doc",
                "case_take_id", "originalCaseId", "cpSerialNum"};
        File csvFile = exportCsv(titles, bugDetailList, caseId);
        File jsonFile = exportJson(bugDetailList, caseId);
        String csvObjectName = "xinchuang/output/"+caseId+"/"+cpSerialNum+"/" + csvFile.getName();
        String jsonObjectName = "xinchuang/output/"+caseId+"/"+cpSerialNum+"/" + jsonFile.getName();
        uploadToOss(csvObjectName,csvFile);
        uploadToOss(jsonObjectName,jsonFile);
    }

    private File exportJson(List<BugDetail> bugDetailList, String caseId) {
        try {
            File file = new File(localPrefix+"/xinchuangdata/output/" + caseId + ".json");
            if(!file.getParentFile().exists()) { file.getParentFile().mkdirs(); }
            JSONArray jsonArray = new JSONArray(bugDetailList);
            Writer write = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8);
            write.write(jsonArray.toString());
            write.flush();
            write.close();
            return file;
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }


    private <T> File exportCsv(String[] titles, List<T> list, String caseId) {
        try {
            File file = new File(localPrefix+"/xinchuangdata/output/" + caseId + ".csv");
            if(!file.getParentFile().exists()) { file.getParentFile().mkdirs(); }
            //构建输出流，同时指定编码
            OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8);
            //csv文件是逗号分隔，除第一个外，每次写入一个单元格数据后需要输入逗号
            for (String title : titles) {
                ow.write(title);
                ow.write(",");
            }
            //写完文件头后换行
            ow.write("\r\n");
            //写内容
            for (Object obj : list) {
                //利用反射获取所有字段
                Field[] fields = obj.getClass().getDeclaredFields();
                for (Field field : fields) {
                    //设置字段可见性
                    field.setAccessible(true);
                    //防止某个field没有赋值
                    if (field.get(obj) == null) {
                        ow.write("");
                    } else {
                        //解决csv文件中对于逗号和双引号的转义问题
                        ow.write("\"" + field.get(obj).toString().replaceAll("\"", "\"\"") + "\"");
                    }
                    ow.write(",");
                }
                //写完一行换行
                ow.write("\r\n");
            }
            ow.flush();
            ow.close();
            return file;
        } catch (IOException | IllegalAccessException e) {
            e.printStackTrace();
        }
        return null;
    }

    private void uploadToOss(String objectName,File file) {
        if (file != null) {
            OSS ossClient = OssAliyun.initShangHaiOss();
            OssAliyun.uploadFile(ossClient, bucketName, objectName, file);
        } else {
            System.out.println("file is null");
        }
    }

    /**
     * zip解压
     *
     * @param srcFile     zip源文件
     * @param destDirPath 解压后的目标文件夹
     * @throws RuntimeException 解压失败会抛出运行时异常
     */

    private void unZip(File srcFile, String destDirPath,String originalCaseId,String fromCpSerialNum) throws RuntimeException {
        long start = System.currentTimeMillis();
        // 判断源文件是否存在
        if (!srcFile.exists()) {
            throw new RuntimeException(srcFile.getPath() + "所指文件不存在");
        }
        // 开始解压
        ZipFile zipFile = null;
        try {
            zipFile = new ZipFile(srcFile);
            Enumeration<?> entries = zipFile.entries();

            while (entries.hasMoreElements()) {
                ZipEntry entry = (ZipEntry) entries.nextElement();
                System.out.println("解压" + entry.getName());
                // 如果是文件夹，就创建个文件夹
                if (entry.isDirectory()) {
                    String dirPath = destDirPath + "/" + entry.getName();
                    File dir = new File(dirPath);
                    dir.mkdirs();
                } else {
                    // 如果是文件，就先创建一个文件，然后用io流把内容copy过去
                    File targetFile = new File(destDirPath + "/" + entry.getName());
                    // 保证这个文件的父文件夹必须要存在
                    if (!targetFile.getParentFile().exists()) {
                        targetFile.getParentFile().mkdirs();
                    }
                    targetFile.createNewFile();
                    // 将压缩文件内容写入到这个文件中
                    InputStream is = zipFile.getInputStream(entry);
                    FileOutputStream fos = new FileOutputStream(targetFile);
                    int len;
                    byte[] buf = new byte[BUFFER_SIZE];
                    while ((len = is.read(buf)) != -1) {
                        fos.write(buf, 0, len);
                    }
                    // 关流顺序，先打开的后关闭
                    fos.close();
                    is.close();
                    //图片文件上传至oss
                    String objectName = "xinchuang/image/"+originalCaseId+"/"+fromCpSerialNum+"/" + targetFile.getName();
                    uploadToOss(objectName,targetFile);
                }
            }
            long end = System.currentTimeMillis();
            System.out.println("解压完成，耗时：" + (end - start) + " ms");
        } catch (Exception e) {
            throw new RuntimeException("unzip error from ZipUtils", e);
        } finally {
            if (zipFile != null) {
                try {
                    zipFile.close();

                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    private void toZip(String srcDir, OutputStream out, boolean keepDirStructure)
            throws RuntimeException {
        long start = System.currentTimeMillis();
        ZipOutputStream zos = null;
        try {
            zos = new ZipOutputStream(out);
            File sourceFile = new File(srcDir);
            compress(sourceFile, zos, sourceFile.getName(), keepDirStructure);
            long end = System.currentTimeMillis();
            System.out.println("压缩完成，耗时：" + (end - start) + " ms");
        } catch (Exception e) {
            throw new RuntimeException("zip error from ZipUtils", e);
        } finally {
            if (zos != null) {
                try {
                    zos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 递归压缩方法
     *
     * @param sourceFile       源文件
     * @param zos              zip输出流
     * @param name             压缩后的名称
     * @param keepDirStructure 是否保留原来的目录结构,true:保留目录结构;
     *                         false:所有文件跑到压缩包根目录下(注意：不保留目录结构可能会出现同名文件,会压缩失败)
     * @throws Exception
     */
    private void compress(File sourceFile, ZipOutputStream zos, String name,
                          boolean keepDirStructure) throws Exception {
        byte[] buf = new byte[BUFFER_SIZE];
        if (sourceFile.isFile()) {
            // 向zip输出流中添加一个zip实体，构造器中name为zip实体的文件的名字
            zos.putNextEntry(new ZipEntry(name));
            // copy文件到zip输出流中
            int len;
            FileInputStream in = new FileInputStream(sourceFile);
            while ((len = in.read(buf)) != -1) {
                zos.write(buf, 0, len);
            }
            // Complete the entry
            zos.closeEntry();
            in.close();
        } else {
            File[] listFiles = sourceFile.listFiles();
            if (listFiles == null || listFiles.length == 0) {
                // 需要保留原来的文件结构时,需要对空文件夹进行处理
                if (keepDirStructure) {
                    // 空文件夹的处理
                    zos.putNextEntry(new ZipEntry(name + "/"));
                    // 没有文件，不需要文件的copy
                    zos.closeEntry();
                }

            } else {
                for (File file : listFiles) {
                    // 判断是否需要保留原来的文件结构
                    if (keepDirStructure) {
                        // 注意：file.getName()前面需要带上父文件夹的名字加一斜杠,
                        // 不然最后压缩包中就不能保留原来的文件结构,即：所有文件都跑到压缩包根目录下了
                        compress(file, zos, name + "/" + file.getName(), true);
                    } else {
                        compress(file, zos, file.getName(), false);
                    }
                }
            }
        }
    }
}