package edu.nju.service; import edu.nju.entities.BugDetail; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.web.multipart.MultipartFile; import java.util.List; import com.alibaba.fastjson.JSON; import com.aliyun.oss.OSS; import edu.nju.dao.*; import edu.nju.entities.*; import edu.nju.util.OssAliyun; import edu.nju.util.TransUtil; import org.json.JSONArray; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.*; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; /** * @Author JiaWei Xu * @Date 2021-01-04 15:53 * @Email xjwhhh233@outlook.com */ @Service @ConditionalOnExpression("${useOss}==true") public class OssFileService implements FileService { private static final int BUFFER_SIZE = 2048; @Autowired ExamDao examDao; @Autowired BugDao bugDao; @Autowired ReportDao reportDao; @Autowired TestCaseDao testCaseDao; @Autowired CaseToBugDao caseToBugDao; @Autowired BugScoreDao bugScoreDao; @Autowired BugMirrorDao bugMirrorDao; @Autowired BugHistoryDao bugHistoryDao; @Autowired BugDetailDao bugDetailDao; @Value("${cpSerialNum}") private String cpSerialNum; @Value("${oss.bucketName}") private String bucketName; @Value("${oss.imageUrlPrefix}") private String ossImageUrlPrefix; String localPrefix =""; @Override public void uploadImage() { } @Override public List importBugInfo(MultipartFile sourceZipFile, MultipartFile sourceJsonFile, String originalCaseId, String cpSerialNum) { return saveBugDetail(sourceZipFile,sourceJsonFile,originalCaseId,cpSerialNum); } @Override public List exportBugInfo(String caseId) { List bugDetailList = getBugDetailListByCaseId(caseId); bugDetailToFile(bugDetailList,caseId); return bugDetailList; } /** * 根据caseId获取对应bug信息 * * @param caseId * @return */ public List getBugDetailListByCaseId(String caseId) { List bugDetailList = new ArrayList<>(); Exam crowdCase = examDao.findById(caseId); if (crowdCase != null) { List reportList = reportDao.findByCaseId(caseId); for (Report report : reportList) { String reportId = report.getId(); List testCaseList = testCaseDao.findByReport(reportId); for (TestCase testCase : testCaseList) { String testCaseId = testCase.getId(); CaseToBug caseToBug = caseToBugDao.findById(testCaseId); if (caseToBug != null) { List bugIdList = caseToBug.getBug_id(); for (String bugId : bugIdList) { BugDetail bugDetail = new BugDetail(); bugDetail.setId(bugId); //bug基本属性 Bug bug = bugDao.findByid(bugId); if (bug != null) { bugDetail.setBugCategory(bug.getBug_category()); bugDetail.setSeverity(TransUtil.severityTransFromInt(bug.getSeverity())); bugDetail.setRecurrent(TransUtil.recurrentTransFromInt(bug.getRecurrent())); bugDetail.setBugCreateTime(TransUtil.formatTimeMillis(bug.getCreate_time_millis())); bugDetail.setBugPage(bug.getBug_page()); bugDetail.setTitle(bug.getTitle()); bugDetail.setBugDescription(bug.getDescription()); bugDetail.setImgUrl(bug.getImg_url()); } //bugScore属性 BugScore bugScore = bugScoreDao.findById(bugId); if (bugScore != null) { bugDetail.setScore(bugScore.getGrade()); } //bugMirror属性 BugMirror bugMirror = bugMirrorDao.findById(bugId); if (bugMirror != null) { Set goodWorkerIdSet = new HashSet<>(); Set badWorkerIdSet = new HashSet<>(); Set goodReportIdSet = bugMirror.getGood(); Set badReportIdSet = bugMirror.getBad(); int goodNum = 0; int badNum = 0; for (String goodReportId : goodReportIdSet) { Report goodReport = reportDao.findById(goodReportId); if (goodReport != null) { goodNum++; goodWorkerIdSet.add(goodReport.getWorker_id()); } } for (String badReportId : badReportIdSet) { Report badReport = reportDao.findById(badReportId); if (badReport != null) { badNum++; badWorkerIdSet.add(badReport.getWorker_id()); } } bugDetail.setGoodNum(goodNum); bugDetail.setBadNum(badNum); bugDetail.setGoodWorkerId(goodWorkerIdSet); bugDetail.setBadWorkerId(badWorkerIdSet); } //bugHistory属性 BugHistory bugHistory = bugHistoryDao.findByid(bugId); if (bugHistory != null) { bugDetail.setParent(bugHistory.getParent()); bugDetail.setChildren(bugHistory.getChildren()); bugDetail.setRoot(bugHistory.getRoot()); } //testCase属性 bugDetail.setTestCaseId(testCase.getId()); bugDetail.setTestCaseName(testCase.getName()); bugDetail.setTestCaseFront(testCase.getFront()); bugDetail.setTestCaseBehind(testCase.getBehind()); bugDetail.setTestCaseDescription(testCase.getDescription()); bugDetail.setTestCaseCreateTime(TransUtil.formatTimeMillis(testCase.getCreate_time_millis())); //report属性 bugDetail.setReportId(report.getId()); bugDetail.setReportName(report.getName()); bugDetail.setScriptLocation(report.getScript_location()); bugDetail.setReportLocation(report.getReport_location()); bugDetail.setLogLocation(report.getLog_location()); bugDetail.setDeviceModel(report.getDevice_model()); bugDetail.setDeviceBrand(report.getDevice_brand()); bugDetail.setDeviceOs(report.getDevice_os()); //worker属性 bugDetail.setWorkerId(report.getWorker_id()); //众测任务属性 bugDetail.setCaseAppName(crowdCase.getName()); bugDetail.setCasePaperType(crowdCase.getPaper_type()); bugDetail.setCaseTestType(crowdCase.getTest_type()); bugDetail.setCaseDescription(crowdCase.getDescription()); bugDetail.setCaseRequireDoc(""); bugDetail.setCaseTakeId(report.getCase_take_id()); //cp序列号 bugDetail.setCpSerialNum(cpSerialNum); bugDetailList.add(bugDetail); } } } } } return bugDetailList; } public List saveBugDetail(MultipartFile sourceZipFile, MultipartFile sourceJsonFile, String originalCaseId, String cpSerialNum) { try { //读取文件流并保存在本地 String zipFilePath= localPrefix +"/xinchuangdata/input/imageZip/"+originalCaseId+"/"+cpSerialNum+"/"+originalCaseId+".zip"; File zipFile=new File(zipFilePath); if(!zipFile.getParentFile().exists()) { zipFile.getParentFile().mkdirs(); } if(!sourceZipFile.isEmpty()) { sourceZipFile.transferTo(zipFile); } String jsonFilePath= localPrefix +"/xinchuangdata/input/"+originalCaseId+"/"+cpSerialNum+"/"+originalCaseId+".json"; File jsonFile=new File(jsonFilePath); if(!jsonFile.getParentFile().exists()) { jsonFile.getParentFile().mkdirs(); } if(!sourceJsonFile.isEmpty()) { sourceJsonFile.transferTo(jsonFile); } //读取本地文件 BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(jsonFile)); ByteArrayOutputStream buf = new ByteArrayOutputStream(); int result = bufferedInputStream.read(); while (result != -1) { buf.write((byte) result); result = bufferedInputStream.read(); } String json = buf.toString(); //转为bugDetail List bugDetailList = JSON.parseArray(json, BugDetail.class); for (BugDetail bugDetail : bugDetailList) { bugDetail.setOriginalCaseId(originalCaseId); //修改图片文件路径为oss路径 String imageUrl = bugDetail.getImgUrl(); String[] imageUrlArray = imageUrl.split(","); StringBuilder stringBuilder = new StringBuilder(); for (String imageUrlStr : imageUrlArray) { if(!"".equals(imageUrl)) { String[] filePath = imageUrlStr.split("/"); String fileName = filePath[filePath.length - 1]; String newImageUrl = ossImageUrlPrefix + originalCaseId + "/" + cpSerialNum + "/" + fileName; stringBuilder.append(newImageUrl).append(","); } } bugDetail.setImgUrl(stringBuilder.toString()); bugDetailDao.save(bugDetail); } //解压图片文件,上传至oss String destPath= localPrefix +"/xinchuangdata/input/imageUnzip/"+originalCaseId+"/"+cpSerialNum; File unzipFile=new File(destPath); if(!unzipFile.getParentFile().exists()) { unzipFile.getParentFile().mkdirs(); } unZip(zipFile,destPath,originalCaseId,cpSerialNum); return bugDetailList; } catch (IOException e) { e.printStackTrace(); } return new ArrayList<>(); } private void bugDetailToFile(List bugDetailList, String caseId) { String[] titles = {"bug_id", "bug_category", "severity", "recurrent", "bug_create_time", "bug_page", "title", "description", "img_url", "score", "parent", "children", "root", "good_num", "good_worker_id", "bad_num", "bad_worker_id", "test_case_id", "test_case_name", "test_case_front", "test_case_behind", "test_case_description", "test_case_create_time", "report_id", "report_name", "report_create_time", "script_location", "report_location", "log_location", "device_model", "device_brand", "device_os", "worker_id", "case_app_name", "case_paper_type", "case_test_type", "case_description", "case_require_doc", "case_take_id", "originalCaseId", "cpSerialNum"}; File csvFile = exportCsv(titles, bugDetailList, caseId); File jsonFile = exportJson(bugDetailList, caseId); String csvObjectName = "xinchuang/output/"+caseId+"/"+cpSerialNum+"/" + csvFile.getName(); String jsonObjectName = "xinchuang/output/"+caseId+"/"+cpSerialNum+"/" + jsonFile.getName(); uploadToOss(csvObjectName,csvFile); uploadToOss(jsonObjectName,jsonFile); } private File exportJson(List bugDetailList, String caseId) { try { File file = new File(localPrefix+"/xinchuangdata/output/" + caseId + ".json"); if(!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } JSONArray jsonArray = new JSONArray(bugDetailList); Writer write = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8); write.write(jsonArray.toString()); write.flush(); write.close(); return file; } catch (IOException e) { e.printStackTrace(); } return null; } private File exportCsv(String[] titles, List list, String caseId) { try { File file = new File(localPrefix+"/xinchuangdata/output/" + caseId + ".csv"); if(!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } //构建输出流,同时指定编码 OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8); //csv文件是逗号分隔,除第一个外,每次写入一个单元格数据后需要输入逗号 for (String title : titles) { ow.write(title); ow.write(","); } //写完文件头后换行 ow.write("\r\n"); //写内容 for (Object obj : list) { //利用反射获取所有字段 Field[] fields = obj.getClass().getDeclaredFields(); for (Field field : fields) { //设置字段可见性 field.setAccessible(true); //防止某个field没有赋值 if (field.get(obj) == null) { ow.write(""); } else { //解决csv文件中对于逗号和双引号的转义问题 ow.write("\"" + field.get(obj).toString().replaceAll("\"", "\"\"") + "\""); } ow.write(","); } //写完一行换行 ow.write("\r\n"); } ow.flush(); ow.close(); return file; } catch (IOException | IllegalAccessException e) { e.printStackTrace(); } return null; } private void uploadToOss(String objectName,File file) { if (file != null) { OSS ossClient = OssAliyun.initShangHaiOss(); OssAliyun.uploadFile(ossClient, bucketName, objectName, file); } else { System.out.println("file is null"); } } /** * zip解压 * * @param srcFile zip源文件 * @param destDirPath 解压后的目标文件夹 * @throws RuntimeException 解压失败会抛出运行时异常 */ private void unZip(File srcFile, String destDirPath,String originalCaseId,String fromCpSerialNum) throws RuntimeException { long start = System.currentTimeMillis(); // 判断源文件是否存在 if (!srcFile.exists()) { throw new RuntimeException(srcFile.getPath() + "所指文件不存在"); } // 开始解压 ZipFile zipFile = null; try { zipFile = new ZipFile(srcFile); Enumeration entries = zipFile.entries(); while (entries.hasMoreElements()) { ZipEntry entry = (ZipEntry) entries.nextElement(); System.out.println("解压" + entry.getName()); // 如果是文件夹,就创建个文件夹 if (entry.isDirectory()) { String dirPath = destDirPath + "/" + entry.getName(); File dir = new File(dirPath); dir.mkdirs(); } else { // 如果是文件,就先创建一个文件,然后用io流把内容copy过去 File targetFile = new File(destDirPath + "/" + entry.getName()); // 保证这个文件的父文件夹必须要存在 if (!targetFile.getParentFile().exists()) { targetFile.getParentFile().mkdirs(); } targetFile.createNewFile(); // 将压缩文件内容写入到这个文件中 InputStream is = zipFile.getInputStream(entry); FileOutputStream fos = new FileOutputStream(targetFile); int len; byte[] buf = new byte[BUFFER_SIZE]; while ((len = is.read(buf)) != -1) { fos.write(buf, 0, len); } // 关流顺序,先打开的后关闭 fos.close(); is.close(); //图片文件上传至oss String objectName = "xinchuang/image/"+originalCaseId+"/"+fromCpSerialNum+"/" + targetFile.getName(); uploadToOss(objectName,targetFile); } } long end = System.currentTimeMillis(); System.out.println("解压完成,耗时:" + (end - start) + " ms"); } catch (Exception e) { throw new RuntimeException("unzip error from ZipUtils", e); } finally { if (zipFile != null) { try { zipFile.close(); } catch (IOException e) { e.printStackTrace(); } } } } private void toZip(String srcDir, OutputStream out, boolean keepDirStructure) throws RuntimeException { long start = System.currentTimeMillis(); ZipOutputStream zos = null; try { zos = new ZipOutputStream(out); File sourceFile = new File(srcDir); compress(sourceFile, zos, sourceFile.getName(), keepDirStructure); long end = System.currentTimeMillis(); System.out.println("压缩完成,耗时:" + (end - start) + " ms"); } catch (Exception e) { throw new RuntimeException("zip error from ZipUtils", e); } finally { if (zos != null) { try { zos.close(); } catch (IOException e) { e.printStackTrace(); } } } } /** * 递归压缩方法 * * @param sourceFile 源文件 * @param zos zip输出流 * @param name 压缩后的名称 * @param keepDirStructure 是否保留原来的目录结构,true:保留目录结构; * false:所有文件跑到压缩包根目录下(注意:不保留目录结构可能会出现同名文件,会压缩失败) * @throws Exception */ private void compress(File sourceFile, ZipOutputStream zos, String name, boolean keepDirStructure) throws Exception { byte[] buf = new byte[BUFFER_SIZE]; if (sourceFile.isFile()) { // 向zip输出流中添加一个zip实体,构造器中name为zip实体的文件的名字 zos.putNextEntry(new ZipEntry(name)); // copy文件到zip输出流中 int len; FileInputStream in = new FileInputStream(sourceFile); while ((len = in.read(buf)) != -1) { zos.write(buf, 0, len); } // Complete the entry zos.closeEntry(); in.close(); } else { File[] listFiles = sourceFile.listFiles(); if (listFiles == null || listFiles.length == 0) { // 需要保留原来的文件结构时,需要对空文件夹进行处理 if (keepDirStructure) { // 空文件夹的处理 zos.putNextEntry(new ZipEntry(name + "/")); // 没有文件,不需要文件的copy zos.closeEntry(); } } else { for (File file : listFiles) { // 判断是否需要保留原来的文件结构 if (keepDirStructure) { // 注意:file.getName()前面需要带上父文件夹的名字加一斜杠, // 不然最后压缩包中就不能保留原来的文件结构,即:所有文件都跑到压缩包根目录下了 compress(file, zos, name + "/" + file.getName(), true); } else { compress(file, zos, file.getName(), false); } } } } } }