package edu.nju.service; import com.alibaba.fastjson.JSON; import com.aliyun.oss.OSS; import edu.nju.dao.*; import edu.nju.entities.*; import edu.nju.util.OssAliyun; import edu.nju.util.TransUtil; import org.json.JSONArray; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.io.*; import java.lang.reflect.Field; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; /** * @Author JiaWei Xu * @Date 2020-12-25 10:45 * @Email xjwhhh233@outlook.com */ @Service public class DataService { private static final int BUFFER_SIZE = 2048; @Autowired ExamDao examDao; @Autowired BugDao bugDao; @Autowired ReportDao reportDao; @Autowired TestCaseDao testCaseDao; @Autowired CaseToBugDao caseToBugDao; @Autowired BugScoreDao bugScoreDao; @Autowired BugMirrorDao bugMirrorDao; @Autowired BugHistoryDao bugHistoryDao; @Autowired BugDetailDao bugDetailDao; @Value("${cpSerialNum}") private String cpSerialNum; private static final String bucketName = "mooctest-site"; private static final String imageUrlPrefix = "https://mooctest-site.oss-cn-shanghai.aliyuncs.com/xinchuang/image/"; /** * 根据caseId获取对应bug信息 * * @param caseId * @return */ public List getBugDetailByCaseId(String caseId) { List bugDetailList = new ArrayList<>(); Exam crowdCase = examDao.findById(caseId); if (crowdCase != null) { List reportList = reportDao.findByCaseId(caseId); for (Report report : reportList) { String reportId = report.getId(); List testCaseList = testCaseDao.findByReport(reportId); for (TestCase testCase : testCaseList) { String testCaseId = testCase.getId(); CaseToBug caseToBug = caseToBugDao.findById(testCaseId); if (caseToBug != null) { List bugIdList = caseToBug.getBug_id(); for (String bugId : bugIdList) { BugDetail bugDetail = new BugDetail(); bugDetail.setId(bugId); //bug基本属性 Bug bug = bugDao.findByid(bugId); if (bug != null) { bugDetail.setBugCategory(bug.getBug_category()); bugDetail.setSeverity(TransUtil.severityTransFromInt(bug.getSeverity())); bugDetail.setRecurrent(TransUtil.recurrentTransFromInt(bug.getRecurrent())); bugDetail.setBugCreateTime(TransUtil.formatTimeMillis(bug.getCreate_time_millis())); bugDetail.setBugPage(bug.getBug_page()); bugDetail.setTitle(bug.getTitle()); bugDetail.setBugDescription(bug.getDescription()); bugDetail.setImgUrl(bug.getImg_url()); } //bugScore属性 BugScore bugScore = bugScoreDao.findById(bugId); if (bugScore != null) { bugDetail.setScore(bugScore.getGrade()); } //bugMirror属性 BugMirror bugMirror = bugMirrorDao.findById(bugId); if (bugMirror != null) { Set goodWorkerIdSet = new HashSet<>(); Set badWorkerIdSet = new HashSet<>(); Set goodReportIdSet = bugMirror.getGood(); Set badReportIdSet = bugMirror.getBad(); int goodNum = 0; int badNum = 0; for (String goodReportId : goodReportIdSet) { Report goodReport = reportDao.findById(goodReportId); if (goodReport != null) { goodNum++; goodWorkerIdSet.add(goodReport.getWorker_id()); } } for (String badReportId : badReportIdSet) { Report badReport = reportDao.findById(badReportId); if (badReport != null) { badNum++; badWorkerIdSet.add(badReport.getWorker_id()); } } bugDetail.setGoodNum(goodNum); bugDetail.setBadNum(badNum); bugDetail.setGoodWorkerId(goodWorkerIdSet); bugDetail.setBadWorkerId(badWorkerIdSet); } //bugHistory属性 BugHistory bugHistory = bugHistoryDao.findByid(bugId); if (bugHistory != null) { bugDetail.setParent(bugHistory.getParent()); bugDetail.setChildren(bugHistory.getChildren()); bugDetail.setRoot(bugHistory.getRoot()); } //testCase属性 bugDetail.setTestCaseId(testCase.getId()); bugDetail.setTestCaseName(testCase.getName()); bugDetail.setTestCaseFront(testCase.getFront()); bugDetail.setTestCaseBehind(testCase.getBehind()); bugDetail.setTestCaseDescription(testCase.getDescription()); bugDetail.setTestCaseCreateTime(TransUtil.formatTimeMillis(testCase.getCreate_time_millis())); //report属性 bugDetail.setReportId(report.getId()); bugDetail.setReportName(report.getName()); bugDetail.setScriptLocation(report.getScript_location()); bugDetail.setReportLocation(report.getReport_location()); bugDetail.setLogLocation(report.getLog_location()); bugDetail.setDeviceModel(report.getDevice_model()); bugDetail.setDeviceBrand(report.getDevice_brand()); bugDetail.setDeviceOs(report.getDevice_os()); //worker属性 bugDetail.setWorkerId(report.getWorker_id()); //众测任务属性 bugDetail.setCaseAppName(crowdCase.getName()); bugDetail.setCasePaperType(crowdCase.getPaper_type()); bugDetail.setCaseTestType(crowdCase.getTest_type()); bugDetail.setCaseDescription(crowdCase.getDescription()); bugDetail.setCaseRequireDoc(""); bugDetail.setCaseTakeId(report.getCase_take_id()); //cp序列号 bugDetail.setCpSerialNum(cpSerialNum); bugDetailList.add(bugDetail); } } } } } bugDetailToFile(bugDetailList, caseId); return bugDetailList; } public List saveBugDetailFromOss(MultipartFile sourceZipFile, MultipartFile sourceJsonFile, String originalCaseId, String cpSerialNum) { try { // //从oss下载json文件 // URL url = new URL(jsonFilePath); // URLConnection urlConnection = url.openConnection(); // HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection; // httpURLConnection.setConnectTimeout(1000 * 5); // httpURLConnection.setRequestProperty("Charset", "UTF-8"); // httpURLConnection.connect(); //读取文件流并保存在本地 String zipFilePath="xinchuangdata/input/imageZip/"+originalCaseId+"/"+cpSerialNum+"/"+originalCaseId+".zip"; File zipFile=new File(zipFilePath); if(!zipFile.getParentFile().exists()) { zipFile.getParentFile().mkdirs(); } if(!sourceZipFile.isEmpty()) { sourceZipFile.transferTo(zipFile); } String jsonFilePath="xinchuangdata/input/"+originalCaseId+"/"+cpSerialNum+"/"+originalCaseId+".json"; File jsonFile=new File(jsonFilePath); if(!jsonFile.getParentFile().exists()) { jsonFile.getParentFile().mkdirs(); } if(!sourceJsonFile.isEmpty()) { sourceJsonFile.transferTo(zipFile); } //读取本地文件 BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(jsonFile)); ByteArrayOutputStream buf = new ByteArrayOutputStream(); int result = bufferedInputStream.read(); while (result != -1) { buf.write((byte) result); result = bufferedInputStream.read(); } String json = buf.toString(); //转为bugDetail List bugDetailList = JSON.parseArray(json, BugDetail.class); for (BugDetail bugDetail : bugDetailList) { bugDetail.setOriginalCaseId(originalCaseId); //修改图片文件路径为oss路径 String imageUrl = bugDetail.getImgUrl(); String[] imageUrlArray = imageUrl.split(","); StringBuilder stringBuilder = new StringBuilder(); for (String imageUrlStr : imageUrlArray) { String[] filePath = imageUrlStr.split("/"); String fileName = filePath[filePath.length - 1]; String newImageUrl = imageUrlPrefix + originalCaseId + "/" + cpSerialNum + "/" + fileName; stringBuilder.append(newImageUrl).append(","); } bugDetail.setImgUrl(stringBuilder.toString()); bugDetailDao.save(bugDetail); } //解压图片文件,上传至oss String destPath="xinchuangdata/input/imageUnzip/"+originalCaseId+"/"+cpSerialNum; File unzipFile=new File(destPath); if(!unzipFile.getParentFile().exists()) { unzipFile.getParentFile().mkdirs(); } unZip(zipFile,destPath,originalCaseId,cpSerialNum); return bugDetailList; } catch (IOException e) { e.printStackTrace(); } return new ArrayList<>(); } private void bugDetailToFile(List bugDetailList, String caseId) { String[] titles = {"bug_id", "bug_category", "severity", "recurrent", "bug_create_time", "bug_page", "title", "description", "img_url", "score", "parent", "children", "root", "good_num", "good_worker_id", "bad_num", "bad_worker_id", "test_case_id", "test_case_name", "test_case_front", "test_case_behind", "test_case_description", "test_case_create_time", "report_id", "report_name", "report_create_time", "script_location", "report_location", "log_location", "device_model", "device_brand", "device_os", "worker_id", "case_app_name", "case_paper_type", "case_test_type", "case_description", "case_require_doc", "case_take_id", "originalCaseId", "cpSerialNum"}; File csvFile = exportCsv(titles, bugDetailList, caseId); File jsonFile = exportJson(bugDetailList, caseId); // uploadToOss(csvFile); // uploadToOss(jsonFile); } private File exportJson(List bugDetailList, String caseId) { try { File file = new File("data/output/" + caseId + ".json"); JSONArray jsonArray = new JSONArray(bugDetailList); Writer write = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8); write.write(jsonArray.toString()); write.flush(); write.close(); return file; } catch (IOException e) { e.printStackTrace(); } return null; } private File exportCsv(String[] titles, List list, String caseId) { try { File file = new File("data/output/" + caseId + ".csv"); //构建输出流,同时指定编码 OutputStreamWriter ow = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8); //csv文件是逗号分隔,除第一个外,每次写入一个单元格数据后需要输入逗号 for (String title : titles) { ow.write(title); ow.write(","); } //写完文件头后换行 ow.write("\r\n"); //写内容 for (Object obj : list) { //利用反射获取所有字段 Field[] fields = obj.getClass().getDeclaredFields(); for (Field field : fields) { //设置字段可见性 field.setAccessible(true); //防止某个field没有赋值 if (field.get(obj) == null) { ow.write(""); } else { //解决csv文件中对于逗号和双引号的转义问题 ow.write("\"" + field.get(obj).toString().replaceAll("\"", "\"\"") + "\""); } ow.write(","); } //写完一行换行 ow.write("\r\n"); } ow.flush(); ow.close(); return file; } catch (IOException | IllegalAccessException e) { e.printStackTrace(); } return null; } private static void uploadToOss(String objectName,File file) { if (file != null) { OSS ossClient = OssAliyun.initShangHaiOss(); OssAliyun.uploadFile(ossClient, bucketName, objectName, file); } else { System.out.println("file is null"); } } /** * zip解压 * * @param srcFile zip源文件 * @param destDirPath 解压后的目标文件夹 * @throws RuntimeException 解压失败会抛出运行时异常 */ public static void unZip(File srcFile, String destDirPath,String originalCaseId,String fromCpSerialNum) throws RuntimeException { long start = System.currentTimeMillis(); // 判断源文件是否存在 if (!srcFile.exists()) { throw new RuntimeException(srcFile.getPath() + "所指文件不存在"); } // 开始解压 ZipFile zipFile = null; try { zipFile = new ZipFile(srcFile); Enumeration entries = zipFile.entries(); while (entries.hasMoreElements()) { ZipEntry entry = (ZipEntry) entries.nextElement(); System.out.println("解压" + entry.getName()); // 如果是文件夹,就创建个文件夹 if (entry.isDirectory()) { String dirPath = destDirPath + "/" + entry.getName(); File dir = new File(dirPath); dir.mkdirs(); } else { // 如果是文件,就先创建一个文件,然后用io流把内容copy过去 File targetFile = new File(destDirPath + "/" + entry.getName()); // 保证这个文件的父文件夹必须要存在 if (!targetFile.getParentFile().exists()) { targetFile.getParentFile().mkdirs(); } targetFile.createNewFile(); // 将压缩文件内容写入到这个文件中 InputStream is = zipFile.getInputStream(entry); FileOutputStream fos = new FileOutputStream(targetFile); int len; byte[] buf = new byte[BUFFER_SIZE]; while ((len = is.read(buf)) != -1) { fos.write(buf, 0, len); } // 关流顺序,先打开的后关闭 fos.close(); is.close(); //图片文件上传至oss String objectName = "xinchuang/image/"+originalCaseId+"/"+fromCpSerialNum+"/" + targetFile.getName(); uploadToOss(objectName,targetFile); } } long end = System.currentTimeMillis(); System.out.println("解压完成,耗时:" + (end - start) + " ms"); } catch (Exception e) { throw new RuntimeException("unzip error from ZipUtils", e); } finally { if (zipFile != null) { try { zipFile.close(); } catch (IOException e) { e.printStackTrace(); } } } } // public static void main(String[] args){ // File a=new File("/Users/xujiawei/crowd/crowdsource-backend/data/a.zip"); // unZip(a,"/Users/xujiawei/crowd/crowdsource-backend/data/test","1","test"); // } }