迁移数据到mongodb

pom.xml

@@ -140,6 +140,12 @@
             <artifactId>guava</artifactId>
             <version>19.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-mongodb</artifactId>
+        </dependency>
+
+
 
     </dependencies>
 

src/main/java/com/mooctest/annotation/AutoValue.java

@@ -0,0 +1,18 @@
+package com.mooctest.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * 自定义自增长ID注解
+ *
+ * @author xq
+ *
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.FIELD })
+public @interface AutoValue {
+
+}

src/main/java/com/mooctest/controller/ReportController.java

@@ -94,7 +94,7 @@ public class ReportController {
         boolean bugReviewed = bugReviewService.isBugReviewed(masterId);
         model.addAttribute("reviewed", bugReviewed);
         if (finalReportId != null) {
-            Optional<FinalReportDTO> finalReportDTO = finalReports.stream().filter(finalReport -> finalReport.getId().equals(finalReportId)).findFirst();
+            Optional<FinalReportDTO> finalReportDTO = finalReports.stream().filter(finalReport -> finalReport.getId()==finalReportId).findFirst();
             model.addAttribute("finalReportId", finalReportId);
             model.addAttribute("editReport", finalReportDTO.get());
 

src/main/java/com/mooctest/dao/AggTaskStatusDao.java

@@ -2,11 +2,13 @@ package com.mooctest.dao;
 
 import com.mooctest.model.AggTaskStatus;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.mongodb.repository.MongoRepository;
 
 import java.sql.Timestamp;
+import java.util.Date;
 import java.util.List;
 
-public interface AggTaskStatusDao extends JpaRepository<AggTaskStatus, Long> {
+public interface AggTaskStatusDao extends MongoRepository<AggTaskStatus, Long> {
     List<AggTaskStatus> findByTaskIdOrderByStartTimeDesc(String taskId);
-    AggTaskStatus findByTaskIdAndStartTime(String taskId, Timestamp startTime);
+    AggTaskStatus findByTaskIdAndStartTime(String taskId, Date startTime);
 }

src/main/java/com/mooctest/dao/FinalReportDao.java

@@ -2,12 +2,13 @@ package com.mooctest.dao;
 
 import com.mooctest.model.FinalReport;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
 
 @Transactional
-public interface FinalReportDao extends JpaRepository<FinalReport, Long> {
+public interface FinalReportDao extends MongoRepository<FinalReport, Long> {
 
     List<FinalReport> findByExamIdAndCaseId(long examId, long caseId);
     List<FinalReport> findBySourceId(String sourceId);

src/main/java/com/mooctest/dao/MasterReportDao.java

@@ -2,12 +2,13 @@ package com.mooctest.dao;
 
 import com.mooctest.model.MasterReport;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
 
 @Transactional
-public interface MasterReportDao extends JpaRepository<MasterReport, Long> {
+public interface MasterReportDao extends MongoRepository<MasterReport, Long> {
 
     List<MasterReport> findByMasterId(String masterId);
 

src/main/java/com/mooctest/dao/SupplementDao.java

@@ -3,23 +3,20 @@ package com.mooctest.dao;
 import com.mooctest.model.SupplementItem;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Query;
+import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.data.repository.query.Param;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
 
 @Transactional
-public interface SupplementDao extends JpaRepository<SupplementItem, Long> {
+public interface SupplementDao extends MongoRepository<SupplementItem, Long> {
 
     List<SupplementItem> findByMasterId(String masterId);
 
     List<SupplementItem> findBySupplementId(String supId);
 
-    @Query("SELECT s.supplementId, count(distinct s.bugId) as num FROM SupplementItem s WHERE s.masterId = :masterId GROUP BY s.supplementId")
-    List<Object[]> countBugs(@Param("masterId") String masterId);
-
-    @Query("SELECT DISTINCT s.masterId, s.bugId FROM SupplementItem s WHERE s.masterId IN (:masterIds)")
-    List<Object[]> findAllBugsByMasterIds(@Param("masterIds") List<String> masterIds);
+    List<SupplementItem> findByMasterIdIn(List<String> masterIds);
 
     void deleteByMasterIdIn(List<String> masterIds);
 }

src/main/java/com/mooctest/data/FinalReportDTO.java

@@ -3,11 +3,12 @@ package com.mooctest.data;
 import lombok.Data;
 
 import java.sql.Timestamp;
+import java.util.Date;
 import java.util.List;
 
 @Data
 public class FinalReportDTO {
-    private Long id;
+    private long id;
 
 
     private String description;
@@ -24,7 +25,7 @@ public class FinalReportDTO {
 
     private String sourceId;
 
-    private Timestamp createTime;
+    private Date createTime;
 
     private long examId;
 

src/main/java/com/mooctest/listener/SaveEventListener.java

@@ -0,0 +1,65 @@
+package com.mooctest.listener;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Field;
+
+import com.mooctest.annotation.AutoValue;
+import com.mooctest.model.SequenceId;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.mongodb.core.FindAndModifyOptions;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.mapping.event.AbstractMongoEventListener;
+import org.springframework.data.mongodb.core.mapping.event.BeforeConvertEvent;
+import org.springframework.data.mongodb.core.mapping.event.BeforeSaveEvent;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.stereotype.Component;
+import org.springframework.util.ReflectionUtils;
+
+@Component
+public class SaveEventListener extends AbstractMongoEventListener<Object> {
+    @Autowired
+    private MongoTemplate mongo;
+
+    @Override
+    public void onBeforeConvert(BeforeConvertEvent<Object> event) {
+        Object source = event.getSource();
+        if (source != null) {
+            ReflectionUtils.doWithFields(source.getClass(), new ReflectionUtils.FieldCallback() {
+                public void doWith(Field field) throws IllegalArgumentException, IllegalAccessException {
+                    ReflectionUtils.makeAccessible(field);
+                    // 如果字段添加了我们自定义的AutoValue注解
+                    if (field.isAnnotationPresent(AutoValue.class) && field.get(source) instanceof Number
+                            && field.getLong(source) == 0) {
+                        // field.get(source) instanceof Number &&
+                        // field.getLong(source)==0
+                        // 判断注解的字段是否为number类型且值是否等于0.如果大于0说明有ID不需要生成ID
+                        // 设置自增ID
+
+                        field.set(source, getNextId(event.getCollectionName()));
+
+                    }
+                }
+            });
+        }
+    }
+
+    /**
+     * 获取下一个自增ID
+     *
+     * @param collName
+     *            集合（这里用类名，就唯一性来说最好还是存放长类名）名称
+     * @return 序列值
+     */
+    private Long getNextId(String collName) {
+        Query query = new Query(Criteria.where("collName").is(collName));
+        Update update = new Update();
+        update.inc("seqId", 1);
+        FindAndModifyOptions options = new FindAndModifyOptions();
+        options.upsert(true);
+        options.returnNew(true);
+        SequenceId seq = mongo.findAndModify(query, update, options, SequenceId.class);
+        return seq.getSeqId();
+    }
+}

src/main/java/com/mooctest/model/AggTaskStatus.java

@@ -1,34 +1,31 @@
 package com.mooctest.model;
 
+import com.mooctest.annotation.AutoValue;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.springframework.data.mongodb.core.mapping.Document;
 
 import javax.persistence.*;
-import java.sql.Timestamp;
+import java.util.Date;
 
 @Data
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-@Entity
-@Table(name = "agg_task_status")
+@Document(collection = "AggTaskStatus")
 public class AggTaskStatus {
 
     @Id
-    @GeneratedValue
-    private Long id;
+    @AutoValue
+    private long id;
 
-    @Column(name = "task_id")
     private String taskId;
 
-    @Column(name = "status")
     private int status;
 
-    @Column(name = "start_time")
-    private Timestamp startTime;
+    private Date startTime;
 
-    @Column(name = "end_time")
-    private Timestamp endTime;
+    private Date endTime;
 }

src/main/java/com/mooctest/model/FinalReport.java

@@ -1,51 +1,52 @@
 package com.mooctest.model;
 
+import com.mooctest.annotation.AutoValue;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.springframework.data.mongodb.core.mapping.Document;
 
-import javax.persistence.*;
-import java.sql.Timestamp;
+import javax.persistence.Id;
+import java.util.Date;
 
 @Data
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-@Entity
-@Table(name = "final_report")
+@Document(collection = "FinalReport")//建议名字和类名一致
 public class FinalReport {
     @Id
-    @GeneratedValue
-    private Long id;
+    @AutoValue
+    private long id;
 
-    @Column(name = "description")
+    
     private String description;
 
-    @Column(name = "img_urls")
+    
     private String imgUrls;
 
-    @Column(name = "severity")
+    
     private short severity;
 
-    @Column(name = "recurrent")
+    
     private short recurrent;
 
-    @Column(name = "category")
+    
     private int category;
 
-    @Column(name = "writer_id")
+    
     private long writerId;
 
-    @Column(name = "source_id")
+    
     private String sourceId;
 
-    @Column(name = "create_time")
-    private Timestamp createTime;
+    
+    private Date createTime;
 
-    @Column(name = "exam_id")
+    
     private long examId;
 
-    @Column(name = "case_id")
+    
     private long caseId;
 }

src/main/java/com/mooctest/model/MasterReport.java

@@ -1,9 +1,11 @@
 package com.mooctest.model;
 
+import com.mooctest.annotation.AutoValue;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.springframework.data.mongodb.core.mapping.Document;
 
 import javax.persistence.*;
 
@@ -11,28 +13,21 @@ import javax.persistence.*;
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-@Entity
-@Table(name = "master_report")
+@Document(collection = "MasterReport")
 public class MasterReport {
     @Id
-    @GeneratedValue
-    private Long id;
+    @AutoValue
+    private long id;
 
-    @Column(name = "master_id")
     private String masterId;
 
-    @Column(name = "exam_id")
     private long examId;
 
-    @Column(name = "case_id")
     private long caseId;
 
-    @Column(name = "bug_id")
     private String bugId;
 
-    @Column(name = "status")
     private int status = 0;
 
-    @Column(name = "reviewer_id")
     private Long reviewerId;
 }

src/main/java/com/mooctest/model/SequenceId.java

@@ -0,0 +1,20 @@
+package com.mooctest.model;
+
+import lombok.Data;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.mapping.Document;
+import org.springframework.data.mongodb.core.mapping.Field;
+
+@Data
+@Document(collection = "sysSequence")
+public class SequenceId {
+    @Id
+//    @Field("_id")
+    private String id;
+
+    private long seqId;
+
+    private String collName;
+
+
+}

src/main/java/com/mooctest/model/SupplementItem.java

@@ -1,8 +1,10 @@
 package com.mooctest.model;
 
+import com.mooctest.annotation.AutoValue;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.springframework.data.mongodb.core.mapping.Document;
 
 import javax.persistence.*;
 
@@ -10,31 +12,25 @@ import javax.persistence.*;
 @AllArgsConstructor
 @NoArgsConstructor
 @Entity
-@Table(name = "supplement_item")
+@Document(collection = "SupplementItem")
 public class SupplementItem {
     @Id
-    @GeneratedValue
-    private Long id;
+    @AutoValue
+    private long id;
 
-    @Column(name = "master_id")
+    
     private String masterId;
 
-    @Column(name = "bug_id")
     private String bugId;
 
-    @Column(name = "idx")
     private int index;
 
-    @Column(name = "supplement_id")
     private String supplementId;
 
-    @Column(name = "pr_score")
     private double pageRankScore;
 
-    @Column(name = "content")
     private String content;
 
-    @Column(name = "is_img")
     private boolean isImg;
 
 }

src/main/java/com/mooctest/service/AggTaskStatusService.java

@@ -33,17 +33,17 @@ public class AggTaskStatusService {
         AggTaskStatus status = AggTaskStatus.builder()
                 .taskId(aggTaskId)
                 .status(0)
-                .startTime(new Timestamp(new Date().getTime()))
+                .startTime(new Date())
                 .build();
         return aggTaskStatusDao.save(status);
 
     }
 
-    public AggTaskStatus updateStatus(String taskId, Timestamp startTime) {
+    public AggTaskStatus updateStatus(String taskId, Date startTime) {
         AggTaskStatus aggTaskStatus = aggTaskStatusDao.findByTaskIdAndStartTime(taskId, startTime);
         if (aggTaskStatus != null) {
             aggTaskStatus.setStatus(1);
-            aggTaskStatus.setEndTime(new Timestamp(new Date().getTime()));
+            aggTaskStatus.setEndTime(new Date());
             return aggTaskStatusDao.save(aggTaskStatus);
         } else {
             logger.error("AggTaskStatusService find null, taskId={}, startTime={}", taskId, startTime.getTime());

src/main/java/com/mooctest/service/FinalReportService.java

@@ -23,7 +23,7 @@ public class FinalReportService {
         FinalReport finalReport = new FinalReport();
         BeanUtils.copyProperties(dto, finalReport);
         finalReport.setImgUrls(String.join(",", dto.getImgUrls()));
-        finalReport.setCreateTime(new Timestamp(new Date().getTime()));
+        finalReport.setCreateTime(new Date());
         finalReport = finalReportDao.save(finalReport);
         BeanUtils.copyProperties(finalReport, dto);
         return dto;

src/main/java/com/mooctest/service/SupplementService.java

@@ -21,8 +21,7 @@ import java.io.File;
 import java.util.*;
 import java.util.stream.Collectors;
 
-import static java.util.stream.Collectors.groupingBy;
-import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.*;
 
 @Service
 public class SupplementService {
@@ -41,7 +40,7 @@ public class SupplementService {
                 DirectedWeightedPseudograph graph = buildDirectedGraph(cluster);
                 VertexScoringAlgorithm<String, Double> pr = new PageRank<>(graph, 0.85, 100, 0.0001);
                 cluster.forEach(diffText -> {
-                    SupplementItem supplementItem = new SupplementItem(null,
+                    SupplementItem supplementItem = new SupplementItem(0,
                             masterId,
                             diffText.getBugId(),
                             diffText.getIndex(),
@@ -89,7 +88,7 @@ public class SupplementService {
                 DirectedWeightedPseudograph graph = buildDirectedGraphForDiffImg(cluster);
                 VertexScoringAlgorithm<String, Double> pr = new PageRank<>(graph, 0.85, 100, 0.0001);
                 cluster.forEach(diffImg -> {
-                    SupplementItem supplementItem = new SupplementItem(null,
+                    SupplementItem supplementItem = new SupplementItem(0,
                             masterId,
                             diffImg.getBugId(),
                             diffImg.getIndex(),
@@ -142,13 +141,12 @@ public class SupplementService {
 
     public List<SupplementDTO> getSupplementByMasterId(String masterId, Map<String, BugDTO> bugMap) {
         List<SupplementItem> supplementItems = supplementDao.findByMasterId(masterId);
-
+        List<String> sups = supplementItems.stream().map(SupplementItem::getSupplementId).distinct().collect(toList());
         Map<String, List<SupplementItem>> supplementMap = supplementItems.stream().collect(groupingBy(SupplementItem::getSupplementId));
-        Map<String, Long> supplementBugNumsMap = countBugsForSupplementReverseOrder(masterId);
         List<SupplementDTO> sortedSupplements = new ArrayList<>(supplementMap.size());
 
 
-        supplementBugNumsMap.forEach((supId, num) -> {
+        sups.forEach(supId -> {
             SupplementDTO su = buildSupplement(supId, supplementMap, bugMap);
             sortedSupplements.add(su);
         });
@@ -193,14 +191,7 @@ public class SupplementService {
         return su;
     }
 
-    private Map<String, Long> countBugsForSupplementReverseOrder(String masterId) {
-        List<Object[]> supplementBugNums = supplementDao.countBugs(masterId);
-        Map<String, Long> supplementBugNumsMap = new HashMap<>();
-        supplementBugNums.forEach(record -> {
-            supplementBugNumsMap.put((String) record[0], (Long) record[1]);
-        });
-        return sortByValue(supplementBugNumsMap);
-    }
+
 
     private Map<String, Long> sortByValue(final Map<String, Long> wordCounts) {
         return wordCounts.entrySet()
@@ -285,11 +276,16 @@ public class SupplementService {
     }
 
     public Map<String, List<String>> getMaster2BugIdsMap(List<String> masterIds) {
-        List<Object[]> masterBugs = supplementDao.findAllBugsByMasterIds(masterIds);
+        List<SupplementItem> supItems = supplementDao.findByMasterIdIn(masterIds);
+        List<String[]> masterBugs = supItems.stream()
+                .map(item -> item.getMasterId() + "-" + item.getBugId())
+                .distinct()
+                .map(s -> s.split("-"))
+                .collect(toList());
         Map<String, List<String>> masterBugIdsMap = new HashMap<>();
-        for (Object[] masterBug : masterBugs) {
-            String masterId = (String) masterBug[0];
-            String bugId = (String) masterBug[1];
+        for (String[] masterBug : masterBugs) {
+            String masterId = masterBug[0];
+            String bugId = masterBug[1];
 
             if (masterBugIdsMap.get(masterId) == null) {
                 List<String> bugs = new ArrayList<>();

src/main/resources/application.yml

@@ -39,7 +39,10 @@ spring:
         username: root
         password: qyff2011
     thymeleaf:
-        cache: false # 这个开发配置为false，避免改了模板还要重启服务器
+        cache: false
+    data:
+      mongodb:
+        uri: mongodb://localhost:27017/crowd_review
 server:
     port: 8090
 logging: