i have react+spring boot application with mongo database, and i have a collection called history which 100M+ records.
from react page when i try to navigate to last page or some pages which are at the bottom it is taking 10 minutes to fetch the entries, which causes performance effect.
is there any way to improve performance.
this is what i tried
Controller class
package com.demo.controller;
import java.util.*;
import com.demo.model.*;
import com.demo.repository.HistoryDao;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.web.bind.annotation.*;
@CrossOrigin(origins = "http://localhost:3000")
@RestController
@RequestMapping("/user/")
public class HistoryController {
private static final Logger log = LogManager.getLogger(HistoryController.class);
private HistoryDao dao;
@Autowired
public HistoryController(HistoryDao dao) {
this.dao = dao;
}
public HistoryController() {
}
@GetMapping("/history")
public List<HistoryRecord> getHistory(@RequestParam(defaultValue = "0") int pageNumber,
@RequestParam(defaultValue = "10") int pageSize){
try {
Pageable page = PageRequest.of(pageNumber, pageSize, Sort.by(Sort.Order.desc("timestamp")););
Query query = new Query();
query.with(page);
List<HistoryRecord> recordsPage = dao.getHistory(query);
return recordsPage;
} catch (Exception de) {
log.error("Exception ",e);
}
return null;
}
}
HistoryDao.java
package com.demo.repository;
import com.demo.model.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Repository;
import java.util.*;
@Repository
public class HistoryDao{
@Autowired
private MongoTemplate mongoTemplate;
private static final Logger log = LogManager.getLogger(HistoryDao.class);
public List<HistoryRecord> getHistory(Query query){
return mongoTemplate.find(query, HistoryRecord.class);
}
}
model class HistoryRecord.java
package com.demo.model;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.data.annotation.Transient;
import org.springframework.data.mongodb.core.index.CompoundIndex;
import org.springframework.data.mongodb.core.index.CompoundIndexes;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.Date;
@Document(collection = "history")
@CompoundIndexes({
@CompoundIndex(name = "timestamp_index", def = "{'timestamp': -1}", unique = false)
})
public class HistoryRecord {
private static final Logger log = LogManager.getLogger(HistoryRecord.class);
private String fromUser;
private String toUser;
private Date timestamp;
public String getFromUser() {
return fromUser;
}
public void setFromUser(String fromUser) {
this.fromUser = fromUser;
}
public String getToUser() {
return toUser;
}
public void setToUser(String toUser) {
this.toUser = toUser;
}
public Date getTimestamp() {
return timestamp;
}
public void setTimestamp() {
try {
this.timestamp = new Date();
} catch (Exception e) {
log.error(e);
}
}
}
collection size: 475876939
tried with query also and it also took around 10 minutes to fetch the entries.
db.history.find({}).sort({ timestamp: -1 }).skip(47587693).limit(10);
note: page number= 47587693, page size= 10;
db.history.find({}).sort({ timestamp: -1 }).skip(47587693).limit(10).explain();
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "demo_db.history",
"indexFilterSet" : false,
"parsedQuery" : {
},
"queryHash" : "17A361F7",
"planCacheKey" : "17A361F7",
"winningPlan" : {
"stage" : "LIMIT",
"limitAmount" : 10,
"inputStage" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "SKIP",
"skipAmount" : 47587693,
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"timestamp" : -1
},
"indexName" : "timestamp_index",
"isMultiKey" : false,
"multiKeyPaths" : {
"timestamp" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"timestamp" : [
"[MaxKey, MinKey]"
]
}
}
}
}
},
"rejectedPlans" : [ ]
},
"serverInfo" : {
"host" : "user-db",
"port" : 27017,
"version" : "4.4.29",
"gitVersion" : "f4dda329a99811c707eb06d05ad023599f9be263"
},
"ok" : 1
}
so when i navigate to last page it taking 10 minutes to fetch entries both from rest request and cli, is there any way to fix the performance issue
thanks