update fetchContestRatings

Author: SysSn13

models/contest.js

@@ -26,7 +26,11 @@ const ContestRankingsSchema = new Schema({
     endTime: Date,
     contest_id: Number,
     num_user: Number,
-    ratings_fetched: {
+    pages_fetched: {
+        type: Number,
+        default:0,
+    },
+    rankings_fetched: {
         type: Boolean,
         default: false,
     },

services/contests.js

@@ -3,41 +3,56 @@ const fetch = require('node-fetch')
 const Contest = require('../models/contest')
 let halfHour = 1000 * 60 * 30
 
+async function pushRankingsToContest(contest_id, rankings, pagesCnt, islastPage) {
+    try {
+        await Contest.findOneAndUpdate({
+            _id: contest_id
+        }, {
+            $push: {
+                rankings: rankings,
+            },
+            $inc: {
+                pages_fetched: pagesCnt,
+            },
+            $set: {
+                rankings_fetched: islastPage,
+            },
+        });
+
+        return null
+    } catch (err) {
+        console.log(err)
+        return err
+    }
+}
 const fetchContestRankings = async function (contestSlug) {
     try {
-        let contest = await Contest.findById(contestSlug)
+        let contest = await Contest.findById(contestSlug, {
+            rankings: 0
+        })
         if (!contest) {
-            console.error(`Contest ${contestSlug} not found in the db`)
-            return null
+            return Error(`Contest ${contestSlug} not found in the db`)
         }
 
         console.log(`fetching ${contestSlug} ...`)
-        rankings = []
-        let resp = await fetch(`https://leetcode.com/contest/api/ranking/${contestSlug}/?pagination=1&region=global`);
+        let resp = await fetch(`https://leetcode.com/contest/api/ranking/${contestSlug}/?region=global`);
         resp = await resp.json()
-        let contest_id = resp.total_rank[0].contest_id
-        let num_user = resp.user_num
-        let pages = Math.floor(resp.user_num / 25)
-        for (let i = 1; i <= pages; i++) {
+        // let num_user = resp.user_num
+        let pages = Math.floor(resp.user_num / 25),
+            page = contest.pages_fetched + 1
+        for (let i = page; i <= pages; i++) {
             console.log(`Fetching rankings (${contestSlug}): page: ${i}`)
             let res = await fetch(`https://leetcode.com/contest/api/ranking/${contestSlug}/?pagination=${i}&region=global`);
             res = await res.json()
-            for (ranks of res.total_rank) {
+            rankings = res.total_rank.filter(ranks => !(ranks.score == 0 && ranks.finish_time * 1000 == contest.startTime.getTime())).map((ranks) => {
                 let {
                     username,
                     user_slug,
                     country_code,
                     country_name,
                     data_region,
                     rank,
-                    score,
-                    finish_time,
                 } = ranks
-
-                // no submission 
-                if (score === 0 && finish_time * 1000 == contest.startTime.getTime()) {
-                    break // can also break page loop
-                }
                 let ranking = {
                     username,
                     user_slug,
@@ -47,31 +62,21 @@ const fetchContestRankings = async function (contestSlug) {
                     rank
                 }
                 ranking["_id"] = username
-                rankings.push(ranking)
+                return ranking
+            })
+            let err = await pushRankingsToContest(contest._id, rankings, 1, i === pages)
+            if (err) {
+                return err
             }
         }
-
-        let updatedContest = new Contest({
-            contest_id: contest_id,
-            lastUpdated: Date.now(),
-            rankings: rankings,
-            num_user: num_user,
-            ratings_fetched: true
-        })
-
-        contest = await Contest.findByIdAndUpdate(contestSlug, updatedContest, {
-            new: true
-        })
-        console.log(`Updated Rankings in ${contestSlug}`)
-
-        return contest
-    } catch (err) {
-        console.error(err);
+        console.log(`Updated Rankings in ${contestSlug}.`)
         return null
+    } catch (err) {
+        return err
     }
 }
-const fetchContest = async () => {
-
+const fetchContestsMetaData = async () => {
+    console.log("fetching meta data for all contests...")
     try {
         let res = await fetch("https://leetcode.com/graphql", {
             "headers": {
@@ -129,17 +134,20 @@ const fetchContest = async () => {
         return null
     }
 }
-const getContestRankings = async function (contestSlug) {
+const updateContestRankings = async function (contestSlug) {
     let contest = await Contest.findById(contestSlug, {
         rankings: 0
     })
-    if (!contest || !contest.ratings_fetched) {
-        contest = await fetchContestRankings(contestSlug)
+    if (!contest) {
+        return Error(`contest ${contestSlug} not found in the db.`)
+    } else if (!contest.rankings_fetched) {
+        let err = await fetchContestRankings(contestSlug)
+        return err
     }
-    return contest
+    return null
 }
 
 // exports 
-module.exports.fetchContest = fetchContest
-exports.getContestRankings = getContestRankings
+exports.fetchContestsMetaData = fetchContestsMetaData
+exports.updateContestRankings = updateContestRankings
 exports.fetchContestRankings = fetchContestRankings

services/scheduling.js

@@ -1,38 +1,60 @@
-const {getContestRankings , fetchContest} = require('../services/contests')
+const {
+    updateContestRankings,
+    fetchContestsMetaData
+} = require('../services/contests')
 const schedule = require('node-schedule');
 
 //Variables For testing
-const jobScheduleTime = {hour: 00,minute: 00}
-const contestScheduleCall = { start: Date.now(), end: Date.now() + 1000*60*60, rule: '*/5 * * * *' }
+const jobScheduleTime = {
+    hour: 00,
+    minute: 00
+}
+const contestScheduleCall = {
+    start: Date.now(),
+    end: Date.now() + 1000 * 60 * 60,
+    rule: '*/5 * * * *'
+}
 
 
-const job = schedule.scheduleJob({hour: 00, minute: 00,},async function(){
-    let contestList = await fetchContest()
-    for(let i=0;i<contestList.length;i++){
-        if(Date.now() + 1000*60*60*24 > contestList[i].startTime && Date.now()<contestList[i].startTime){
-            const event = schedule.scheduleJob({ start: contestList[todaysContest].startTime, end: contestList[todaysContest].endTime, rule: '*/5 * * * *' }, 
-            async function(){
-                getContestRankings(contestList[i].titleSlug)
-            });
+const job = schedule.scheduleJob({
+    hour: 00,
+    minute: 00,
+}, async function () {
+    let contestList = await fetchContestsMetaData()
+    for (let i = 0; i < contestList.length; i++) {
+        if (Date.now() + 1000 * 60 * 60 * 24 > contestList[i].startTime && Date.now() < contestList[i].startTime) {
+            const event = schedule.scheduleJob({
+                    start: contestList[todaysContest].startTime,
+                    end: contestList[todaysContest].endTime,
+                    rule: '*/5 * * * *'
+                },
+                async function () {
+                    getContestRankings(contestList[i].titleSlug)
+                });
         }
-        let endTime = contestList[i].startTime*1000 + contestList[i].duration*1000
-        if(Date.now()>endTime){
+        let endTime = contestList[i].startTime * 1000 + contestList[i].duration * 1000
+        if (Date.now() > endTime) {
             await getContestRankings(contestList[i].titleSlug)
         }
     }
 });
 
-const fetchNow = async function(){
-    let contestList = await fetchContest()
-    if(!contestList)
+const fetchAllContests = async function () {
+    let contestList = await fetchContestsMetaData()
+    if (!contestList)
         return
-    let promises = contestList.map(async (contest)=>{
-        // console.log(contest)
-        let endTime = contest.startTime*1000 + contest.duration*1000
-        if(Date.now()> endTime){
-            // console.log(endTime)
-            await getContestRankings(contest.titleSlug)
+    let promises = contestList.map(async (contest) => {
+        let endTime = contest.startTime * 1000 + contest.duration * 1000
+        if (Date.now() > endTime) {
+            try {
+                var err = await updateContestRankings(contest.titleSlug)
+                if (err) {
+                    console.error(err)
+                }
+            } catch (err) {
+                console.error(err)
+            }
         }
     })
 }
-exports.fetchAllContests = fetchNow
+exports.fetchAllContests = fetchAllContests