From b6516ab0f97c406bb281bc58c446d14b8a82e741 Mon Sep 17 00:00:00 2001 From: Anshul Rajput Date: Thu, 23 Sep 2021 20:06:39 +0530 Subject: [PATCH 1/2] Added Interviewbit parser --- package.json | 4 +- src/index.js | 3 ++ src/parsers/interviewbit.js | 93 +++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 src/parsers/interviewbit.js diff --git a/package.json b/package.json index bfde5ee..bed805d 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,9 @@ "version": "1.0.0", "description": "An API listing programming contests across multiple platforms", "main": "index.js", - "scripts": {}, + "scripts": { + "start": "node src/index.js" + }, "repository": { "type": "git", "url": "git+https://github.com/nishanthvijayan/programming-contests-api.git" diff --git a/src/index.js b/src/index.js index 8167ec1..4da251a 100644 --- a/src/index.js +++ b/src/index.js @@ -13,6 +13,8 @@ const atcoder = require('./parsers/atcoder'); const csacademy = require('./parsers/csacademy'); const coj = require('./parsers/coj'); const kaggle = require('./parsers/kaggle'); +const interviewbit = require('./parsers/interviewbit'); + const s3bucket = new AWS.S3({}); @@ -29,6 +31,7 @@ exports.handler = async (event) => { csacademy(), coj(), kaggle(), + interviewbit(), ]) .then((contestsByPlatform) => { const contests = flat(contestsByPlatform.filter(it => Array.isArray(it))); diff --git a/src/parsers/interviewbit.js b/src/parsers/interviewbit.js new file mode 100644 index 0000000..75bf8fa --- /dev/null +++ b/src/parsers/interviewbit.js @@ -0,0 +1,93 @@ +const axios = require('axios'); +const cheerio = require('cheerio'); +const { parserErrorHandler } = require('./../utils'); + +const PLATFORM = 'INTERVIEW BIT'; + + +const interviewbit = () => { + const config = { + timeout: 30000, + headers: { + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,im', + }, + }; + return axios.get('https://www.interviewbit.com/contests/', config) + .then(async (response) => { + var $ = cheerio.load(response.data); + var upcomingContestLinkElementSelector = '#upcoming_contests > div > div > div > div > div.col-xs-12.col-sm-8.col-md-8.card-detail > div > div > a'; + var upcomingContestsLink = []; + $(upcomingContestLinkElementSelector).each((parentIdx, parentElem) => { + var actualLink = "https://www.interviewbit.com".concat($(parentElem).attr('href')); + var name = $(parentElem).text().trim(); + nameAndLink = [actualLink, name]; + upcomingContestsLink.push(nameAndLink); + }) + var upcomingContestStartTimeElementSelector = '#upcoming_contests > div > div > div > div > div.col-xs-12.col-sm-8.col-md-8.card-detail > div > div > div > div.info-value'; + var upcomingContestsStartTime = []; + $(upcomingContestStartTimeElementSelector).each((parentIdx, parentElem) => { + startTime = $(parentElem).text().trim().slice(0, 11); + upcomingContestsStartTime.push(startTime); + }) + var upcomingContestEndTimeAndDuration = [] + for (let i = 0; i < upcomingContestsLink.length; i++) { + var url = upcomingContestsLink[i][0] + var config = { + timeout: 30000, + headers: { + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,im', + }, + }; + + var response = await axios.get(url, config); + var $ = cheerio.load(response.data); + var upcominContestEndTimeElementSelector = '#hackathon-details-div > div:nth-child(2) > span.details-wrapper > span.info-value'; + var upcominContestDurationElementSelector = '#hackathon-details-div > div:nth-child(3) > span.details-wrapper > span.info-value'; + var endTime = $(upcominContestEndTimeElementSelector).first().text().trim().slice(0, 11); + var duration = $(upcominContestDurationElementSelector).first().text().trim(); + var endTimeAndDurationDetails = [endTime, duration]; + upcomingContestEndTimeAndDuration.push(endTimeAndDurationDetails) + } + upcomingList = [] + for (let i = 0; i < upcomingContestsLink.length; i++) { + addDict = {}; + addDict["Name"] = upcomingContestsLink[i][1] + addDict["Platform"] = "Interview Bit" + addDict["challenge_type"] = "Contest" + addDict["url"] = upcomingContestsLink[i][0] + addDict["start_time"] = upcomingContestsStartTime[i] + addDict["end_time"] = upcomingContestEndTimeAndDuration[i][0] + addDict["duration"] = upcomingContestEndTimeAndDuration[i][1] + upcomingList.push(addDict) + } + var activeContestLinkElementSelector = '#active_contests > div > div > div > div > div.col-xs-12.col-sm-8.col-md-8.card-detail > div > div > a'; + var activeContestsLink = []; + $(activeContestLinkElementSelector).each((parentIdx, parentElem) => { + var actualLink = "https://www.interviewbit.com".concat($(parentElem).attr('href')); + var name = $(parentElem).text().trim(); + nameAndLink = [actualLink, name]; + activeContestsLink.push(nameAndLink); + }) + var activeContestEndTimeElementSelector = '#active_contests > div > div > div > div > div.col-xs-12.col-sm-8.col-md-8.card-detail > div > div > div > div.info-value'; + var activeContestsEndTime = []; + $(activeContestEndTimeElementSelector).each((parentIdx, parentElem) => { + EndTime = $(parentElem).text().trim().slice(0, 11); + activeContestsEndTime.push(EndTime); + }) + activeList = [] + for (let i = 0; i < activeContestsLink.length; i++) { + addDict = {}; + addDict["Name"] = activeContestsLink[i][1] + addDict["Platform"] = "Interview Bit" + addDict["challenge_type"] = "Contest" + addDict["url"] = activeContestsLink[i][0] + addDict["end_time"] = activeContestsEndTime[i] + activeList.push(addDict) + } + finalContestList = upcomingList.concat(activeList); + return finalContestList + }) + .catch(parserErrorHandler(PLATFORM)); +}; + +module.exports = interviewbit; From d7a8fb1251c8f7001969c61522948dc598c6f74b Mon Sep 17 00:00:00 2001 From: Anshul Rajput Date: Fri, 24 Sep 2021 20:54:54 +0530 Subject: [PATCH 2/2] Format interviewbit parser result --- src/index.js | 5 --- src/parsers/interviewbit.js | 67 +++++++++++++++++++++++++------------ 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/src/index.js b/src/index.js index 4da251a..08ec75a 100644 --- a/src/index.js +++ b/src/index.js @@ -15,10 +15,8 @@ const coj = require('./parsers/coj'); const kaggle = require('./parsers/kaggle'); const interviewbit = require('./parsers/interviewbit'); - const s3bucket = new AWS.S3({}); - exports.handler = async (event) => { return axios.all([ codeforces(), @@ -62,11 +60,8 @@ exports.handler = async (event) => { ContentType: "application/json;charset=UTF-8", ACL: 'public-read' }; - return s3bucket.upload(params).promise().then((data) => { console.log(`File uploaded successfully at ${data.Location}`) }); }); }; - - diff --git a/src/parsers/interviewbit.js b/src/parsers/interviewbit.js index 75bf8fa..e04d228 100644 --- a/src/parsers/interviewbit.js +++ b/src/parsers/interviewbit.js @@ -26,10 +26,10 @@ const interviewbit = () => { var upcomingContestStartTimeElementSelector = '#upcoming_contests > div > div > div > div > div.col-xs-12.col-sm-8.col-md-8.card-detail > div > div > div > div.info-value'; var upcomingContestsStartTime = []; $(upcomingContestStartTimeElementSelector).each((parentIdx, parentElem) => { - startTime = $(parentElem).text().trim().slice(0, 11); + startTime = $(parentElem).text().trim().slice(0, -4); upcomingContestsStartTime.push(startTime); }) - var upcomingContestEndTimeAndDuration = [] + var upcomingContestEndTime = [] for (let i = 0; i < upcomingContestsLink.length; i++) { var url = upcomingContestsLink[i][0] var config = { @@ -42,23 +42,42 @@ const interviewbit = () => { var response = await axios.get(url, config); var $ = cheerio.load(response.data); var upcominContestEndTimeElementSelector = '#hackathon-details-div > div:nth-child(2) > span.details-wrapper > span.info-value'; - var upcominContestDurationElementSelector = '#hackathon-details-div > div:nth-child(3) > span.details-wrapper > span.info-value'; - var endTime = $(upcominContestEndTimeElementSelector).first().text().trim().slice(0, 11); - var duration = $(upcominContestDurationElementSelector).first().text().trim(); - var endTimeAndDurationDetails = [endTime, duration]; - upcomingContestEndTimeAndDuration.push(endTimeAndDurationDetails) + var endTime = $(upcominContestEndTimeElementSelector).first().text().trim().slice(0, -4); + upcomingContestEndTime.push(endTime) } + + const calcTimeUTC = (datetimeString) => { + var fiveThirtyInSeconds = (5*60 + 30)*60 + function checkNull(val) { + if (val != null){ + return val + }; + } + var datetimeList = datetimeString.split(" ").filter(checkNull); + var year = datetimeList[2]; + var day = datetimeList[0]; + var hour = datetimeList[3].split(":")[0]; + if (datetimeList[4] == "PM"){ + hour = parseInt(hour) + 12 + hour = hour.toString() + }; + var minute = datetimeList[3].split(":")[1]; + var parseMonth = {"Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6, "Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12}; + var month = parseMonth[datetimeList[1]].toString(); + + // Date provided by atcoder follows Tokyo timezone(GMT+09:00) + return new Date(Date.UTC(year, month, day, hour, minute)).getTime() / 1000 - fiveThirtyInSeconds; + }; + upcomingList = [] for (let i = 0; i < upcomingContestsLink.length; i++) { addDict = {}; - addDict["Name"] = upcomingContestsLink[i][1] - addDict["Platform"] = "Interview Bit" - addDict["challenge_type"] = "Contest" - addDict["url"] = upcomingContestsLink[i][0] - addDict["start_time"] = upcomingContestsStartTime[i] - addDict["end_time"] = upcomingContestEndTimeAndDuration[i][0] - addDict["duration"] = upcomingContestEndTimeAndDuration[i][1] - upcomingList.push(addDict) + addDict["name"] = upcomingContestsLink[i][1]; + addDict["platform"] = "Interview Bit"; + addDict["url"] = upcomingContestsLink[i][0]; + addDict["startTime"] = calcTimeUTC(upcomingContestsStartTime[i]); + addDict["endTime"] = calcTimeUTC(upcomingContestEndTime[i]); + upcomingList.push(addDict); } var activeContestLinkElementSelector = '#active_contests > div > div > div > div > div.col-xs-12.col-sm-8.col-md-8.card-detail > div > div > a'; var activeContestsLink = []; @@ -68,6 +87,12 @@ const interviewbit = () => { nameAndLink = [actualLink, name]; activeContestsLink.push(nameAndLink); }) + var activeContestStartTimeElementSelector = '#active_contests > div > div > div > div > div.col-xs-12.col-sm-8.col-md-8.card-detail > div > div > div > div.info-value'; + var activeContestsStartTime = []; + $(activeContestStartTimeElementSelector).each((parentIdx, parentElem) => { + startTime = $(parentElem).text().trim().slice(0, 11); + activeContestsStartTime.push(startTime); + }) var activeContestEndTimeElementSelector = '#active_contests > div > div > div > div > div.col-xs-12.col-sm-8.col-md-8.card-detail > div > div > div > div.info-value'; var activeContestsEndTime = []; $(activeContestEndTimeElementSelector).each((parentIdx, parentElem) => { @@ -77,12 +102,12 @@ const interviewbit = () => { activeList = [] for (let i = 0; i < activeContestsLink.length; i++) { addDict = {}; - addDict["Name"] = activeContestsLink[i][1] - addDict["Platform"] = "Interview Bit" - addDict["challenge_type"] = "Contest" - addDict["url"] = activeContestsLink[i][0] - addDict["end_time"] = activeContestsEndTime[i] - activeList.push(addDict) + addDict["name"] = activeContestsLink[i][1]; + addDict["platform"] = "Interview Bit"; + addDict["url"] = activeContestsLink[i][0]; + addDict["startTime"] = calcTimeUTC(activeContestsStartTime[i]); + addDict["endTime"] = calcTimeUTC(activeContestsEndTime[i]); + activeList.push(addDict); } finalContestList = upcomingList.concat(activeList); return finalContestList