You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
86 lines
2.8 KiB
86 lines
2.8 KiB
const puppeteer = require('puppeteer');
|
|
|
|
const delay = (ms) => {
|
|
return new Promise((resolve) => {
|
|
setTimeout(resolve, ms);
|
|
});
|
|
}
|
|
|
|
const scrapeInstance = async (link, page) => {
|
|
return new Promise(async (resolve)=>{
|
|
console.log('scraping instance');
|
|
await link.click();
|
|
await page.waitForNavigation();
|
|
|
|
const totalStudentsElem = await page.$('.instance-student-detail .students');
|
|
const totalStudentsHTML = await totalStudentsElem.getProperty('innerHTML');
|
|
const totalStudents = parseInt(await totalStudentsHTML.jsonValue());
|
|
|
|
const droppedElem = await page.$('.instance-student-detail .dropped');
|
|
const droppedHTML = await droppedElem.getProperty('innerHTML');
|
|
const dropped = parseInt(await droppedHTML.jsonValue())
|
|
|
|
const outcomes90elem = await page.$('.full-time-90-days-actuals')
|
|
const outcomes90HTML = await outcomes90elem.getProperty('innerHTML');
|
|
const outcomes90 = await outcomes90HTML.jsonValue();
|
|
const outcomes90Numeric = parseInt(outcomes90.split(' ')[0])
|
|
const outcomes90Percent = outcomes90.split(' ')[1].replace(/[()]/g, '')
|
|
|
|
const instanceID = parseInt(page.url().match(/[0-9]*$/g)[0])
|
|
|
|
const graduationElem = await page.$('.course-header__detail')
|
|
const graduationHTML = await graduationElem.getProperty('innerHTML');
|
|
const instanceHeaderText = await graduationHTML.jsonValue()
|
|
const graduationDate = instanceHeaderText.split(' - ')[1].trim();
|
|
|
|
console.log({ instanceID, graduationDate, totalStudents, dropped, outcomes90Numeric, outcomes90Percent });
|
|
|
|
resolve()
|
|
})
|
|
}
|
|
|
|
(async () => {
|
|
const browser = await puppeteer.launch();
|
|
const page = await browser.newPage();
|
|
await page.goto('https://outcomes.generalassemb.ly/');
|
|
await page.type('input[type="email"]', 'matt.huntington@generalassemb.ly');
|
|
await page.click('button[type="submit"]');
|
|
|
|
console.log('entered email');
|
|
await page.waitForNavigation()
|
|
await delay(500) //wtf
|
|
|
|
await page.type('input[type="text"]', 'matt.huntington@generalassemb.ly');
|
|
await page.type('input[type="password"]', 'Hunt!ngt0n80!');
|
|
await page.click('input[type="submit"]');
|
|
|
|
console.log('entered okta creds');
|
|
await page.waitForSelector('div[data-se="okta_verify-push"] a');
|
|
|
|
await page.click('div[data-se="okta_verify-push"] a');
|
|
|
|
console.log('selected push notification');
|
|
await page.waitForNavigation();
|
|
await page.waitForSelector('#from');
|
|
|
|
console.log('logged in');
|
|
await page.type('#from', '01/01/2015');
|
|
await page.click('input[value="Filter"]');
|
|
|
|
console.log('filtering instances');
|
|
await page.waitForNavigation();
|
|
|
|
const instanceLinks = await page.$$('tr td:nth-child(2) a');
|
|
|
|
await scrapeInstance(instanceLinks[0], page);
|
|
//for(elem of elems){
|
|
//const prop = await elem.getProperty('innerHTML');
|
|
//const value = await prop.jsonValue();
|
|
//console.log(value);
|
|
//}
|
|
|
|
console.log('done');
|
|
//await page.screenshot({ path: 'outcomes.png' })
|
|
await browser.close();
|
|
})();
|