const puppeteer = require('puppeteer'); const delay = (ms) => { return new Promise((resolve) => { setTimeout(resolve, ms); }); } const scrapeInstance = async (link, page) => { await link.click(); await page.waitForNavigation(); const totalStudentsElem = await page.$('.instance-student-detail .students'); const totalStudentsHTML = await totalStudentsElem.getProperty('innerHTML'); const totalStudents = parseInt(await totalStudentsHTML.jsonValue()); const droppedElem = await page.$('.instance-student-detail .dropped'); const droppedHTML = await droppedElem.getProperty('innerHTML'); const dropped = parseInt(await droppedHTML.jsonValue()) const outcomes90elem = await page.$('.full-time-90-days-actuals') const outcomes90HTML = await outcomes90elem.getProperty('innerHTML'); const outcomes90 = await outcomes90HTML.jsonValue(); const outcomes90Numeric = parseInt(outcomes90.split(' ')[0]) const outcomes90Percent = outcomes90.split(' ')[1].replace(/[()]/g, '') const instanceID = parseInt(page.url().match(/[0-9]*$/g)[0]) const graduationElem = await page.$('.course-header__detail') const graduationHTML = await graduationElem.getProperty('innerHTML'); const instanceHeaderText = await graduationHTML.jsonValue() const graduationDate = instanceHeaderText.split(' - ')[1].trim(); console.log({ instanceID, graduationDate, totalStudents, dropped, outcomes90Numeric, outcomes90Percent }); return page.goBack(); } (async () => { const browser = await puppeteer.launch(); const page = await browser.newPage(); await page.goto('https://outcomes.generalassemb.ly/'); await page.type('input[type="email"]', 'matt.huntington@generalassemb.ly'); await page.click('button[type="submit"]'); console.log('entered email'); await page.waitForNavigation() await delay(500) //wtf await page.type('input[type="text"]', 'matt.huntington@generalassemb.ly'); await page.type('input[type="password"]', 'Hunt!ngt0n80!'); await page.click('input[type="submit"]'); console.log('entered okta creds'); await page.waitForSelector('div[data-se="okta_verify-push"] a'); await page.click('div[data-se="okta_verify-push"] a'); console.log('selected push notification'); await page.waitForNavigation(); await page.waitForSelector('#from'); console.log('logged in'); await page.type('#from', '01/01/2015'); await page.click('input[value="Filter"]'); console.log('filtering instances'); await page.waitForNavigation(); const instanceLinks = await page.$$('tr td:nth-child(2) a'); for(instance of instanceLinks){ await scrapeInstance(instance, page); } console.log('done'); //await page.screenshot({ path: 'outcomes.png' }) await browser.close(); })();