|
|
|
@ -7,7 +7,6 @@ const delay = (ms) => {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
const scrapeInstance = async (link, page) => {
|
|
|
|
const scrapeInstance = async (link, page) => {
|
|
|
|
console.log('scraping instance');
|
|
|
|
|
|
|
|
await link.click();
|
|
|
|
await link.click();
|
|
|
|
await page.waitForNavigation();
|
|
|
|
await page.waitForNavigation();
|
|
|
|
|
|
|
|
|
|
|
|
@ -34,6 +33,7 @@ const scrapeInstance = async (link, page) => {
|
|
|
|
|
|
|
|
|
|
|
|
console.log({ instanceID, graduationDate, totalStudents, dropped, outcomes90Numeric, outcomes90Percent });
|
|
|
|
console.log({ instanceID, graduationDate, totalStudents, dropped, outcomes90Numeric, outcomes90Percent });
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return page.goBack();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
(async () => {
|
|
|
|
(async () => {
|
|
|
|
@ -69,12 +69,9 @@ const scrapeInstance = async (link, page) => {
|
|
|
|
|
|
|
|
|
|
|
|
const instanceLinks = await page.$$('tr td:nth-child(2) a');
|
|
|
|
const instanceLinks = await page.$$('tr td:nth-child(2) a');
|
|
|
|
|
|
|
|
|
|
|
|
await scrapeInstance(instanceLinks[0], page);
|
|
|
|
for(instance of instanceLinks){
|
|
|
|
//for(elem of elems){
|
|
|
|
await scrapeInstance(instance, page);
|
|
|
|
//const prop = await elem.getProperty('innerHTML');
|
|
|
|
}
|
|
|
|
//const value = await prop.jsonValue();
|
|
|
|
|
|
|
|
//console.log(value);
|
|
|
|
|
|
|
|
//}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
console.log('done');
|
|
|
|
console.log('done');
|
|
|
|
//await page.screenshot({ path: 'outcomes.png' })
|
|
|
|
//await page.screenshot({ path: 'outcomes.png' })
|
|
|
|
|