cleaning up

scrape.js

@@ -7,36 +7,33 @@ const delay = (ms) => {
 }
 
 const scrapeInstance = async (link, page) => {
-	return new Promise(async (resolve)=>{
+	console.log('scraping instance');
-		console.log('scraping instance');
+	await link.click();
-		await link.click();
+	await page.waitForNavigation();
-		await page.waitForNavigation();
+
-
+	const totalStudentsElem = await page.$('.instance-student-detail .students');
-		const totalStudentsElem = await page.$('.instance-student-detail .students');
+	const totalStudentsHTML = await totalStudentsElem.getProperty('innerHTML');
-		const totalStudentsHTML = await totalStudentsElem.getProperty('innerHTML');
+	const totalStudents = parseInt(await totalStudentsHTML.jsonValue());
-		const totalStudents = parseInt(await totalStudentsHTML.jsonValue());
+
-
+	const droppedElem = await page.$('.instance-student-detail .dropped');
-		const droppedElem = await page.$('.instance-student-detail .dropped');
+	const droppedHTML = await droppedElem.getProperty('innerHTML');
-		const droppedHTML = await droppedElem.getProperty('innerHTML');
+	const dropped = parseInt(await droppedHTML.jsonValue())
-		const dropped = parseInt(await droppedHTML.jsonValue())
+
-
+	const outcomes90elem = await page.$('.full-time-90-days-actuals')
-		const outcomes90elem = await page.$('.full-time-90-days-actuals')
+	const outcomes90HTML = await outcomes90elem.getProperty('innerHTML');
-		const outcomes90HTML = await outcomes90elem.getProperty('innerHTML');
+	const outcomes90 = await outcomes90HTML.jsonValue();
-		const outcomes90 = await outcomes90HTML.jsonValue();
+	const outcomes90Numeric = parseInt(outcomes90.split(' ')[0])
-		const outcomes90Numeric = parseInt(outcomes90.split(' ')[0])
+	const outcomes90Percent = outcomes90.split(' ')[1].replace(/[()]/g, '')
-		const outcomes90Percent = outcomes90.split(' ')[1].replace(/[()]/g, '')
+	
-		
+	const instanceID = parseInt(page.url().match(/[0-9]*$/g)[0])
-		const instanceID = parseInt(page.url().match(/[0-9]*$/g)[0])
+
-
+	const graduationElem = await page.$('.course-header__detail')
-		const graduationElem = await page.$('.course-header__detail')
+	const graduationHTML = await graduationElem.getProperty('innerHTML');
-		const graduationHTML = await graduationElem.getProperty('innerHTML');
+	const instanceHeaderText = await graduationHTML.jsonValue()
-		const instanceHeaderText = await graduationHTML.jsonValue()
+	const graduationDate = instanceHeaderText.split(' - ')[1].trim();
-		const graduationDate = instanceHeaderText.split(' - ')[1].trim();
+
-
+	console.log({ instanceID, graduationDate, totalStudents, dropped, outcomes90Numeric, outcomes90Percent });
-		console.log({ instanceID, graduationDate, totalStudents, dropped, outcomes90Numeric, outcomes90Percent });
+
-
-		resolve()
-	})
 }
 
 (async () => {