From a907d4f86040ab27cebdb1b92e95a0a9f003b419 Mon Sep 17 00:00:00 2001 From: Matt Huntington Date: Thu, 12 Oct 2023 12:15:47 -0400 Subject: [PATCH 1/2] close db when done --- scrape.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scrape.js b/scrape.js index 0326fd0..77b28b7 100644 --- a/scrape.js +++ b/scrape.js @@ -124,4 +124,5 @@ const scrapeInstance = async (link, page) => { console.log('done'); //await page.screenshot({ path: 'outcomes.png' }) await browser.close(); + await client.end(); })(); From e3f058d1268a91697be267227d40cbdbd4cc9620 Mon Sep 17 00:00:00 2001 From: Matt Huntington Date: Tue, 7 May 2024 10:04:50 -0400 Subject: [PATCH 2/2] start date added --- scrape.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrape.js b/scrape.js index 77b28b7..7f72c9d 100644 --- a/scrape.js +++ b/scrape.js @@ -50,9 +50,10 @@ const scrapeInstance = async (link, page) => { const graduationHTML = await graduationElem.getProperty('innerHTML'); const instanceHeaderText = await graduationHTML.jsonValue() const graduationDate = instanceHeaderText.split(' - ')[1].trim(); + const startDate = instanceHeaderText.split(' - ')[0].trim().split(':')[1].trim(); - console.log({ instanceID, course, graduationDate, totalStudents, dropped, graduates, seeking, outcomes90Numeric, outcomes90Percent }); - const res = await client.query(`INSERT INTO instances (instance_id, course, graduation_date, total_students, dropped, graduates, seekers, ninety_day_outcomes) VALUES (${instanceID}, '${course}', '${graduationDate}', ${totalStudents}, ${dropped}, ${graduates}, ${seeking}, ${outcomes90Numeric})`); + console.log({ instanceID, course, startDate, graduationDate, totalStudents, dropped, graduates, seeking, outcomes90Numeric, outcomes90Percent }); + const res = await client.query(`INSERT INTO instances (instance_id, course, start_date, graduation_date, total_students, dropped, graduates, seekers, ninety_day_outcomes) VALUES (${instanceID}, '${course}', '${startDate}', '${graduationDate}', ${totalStudents}, ${dropped}, ${graduates}, ${seeking}, ${outcomes90Numeric})`); } else { await page.screenshot({ path: 'outcomes.png' }) } @@ -76,7 +77,7 @@ const scrapeInstance = async (link, page) => { await delay(500) //wtf await page.type('input[type="text"]', 'matt.huntington@generalassemb.ly'); - await page.type('input[type="password"]', 'Hunt!ngt0n80!'); + await page.type('input[type="password"]', 'Hunt!ngt0n!2'); await page.click('input[type="submit"]'); console.log('entered okta creds');