check if instance exists in DB

average
Matthew Huntington 2 years ago
parent 1dda570208
commit b8bbfc7cd3

@ -1,5 +1,5 @@
CREATE TABLE instances (
id integer,
id serial,
instance_id integer,
course character varying(64),
graduation_date character varying(128),

@ -64,7 +64,7 @@ const scrapeInstance = async (link, page) => {
const res = await client.query('SELECT $1::text as message', ['Connected to Postgres'])
console.log(res.rows[0].message)
const browser = await puppeteer.launch();
const browser = await puppeteer.launch({headless:'new'});
const page = await browser.newPage();
await page.goto('https://outcomes.generalassemb.ly/');
await page.type('input[type="email"]', 'matt.huntington@generalassemb.ly');
@ -97,7 +97,14 @@ const scrapeInstance = async (link, page) => {
const instanceLinks = await page.$$('tr td:nth-child(2) a');
for(instance of instanceLinks){
await scrapeInstance(instance, page);
const linkHTML = await instance.getProperty('href');
const href = await linkHTML.jsonValue()
const instanceID = href.match(/[0-9]*$/)[0];
const res = await client.query(`SELECT * FROM instances WHERE instance_id = ${instanceID}`);
if(res.rowCount === 0){
await scrapeInstance(instance, page);
}
}
console.log('done');

Loading…
Cancel
Save