check if instance exists in DB

average
Matthew Huntington 2 years ago
parent 1dda570208
commit b8bbfc7cd3

@ -1,5 +1,5 @@
CREATE TABLE instances ( CREATE TABLE instances (
id integer, id serial,
instance_id integer, instance_id integer,
course character varying(64), course character varying(64),
graduation_date character varying(128), graduation_date character varying(128),

@ -64,7 +64,7 @@ const scrapeInstance = async (link, page) => {
const res = await client.query('SELECT $1::text as message', ['Connected to Postgres']) const res = await client.query('SELECT $1::text as message', ['Connected to Postgres'])
console.log(res.rows[0].message) console.log(res.rows[0].message)
const browser = await puppeteer.launch(); const browser = await puppeteer.launch({headless:'new'});
const page = await browser.newPage(); const page = await browser.newPage();
await page.goto('https://outcomes.generalassemb.ly/'); await page.goto('https://outcomes.generalassemb.ly/');
await page.type('input[type="email"]', 'matt.huntington@generalassemb.ly'); await page.type('input[type="email"]', 'matt.huntington@generalassemb.ly');
@ -97,8 +97,15 @@ const scrapeInstance = async (link, page) => {
const instanceLinks = await page.$$('tr td:nth-child(2) a'); const instanceLinks = await page.$$('tr td:nth-child(2) a');
for(instance of instanceLinks){ for(instance of instanceLinks){
const linkHTML = await instance.getProperty('href');
const href = await linkHTML.jsonValue()
const instanceID = href.match(/[0-9]*$/)[0];
const res = await client.query(`SELECT * FROM instances WHERE instance_id = ${instanceID}`);
if(res.rowCount === 0){
await scrapeInstance(instance, page); await scrapeInstance(instance, page);
} }
}
console.log('done'); console.log('done');
//await page.screenshot({ path: 'outcomes.png' }) //await page.screenshot({ path: 'outcomes.png' })

Loading…
Cancel
Save