|
|
|
|
@ -64,7 +64,7 @@ const scrapeInstance = async (link, page) => {
|
|
|
|
|
const res = await client.query('SELECT $1::text as message', ['Connected to Postgres'])
|
|
|
|
|
console.log(res.rows[0].message)
|
|
|
|
|
|
|
|
|
|
const browser = await puppeteer.launch();
|
|
|
|
|
const browser = await puppeteer.launch({headless:'new'});
|
|
|
|
|
const page = await browser.newPage();
|
|
|
|
|
await page.goto('https://outcomes.generalassemb.ly/');
|
|
|
|
|
await page.type('input[type="email"]', 'matt.huntington@generalassemb.ly');
|
|
|
|
|
@ -97,7 +97,14 @@ const scrapeInstance = async (link, page) => {
|
|
|
|
|
const instanceLinks = await page.$$('tr td:nth-child(2) a');
|
|
|
|
|
|
|
|
|
|
for(instance of instanceLinks){
|
|
|
|
|
await scrapeInstance(instance, page);
|
|
|
|
|
|
|
|
|
|
const linkHTML = await instance.getProperty('href');
|
|
|
|
|
const href = await linkHTML.jsonValue()
|
|
|
|
|
const instanceID = href.match(/[0-9]*$/)[0];
|
|
|
|
|
const res = await client.query(`SELECT * FROM instances WHERE instance_id = ${instanceID}`);
|
|
|
|
|
if(res.rowCount === 0){
|
|
|
|
|
await scrapeInstance(instance, page);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
console.log('done');
|
|
|
|
|
|