diff --git a/scrape.js b/scrape.js index 93ecee1..3b1720c 100644 --- a/scrape.js +++ b/scrape.js @@ -81,36 +81,32 @@ const main = async () => { let elements = await driver.findElements(selection); for(currentElement of elements){ - currentElement.click(); - - await delay(3000); - + await delay(4000); try{ selection = selenium.By.css('.full-time-90-days-actuals'); - let element = await driver.findElement(selection); - let innerHTML = await element.getAttribute('innerHTML'); - const percent = parseInt(innerHTML.match(/([0-9]*%)/)[0].replace('%','')) - - const url = await driver.getCurrentUrl(); - const instanceID = parseInt(url.match(/[0-9]*$/i)[0]); - - selection = selenium.By.css('p.course-header__detail'); - element = await driver.findElement(selection); - innerHTML = await element.getAttribute('innerHTML'); - const graduationDateString = innerHTML.split(' - ')[1].trim(); - //const graduationDate = new Date(graduationDateString); - - if(myInstanceIDs.includes(instanceID)){ - console.log("me -->", instanceID, graduationDateString, percent); - } else { - console.log(instanceID, graduationDateString, percent); + let elements = await driver.findElements(selection); + if(elements.length > 0){ + let element = elements[0] + let innerHTML = await element.getAttribute('innerHTML'); + const percent = parseInt(innerHTML.match(/([0-9]*%)/)[0].replace('%','')) + const url = await driver.getCurrentUrl(); + const instanceID = parseInt(url.match(/[0-9]*$/i)[0]); + selection = selenium.By.css('p.course-header__detail'); + element = await driver.findElement(selection); + innerHTML = await element.getAttribute('innerHTML'); + const graduationDateString = innerHTML.split(' - ')[1].trim(); + //const graduationDate = new Date(graduationDateString); + if(myInstanceIDs.includes(instanceID)){ + console.log("me -->", instanceID, graduationDateString, percent); + } else { + console.log(instanceID, graduationDateString, percent); + } + const res = await client.query(`INSERT INTO instances (instance_id, graduation_date, ninety_day_outcomes) VALUES (${instanceID}, '${graduationDateString}', ${percent})`); } - const res = await client.query(`INSERT INTO instances (instance_id, graduation_date, ninety_day_outcomes) VALUES (${instanceID}, '${graduationDateString}', ${percent})`); - } catch (error) { + //console.log(error); } - await driver.navigate().back(); } }