add to db

package-lock.json

@@ -5,6 +5,7 @@
 	"packages": {
 		"": {
 			"dependencies": {
+				"pg": "^8.11.3",
 				"puppeteer": "^21.1.1"
 			}
 		},
@@ -190,6 +191,14 @@
 				"node": "*"
 			}
 		},
+		"node_modules/buffer-writer": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
+			"integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==",
+			"engines": {
+				"node": ">=4"
+			}
+		},
 		"node_modules/callsites": {
 			"version": "3.1.0",
 			"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -693,6 +702,11 @@
 				"node": ">= 14"
 			}
 		},
+		"node_modules/packet-reader": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
+			"integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
+		},
 		"node_modules/parent-module": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -734,6 +748,124 @@
 			"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
 			"integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg=="
 		},
+		"node_modules/pg": {
+			"version": "8.11.3",
+			"resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz",
+			"integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==",
+			"dependencies": {
+				"buffer-writer": "2.0.0",
+				"packet-reader": "1.0.0",
+				"pg-connection-string": "^2.6.2",
+				"pg-pool": "^3.6.1",
+				"pg-protocol": "^1.6.0",
+				"pg-types": "^2.1.0",
+				"pgpass": "1.x"
+			},
+			"engines": {
+				"node": ">= 8.0.0"
+			},
+			"optionalDependencies": {
+				"pg-cloudflare": "^1.1.1"
+			},
+			"peerDependencies": {
+				"pg-native": ">=3.0.1"
+			},
+			"peerDependenciesMeta": {
+				"pg-native": {
+					"optional": true
+				}
+			}
+		},
+		"node_modules/pg-cloudflare": {
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz",
+			"integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==",
+			"optional": true
+		},
+		"node_modules/pg-connection-string": {
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz",
+			"integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA=="
+		},
+		"node_modules/pg-int8": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+			"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+			"engines": {
+				"node": ">=4.0.0"
+			}
+		},
+		"node_modules/pg-pool": {
+			"version": "3.6.1",
+			"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz",
+			"integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==",
+			"peerDependencies": {
+				"pg": ">=8.0"
+			}
+		},
+		"node_modules/pg-protocol": {
+			"version": "1.6.0",
+			"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz",
+			"integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q=="
+		},
+		"node_modules/pg-types": {
+			"version": "2.2.0",
+			"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+			"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+			"dependencies": {
+				"pg-int8": "1.0.1",
+				"postgres-array": "~2.0.0",
+				"postgres-bytea": "~1.0.0",
+				"postgres-date": "~1.0.4",
+				"postgres-interval": "^1.1.0"
+			},
+			"engines": {
+				"node": ">=4"
+			}
+		},
+		"node_modules/pgpass": {
+			"version": "1.0.5",
+			"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+			"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+			"dependencies": {
+				"split2": "^4.1.0"
+			}
+		},
+		"node_modules/postgres-array": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+			"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+			"engines": {
+				"node": ">=4"
+			}
+		},
+		"node_modules/postgres-bytea": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
+			"integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==",
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
+		"node_modules/postgres-date": {
+			"version": "1.0.7",
+			"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+			"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
+		"node_modules/postgres-interval": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+			"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+			"dependencies": {
+				"xtend": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=0.10.0"
+			}
+		},
 		"node_modules/progress": {
 			"version": "2.0.3",
 			"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
@@ -874,6 +1006,14 @@
 				"node": ">=0.10.0"
 			}
 		},
+		"node_modules/split2": {
+			"version": "4.2.0",
+			"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+			"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+			"engines": {
+				"node": ">= 10.x"
+			}
+		},
 		"node_modules/streamx": {
 			"version": "2.15.1",
 			"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz",
@@ -1055,6 +1195,14 @@
 				}
 			}
 		},
+		"node_modules/xtend": {
+			"version": "4.0.2",
+			"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+			"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+			"engines": {
+				"node": ">=0.4"
+			}
+		},
 		"node_modules/y18n": {
 			"version": "5.0.8",
 			"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
@@ -1220,6 +1368,11 @@
 			"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
 			"integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ=="
 		},
+		"buffer-writer": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
+			"integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
+		},
 		"callsites": {
 			"version": "3.1.0",
 			"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
@@ -1582,6 +1735,11 @@
 				"netmask": "^2.0.2"
 			}
 		},
+		"packet-reader": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
+			"integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
+		},
 		"parent-module": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -1611,6 +1769,91 @@
 			"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
 			"integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg=="
 		},
+		"pg": {
+			"version": "8.11.3",
+			"resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz",
+			"integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==",
+			"requires": {
+				"buffer-writer": "2.0.0",
+				"packet-reader": "1.0.0",
+				"pg-cloudflare": "^1.1.1",
+				"pg-connection-string": "^2.6.2",
+				"pg-pool": "^3.6.1",
+				"pg-protocol": "^1.6.0",
+				"pg-types": "^2.1.0",
+				"pgpass": "1.x"
+			}
+		},
+		"pg-cloudflare": {
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz",
+			"integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==",
+			"optional": true
+		},
+		"pg-connection-string": {
+			"version": "2.6.2",
+			"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz",
+			"integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA=="
+		},
+		"pg-int8": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+			"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
+		},
+		"pg-pool": {
+			"version": "3.6.1",
+			"resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz",
+			"integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==",
+			"requires": {}
+		},
+		"pg-protocol": {
+			"version": "1.6.0",
+			"resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz",
+			"integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q=="
+		},
+		"pg-types": {
+			"version": "2.2.0",
+			"resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+			"integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+			"requires": {
+				"pg-int8": "1.0.1",
+				"postgres-array": "~2.0.0",
+				"postgres-bytea": "~1.0.0",
+				"postgres-date": "~1.0.4",
+				"postgres-interval": "^1.1.0"
+			}
+		},
+		"pgpass": {
+			"version": "1.0.5",
+			"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+			"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+			"requires": {
+				"split2": "^4.1.0"
+			}
+		},
+		"postgres-array": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+			"integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="
+		},
+		"postgres-bytea": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
+			"integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w=="
+		},
+		"postgres-date": {
+			"version": "1.0.7",
+			"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+			"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="
+		},
+		"postgres-interval": {
+			"version": "1.2.0",
+			"resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+			"integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+			"requires": {
+				"xtend": "^4.0.0"
+			}
+		},
 		"progress": {
 			"version": "2.0.3",
 			"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
@@ -1720,6 +1963,11 @@
 			"integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
 			"optional": true
 		},
+		"split2": {
+			"version": "4.2.0",
+			"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+			"integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="
+		},
 		"streamx": {
 			"version": "2.15.1",
 			"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz",
@@ -1862,6 +2110,11 @@
 			"integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==",
 			"requires": {}
 		},
+		"xtend": {
+			"version": "4.0.2",
+			"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+			"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
+		},
 		"y18n": {
 			"version": "5.0.8",
 			"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

package.json

@@ -1,5 +1,6 @@
 {
 	"dependencies": {
+		"pg": "^8.11.3",
 		"puppeteer": "^21.1.1"
 	}
 }

scrape.js

@@ -1,4 +1,9 @@
 const puppeteer = require('puppeteer');
+const pg = require('pg');
+
+const client = new pg.Client({
+	database:"outcomes_tracker"
+})
 
 const delay = (ms) => {
 	return new Promise((resolve) => {
@@ -43,6 +48,7 @@ const scrapeInstance = async (link, page) => {
 		const graduationDate = instanceHeaderText.split(' - ')[1].trim();
 
 		console.log({ instanceID, course, graduationDate, totalStudents, dropped, seeking, outcomes90Numeric, outcomes90Percent });
+		const res = await client.query(`INSERT INTO instances (instance_id, course, graduation_date, total_students, dropped, seekers, ninety_day_outcomes) VALUES (${instanceID}, '${course}', '${graduationDate}', ${totalStudents}, ${dropped}, ${seeking}, ${outcomes90Numeric})`);
 
 	}
 
@@ -50,6 +56,10 @@ const scrapeInstance = async (link, page) => {
 }
 
 (async () => {
+	await client.connect();
+	const res = await client.query('SELECT $1::text as message', ['Connected to Postgres'])
+	console.log(res.rows[0].message)
+
 	const browser = await puppeteer.launch();
 	const page = await browser.newPage();
 	await page.goto('https://outcomes.generalassemb.ly/');