├── .prettierignore ├── .gitignore ├── test ├── SouthcoastHealth │ └── sampleNoAvailability.json ├── GreaterLawrenceFHC │ └── noAvailability.html ├── BaystateHealth.js ├── bootstrap.js ├── SouthcoastHealthTest.js ├── PriceChopper.js ├── AtriusTest.js ├── GreaterLawrenceFHCTest.js ├── alerts │ ├── determine_recipients.js │ └── sign_up.js ├── WalgreensDataFormatter.js ├── metrics.js ├── ZocDocTest.js ├── Color.js ├── HeywoodHealthcareTest.js └── dataDefaulter.js ├── .prettierrc.yaml ├── lib ├── stringUtil.js ├── file.js ├── zipRadiusFinder.js ├── twitter.js ├── slack.js ├── db │ └── utils.js └── s3.js ├── site-scrapers ├── FamilyPracticeGroup │ ├── config.js │ └── index.js ├── BaystateHealth │ ├── config.js │ └── index.js ├── UMassAmherst │ ├── config.js │ └── index.js ├── ReverePopup │ ├── config.js │ └── index.js ├── TrinityEMS │ └── config.js ├── MercyMedicalCenter │ ├── config.js │ ├── index.js │ └── fakeMercyResponse.html ├── MAImmunizations │ └── config.js ├── Wegmans │ ├── config.js │ └── index.js ├── index.js ├── Hannaford │ ├── index.js │ └── config.js ├── EastBostonNHC │ └── config.js ├── SouthcoastHealth │ ├── responseParser.js │ ├── config.js │ └── index.js ├── Harrington │ ├── config.js │ └── harrington-notes.txt ├── LowellGeneral │ ├── config.js │ └── index.js ├── StopAndShop │ ├── index.js │ └── config.js └── Walgreens │ ├── config.js │ ├── dataFormatter.js │ └── index.js ├── no-browser-site-scrapers ├── Albertsons │ ├── config.js │ └── index.js ├── WhittierHealth │ └── config.js ├── LynnTech │ ├── config.js │ └── index.js ├── CapeCodHealthcare │ ├── config.js │ └── index.js ├── CVSPharmacy │ ├── config.js │ └── index.js ├── Atrius │ ├── config.js │ └── index.js ├── SouthBostonCHC │ ├── config.js │ └── index.js ├── SouthLawrence │ ├── config.js │ └── index.js ├── BostonMedicalCenter │ ├── config.js │ └── index.js ├── PriceChopper │ ├── config.js │ └── index.js ├── HeywoodHealthcare │ └── config.js ├── PediatricAssociatesOfGreaterSalem │ └── config.js ├── HarborHealth │ ├── config.js │ └── index.js ├── Curative │ ├── config.js │ └── index.js ├── HealthMart │ ├── config.js │ └── index.js ├── index.js ├── UMassMemorial │ ├── config.js │ └── index.js ├── ZocDoc │ ├── index.js │ ├── config.js │ └── zocdocBase.js ├── MassGeneralBrigham │ └── index.js ├── GreaterLawrenceFHC │ └── config.js ├── RevereCataldo │ └── index.js └── Color │ └── index.js ├── scraper_config.js ├── getScraperData.js ├── .eslintrc.json ├── scraper.js ├── samconfig.toml ├── scrapers_no_browser.js ├── LICENSE ├── one-off-scripts ├── 20210423-cancel-reminder.js └── 20210420-send-accidentally-cancelled-text.js ├── .github └── workflows │ ├── nodejs.yaml │ └── prod_deploy.yaml ├── package.json ├── data └── dataDefaulter.js ├── getGeocode.js └── alerts ├── send_texts_and_emails.js ├── determine_recipients.js └── pinpoint └── new_subscriber.js /.prettierignore: -------------------------------------------------------------------------------- 1 | # Ignore all HTML files: 2 | *.html 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | env.json 3 | node_modules/ 4 | proprietary/ 5 | *.zip 6 | lambda/ 7 | out.json 8 | out_no_browser.json 9 | .eslintrc.json 10 | .vscode/ 11 | .DS_Store 12 | .aws-sam/ -------------------------------------------------------------------------------- /test/SouthcoastHealth/sampleNoAvailability.json: -------------------------------------------------------------------------------- 1 | { 2 | "providersInfoById": {}, 3 | "dateToSlots": {}, 4 | "addressByDepartmentId": {}, 5 | "departmentsInfoById": {} 6 | } 7 | -------------------------------------------------------------------------------- /.prettierrc.yaml: -------------------------------------------------------------------------------- 1 | # .prettierrc or .prettierrc.yaml 2 | trailingComma: "es5" 3 | tabWidth: 4 4 | semi: true 5 | singleQuote: false 6 | arrowParens: "always" 7 | endOfLine: "auto" 8 | overrides: 9 | - files: "*.yaml" 10 | options: 11 | tabWidth: 2 -------------------------------------------------------------------------------- /lib/stringUtil.js: -------------------------------------------------------------------------------- 1 | function toTitleCase(string) { 2 | return string.replace(/\w+/g, function (text) { 3 | return text.charAt(0).toUpperCase() + text.substr(1).toLowerCase(); 4 | }); 5 | } 6 | 7 | module.exports = { 8 | toTitleCase, 9 | }; 10 | -------------------------------------------------------------------------------- /site-scrapers/FamilyPracticeGroup/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Family Practice Group", 3 | street: "11 Water Street, Suite 1-A", 4 | city: "Arlington", 5 | zip: "02476", 6 | website: "https://bookfpg.timetap.com/#/", 7 | }; 8 | 9 | module.exports = { 10 | site, 11 | }; 12 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/Albertsons/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Albertsons", 3 | website: "https://www.mhealthappointments.com/covidappt", 4 | nationalStoresJson: 5 | "https://s3-us-west-2.amazonaws.com/mhc.cdn.content/vaccineAvailability.json", 6 | }; 7 | 8 | module.exports = { 9 | site, 10 | }; 11 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/WhittierHealth/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Whittier Rehabilitation Hospital", 3 | street: "145 Ward Hill Ave", 4 | city: "Haverhill", 5 | zip: "01835", 6 | signUpLink: "https://www.whittierhealth.com/covid-vaccine-clinics/", 7 | }; 8 | 9 | module.exports = { 10 | site, 11 | }; 12 | -------------------------------------------------------------------------------- /scraper_config.js: -------------------------------------------------------------------------------- 1 | /* The following scrapers will be skipped */ 2 | const scrapersToSkip = [ 3 | "Atrius", 4 | "FamilyPracticeGroup", 5 | "LowellGeneral", 6 | "ReverePopup", 7 | "SouthLawrence", 8 | "TrinityEMS", 9 | "Walgreens", 10 | "Wegmans", 11 | ]; 12 | 13 | module.exports = { 14 | scrapersToSkip, 15 | }; 16 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/LynnTech/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Lynn Tech High School Field House", 3 | street: "80 Neptune Blvd", 4 | city: "Lynn", 5 | zip: "01902", 6 | signUpLink: 7 | "https://www.lchcnet.org/covid-19-vaccine-scheduling#vaccine-screening", 8 | }; 9 | 10 | module.exports = { 11 | site, 12 | }; 13 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/CapeCodHealthcare/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Hyannis Melody Tent", 3 | street: "21 West Main St", 4 | city: "Hyannis", 5 | zip: "02601", 6 | signUpLink: 7 | "https://mychart-openscheduling.et1149.epichosted.com/MyChart/OpenScheduling", 8 | }; 9 | 10 | module.exports = { 11 | site, 12 | }; 13 | -------------------------------------------------------------------------------- /site-scrapers/BaystateHealth/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Baystate Health", 3 | street: "361 Whitney Ave", 4 | city: "Holyoke", 5 | state: "MA", 6 | zip: "01040", 7 | signUpLink: 8 | "https://workwell.apps.baystatehealth.org/guest/covid-vaccine/register?r=mafirstresp210121", 9 | }; 10 | 11 | module.exports = { 12 | site, 13 | }; 14 | -------------------------------------------------------------------------------- /site-scrapers/UMassAmherst/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "UMass Amherst Campus Center", 3 | street: "1 Campus Center Way", 4 | city: "Amherst", 5 | zip: "01003", 6 | signUpLink: "https://uma.force.com/covidtesting/s/vaccination", 7 | restrictions: "Eligible populations in Western MA", 8 | }; 9 | 10 | module.exports = { 11 | site, 12 | }; 13 | -------------------------------------------------------------------------------- /getScraperData.js: -------------------------------------------------------------------------------- 1 | const scraperData = require("./lib/db/scraper_data"); 2 | exports.handler = async () => { 3 | const data = await scraperData.getAppointmentsForAllLocations(); 4 | 5 | const response = { 6 | statusCode: 200, 7 | headers: { "Access-Control-Allow-Origin": "*" }, 8 | body: JSON.stringify(data), 9 | }; 10 | return response; 11 | }; 12 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/CVSPharmacy/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "CVS", 3 | website: 4 | "https://www.cvs.com/immunizations/covid-19-vaccine?icid=cvs-home-hero1-banner-1-link2-coronavirus-vaccine", 5 | massJson: 6 | "https://www.cvs.com/immunizations/covid-19-vaccine.vaccine-status.ma.json?vaccineinfo", 7 | }; 8 | 9 | module.exports = { 10 | site, 11 | }; 12 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/Atrius/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Atrius Health", 3 | street: "100 Second Avenue", 4 | city: "Needham", 5 | zip: "02494", 6 | website: "https://myhealth.atriushealth.org/fr/", 7 | dphLink: "https://myhealth.atriushealth.org/DPH", 8 | signUpLink: "https://myhealth.atriushealth.org/fr/", 9 | }; 10 | 11 | module.exports = { 12 | site, 13 | }; 14 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true, 5 | "node": true, 6 | "mocha": true 7 | }, 8 | "extends": "eslint:recommended", 9 | "parserOptions": { 10 | "ecmaVersion": 12, 11 | "sourceType": "module" 12 | }, 13 | "rules": { 14 | "no-unused-vars": ["error", { "varsIgnorePattern": "^..." }] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /site-scrapers/ReverePopup/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Revere Board of Health Popup Clinic", 3 | street: "140 American Legion Hwy", 4 | city: "Revere", 5 | zip: "02151", 6 | signUpLink: "https://www.maimmunizations.org//reg/4012583569", 7 | restrictions: "For those who live and work in Revere", 8 | extraData: "Moderna", 9 | }; 10 | 11 | module.exports = { 12 | site, 13 | }; 14 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/SouthBostonCHC/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "South Boston Community Health Center", 3 | street: "409 W Broadway", 4 | city: "South Boston", 5 | zip: "02127", 6 | signUpLink: 7 | "https://forms.office.com/Pages/ResponsePage.aspx?id=J8HP3h4Z8U-yP8ih3jOCukT-1W6NpnVIp4kp5MOEapVUOTNIUVZLODVSMlNSSVc2RlVMQ1o1RjNFUy4u", 8 | }; 9 | 10 | module.exports = { 11 | site, 12 | }; 13 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/SouthLawrence/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "South Lawrence East Elementary School", 3 | signUpLink: "https://lawrencegeneralcovidvaccine.as.me/schedule.php", 4 | website: "https://lawrencegeneralcovidvaccine.as.me/schedule.php", 5 | street: "165 Crawford Street", 6 | city: "Lawrence", 7 | state: "MA", 8 | zip: "01843", 9 | }; 10 | 11 | module.exports = { 12 | site, 13 | }; 14 | -------------------------------------------------------------------------------- /site-scrapers/TrinityEMS/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Trinity EMS (DiBurros Function Facility)", 3 | street: "887 Boston Rd", 4 | city: "Haverhill", 5 | zip: "01835", 6 | signUpLink: 7 | "https://app.acuityscheduling.com/schedule.php?owner=21713854&calendarID=5109380", 8 | signUpLink: 9 | "https://app.acuityscheduling.com/schedule.php?owner=21713854&calendarID=5109380", 10 | }; 11 | 12 | module.exports = { 13 | site, 14 | }; 15 | -------------------------------------------------------------------------------- /site-scrapers/MercyMedicalCenter/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Mercy Medical Center", 3 | street: "299 Carew Street", 4 | city: "Springfield", 5 | state: "MA", 6 | zip: "01902", 7 | signUpLink: "https://apps.sphp.com/THofNECOVIDVaccinations/index.php", 8 | noAppointments: /There are no open appointments on this day\./, 9 | timeslotsUrl: 10 | "https://apps.sphp.com/THofNECOVIDVaccinations/livesearch.php", 11 | }; 12 | 13 | module.exports = { 14 | site, 15 | }; 16 | -------------------------------------------------------------------------------- /lib/file.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | 3 | /** 4 | * 5 | * Read in a file 6 | * 7 | * @param {string} filePath 8 | * @returns {string} 9 | */ 10 | 11 | function read(filePath) { 12 | return fs.readFileSync(filePath); 13 | } 14 | 15 | /** 16 | * 17 | * Write data to a file 18 | * 19 | * @param {string} filePath 20 | * @param {string} data 21 | * @returns {string} 22 | */ 23 | 24 | function write(filePath, data) { 25 | return fs.writeFileSync(filePath, data); 26 | } 27 | 28 | module.exports = { read, write }; 29 | -------------------------------------------------------------------------------- /test/GreaterLawrenceFHC/noAvailability.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | More Times 5 | 6 |
7 |
8 | No times are available in the next month (from April 10, 2021 to May 9, 2021) -------------------------------------------------------------------------------- /site-scrapers/MAImmunizations/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "MAImmunizations", 3 | website: 4 | "https://clinics.maimmunizations.org/appointment/en/clinic/search?location=&search_radius=All&q%5Bvenue_search_name_or_venue_name_i_cont%5D=&clinic_date_eq%5Byear%5D=&clinic_date_eq%5Bmonth%5D=&clinic_date_eq%5Bday%5D=&q%5Bvaccinations_name_i_cont%5D=&commit=Search&page=1#search_results", 5 | baseWebsite: "https://www.maimmunizations.org", 6 | testSignUpLinkWebsite: "https://clinics.maimmunizations.org", 7 | }; 8 | 9 | module.exports = { 10 | site, 11 | }; 12 | -------------------------------------------------------------------------------- /scraper.js: -------------------------------------------------------------------------------- 1 | const dotenv = require("dotenv"); 2 | //note: this only works locally; in Lambda we use environment variables set manually 3 | dotenv.config(); 4 | 5 | const scrapers = require("./site-scrapers"); 6 | const scraperCommon = require("./scraper_common.js"); 7 | 8 | async function executeScrapers() { 9 | return await scraperCommon.execute(true, scrapers); 10 | } 11 | 12 | exports.handler = executeScrapers; 13 | 14 | if (require.main === module) { 15 | (async () => { 16 | console.log("DEV MODE"); 17 | await executeScrapers(); 18 | process.exit(); 19 | })(); 20 | } 21 | -------------------------------------------------------------------------------- /samconfig.toml: -------------------------------------------------------------------------------- 1 | version = 0.1 2 | [default.deploy.parameters] 3 | stack_name = "ma-covid-vaccines-stage" 4 | region = "us-east-1" 5 | confirm_changeset = true 6 | capabilities = "CAPABILITY_IAM" 7 | s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-1qgo7tfzt9818" 8 | s3_prefix = "ma-covid-vaccines-stage" 9 | 10 | [prod.deploy.parameters] 11 | stack_name = "ma-covid-vaccines-prod" 12 | parameter_overrides = "Environment=prod" 13 | region = "us-east-1" 14 | confirm_changeset = true 15 | capabilities = "CAPABILITY_IAM" 16 | s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-1qgo7tfzt9818" 17 | s3_prefix = "ma-covid-vaccines-prod" -------------------------------------------------------------------------------- /no-browser-site-scrapers/BostonMedicalCenter/config.js: -------------------------------------------------------------------------------- 1 | const moment = require("moment"); 2 | 3 | const entityName = "Boston Medical Center"; 4 | 5 | const signUpLink = 6 | "https://mychartscheduling.bmc.org/MyChartscheduling/covid19#/scheduling"; 7 | 8 | /** Site details are aquired from a request to the settingsUrl. Hence, no `site` constant in this config file. */ 9 | const settingsUrl = () => 10 | `https://mychartscheduling.bmc.org/MyChartscheduling/scripts/guest/covid19-screening/custom/settings.js?updateDt=#${moment().unix()}`; 11 | 12 | module.exports = { 13 | entityName, 14 | settingsUrl, 15 | signUpLink, 16 | }; 17 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/PriceChopper/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Price Chopper", 3 | websiteRoot: 4 | "https://scrcxp.pdhi.com/ScreeningEvent/e047c75c-a431-41a8-8383-81613f39dd55/GetLocations", 5 | signUpLink: 6 | "https://pdhi.queue-it.net/?c=pdhi&e=covid19vaccination&t_portalUuid=3e419790-81a3-4639-aa08-6bd223f995df", 7 | zips: [ 8 | "01609", 9 | "02130", 10 | "01748", 11 | "01240", 12 | "01752", 13 | "01201", 14 | "01545", 15 | "01562", 16 | "01590", 17 | "01570", 18 | ], 19 | }; 20 | 21 | module.exports = { 22 | site, 23 | }; 24 | -------------------------------------------------------------------------------- /site-scrapers/Wegmans/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Wegmans", 3 | signUpLink: "https://www.wegmans.com/covid-vaccine-registration/", 4 | }; 5 | 6 | const paths = { 7 | getStartedBtn: "//div[@role='button'][contains(.,'Get Started')]", 8 | massOption: 9 | "//span[contains(@class,'quick_reply')][@role='button'][contains(.,'MA')]", 10 | scheduleBtn: 11 | "//span[contains(@class,'quick_reply')][@role='button'][contains(.,'Schedule')]", 12 | botMessage: "//div[contains(@class,'left_message')]", 13 | noAppointments: 14 | "All available vaccine appointments are reserved at this time", 15 | }; 16 | 17 | module.exports = { 18 | site, 19 | paths, 20 | }; 21 | -------------------------------------------------------------------------------- /scrapers_no_browser.js: -------------------------------------------------------------------------------- 1 | const dotenv = require("dotenv"); 2 | //note: this only works locally; in Lambda we use environment variables set manually 3 | dotenv.config(); 4 | 5 | const noBrowserScrapers = require("./no-browser-site-scrapers"); 6 | const scraperCommon = require("./scraper_common.js"); 7 | 8 | async function executeNoBrowserScrapers() { 9 | return await scraperCommon.execute(false, noBrowserScrapers); 10 | } 11 | 12 | exports.handler = executeNoBrowserScrapers; 13 | 14 | if (process.env.NODE_ENV !== "production" && process.env.NODE_ENV !== "test") { 15 | (async () => { 16 | console.log("DEV MODE"); 17 | await executeNoBrowserScrapers(); 18 | process.exit(); 19 | })(); 20 | } 21 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/HeywoodHealthcare/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | public: { 3 | name: "Heywood Healthcare", 4 | street: "171 Kendall Pond Road W", 5 | city: "Gardner", 6 | state: "MA", 7 | zip: "01440", 8 | signUpLink: "https://gardnervaccinations.as.me/schedule.php", 9 | extraData: { 10 | Note: 11 | "Vaccinations will be at the Polish American Citizens Club (PACC).", 12 | }, 13 | }, 14 | private: { 15 | fetchRequestUrl: 16 | "https://gardnervaccinations.as.me/schedule.php?action=showCalendar&fulldate=1&owner=21588707&template=class", 17 | }, 18 | }; 19 | 20 | module.exports = { 21 | site, 22 | }; 23 | -------------------------------------------------------------------------------- /test/BaystateHealth.js: -------------------------------------------------------------------------------- 1 | const { expect } = require("chai"); 2 | const chai = require("chai"); 3 | chai.use(require("chai-as-promised")); 4 | const baystatehealth = require("./../site-scrapers/BaystateHealth"); 5 | const { site } = require("./../site-scrapers/BaystateHealth/config"); 6 | 7 | describe(`${site.name}`, function () { 8 | it("Test against live site", async function () { 9 | await expect( 10 | baystatehealth(browser).then((res) => res.individualLocationData[0]) 11 | ).to.eventually.have.keys([ 12 | "name", 13 | "street", 14 | "city", 15 | "state", 16 | "zip", 17 | "signUpLink", 18 | "hasAvailability", 19 | ]); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/PediatricAssociatesOfGreaterSalem/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Pediatric Associates of Greater Salem and Beverly", 3 | bearerTokenUrl: "https://framework-backend.scheduling.athena.io/t", 4 | schedulingTokenUrl: 5 | "https://framework-backend.scheduling.athena.io/u?locationId=2804-102&practitionerId=&contextId=2804", 6 | graphQLUrl: "https://framework-backend.scheduling.athena.io/v1/graphql", 7 | locations: [ 8 | { 9 | street: "84 Highland Ave", 10 | city: "Salem", 11 | state: "MA", 12 | zip: "01970", 13 | }, 14 | { 15 | street: "30 Tozer Rd", 16 | city: "Beverly", 17 | state: "MA", 18 | zip: "01915", 19 | }, 20 | ], 21 | }; 22 | 23 | module.exports = { 24 | site, 25 | }; 26 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/HarborHealth/config.js: -------------------------------------------------------------------------------- 1 | const sites = [ 2 | { 3 | siteId: "3210339", 4 | vt: "1089", 5 | dept: "222004005", 6 | name: "Zion Union Church", 7 | street: "805 Attucks Lane", 8 | city: "Hyannis", 9 | zip: "02601", 10 | signUpLink: "https://bit.ly/HHTHVax", 11 | }, 12 | { 13 | siteId: "3210238", 14 | vt: "1089", 15 | dept: "222001009", 16 | name: "Florian Hall", 17 | street: "55 Hallet Street", 18 | city: "Dorchester", 19 | zip: "02124", 20 | signUpLink: "https://bit.ly/HHTFVax", 21 | }, 22 | { 23 | siteId: "3210243", 24 | vt: "1089", 25 | dept: "222003005", 26 | name: "Quincy College", 27 | street: "36 Cordage Park Circle", 28 | city: "Plymouth", 29 | zip: "02360", 30 | signUpLink: "https://bit.ly/HHTPVax", 31 | }, 32 | ]; 33 | 34 | module.exports = { 35 | sites, 36 | }; 37 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/Curative/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Curative", 3 | website: "https://labtools.curativeinc.com/api/v1/testing_sites/", 4 | linkWebsite: "https://curative.com/sites/", 5 | locations: [ 6 | { 7 | id: 24181, 8 | massVax: true, // This is a MassVax site that only allows preregistration 9 | }, 10 | { 11 | id: 24182, 12 | massVax: true, // This is a MassVax site that only allows preregistration 13 | }, 14 | { 15 | id: 25336, 16 | massVax: true, // This is a MassVax site that only allows preregistration 17 | }, 18 | { 19 | id: 28418, // DoubleTree Hotel - Danvers (Public) 20 | }, 21 | { 22 | id: 28417, // Circuit City - Dartmouth (Public) 23 | }, 24 | { 25 | id: 28419, // Eastfield Mall - Springfield (Public 26 | }, 27 | ], 28 | }; 29 | 30 | module.exports = { 31 | site, 32 | }; 33 | -------------------------------------------------------------------------------- /site-scrapers/index.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | 3 | let scrapers = []; 4 | 5 | // args override directory list allowing single site runs, e.g. `node scraper.js MAImmunizations` 6 | if (process.argv.length > 2 && process.env.NODE_ENV !== "test") { 7 | for (let i = 2; i < process.argv.length; i++) { 8 | const scraper = require(`./${process.argv[i]}`); 9 | scrapers.push({ 10 | run: scraper, 11 | name: process.argv[i], 12 | }); 13 | } 14 | } else { 15 | const ls = fs 16 | .readdirSync("./site-scrapers", { withFileTypes: true }) 17 | .filter((item) => item.isDirectory()) 18 | .map((item) => item.name); 19 | 20 | ls.map((fileName) => { 21 | let scraper = require(`./${fileName}`); 22 | scrapers.push({ run: scraper, name: fileName }); 23 | }); 24 | } 25 | 26 | if (process.env.PROPRIETARY_SITE_SCRAPERS_PATH) { 27 | const otherScrapers = require(process.env.PROPRIETARY_SITE_SCRAPERS_PATH); 28 | scrapers.push(...otherScrapers); 29 | } 30 | 31 | module.exports = scrapers; 32 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/HealthMart/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Health Mart Pharmacy", 3 | websiteRoot: 4 | "https://scrcxp.pdhi.com/ScreeningEvent/fed87cd2-f120-48cc-b098-d72668838d8b/GetLocations", 5 | signUpLink: "https://healthmartcovidvaccine.com", 6 | zips: [ 7 | "01845", 8 | "01702", 9 | "02458", 10 | "02459", 11 | "02467", 12 | "01851", 13 | "02119", 14 | "02127", 15 | "02122", 16 | "01826", 17 | "01757", 18 | "02170", 19 | "02035", 20 | "01845", 21 | "01469", 22 | "01501", 23 | "01543", 24 | "01982", 25 | "01944", 26 | "01005", 27 | "02360", 28 | "02747", 29 | "01040", 30 | "01040", 31 | "01060", 32 | "01013", 33 | "01107", 34 | "01370", 35 | "01085", 36 | "02601", 37 | "02568", 38 | "02575", 39 | "02568", 40 | "02554", 41 | ], 42 | }; 43 | 44 | module.exports = { 45 | site, 46 | }; 47 | -------------------------------------------------------------------------------- /site-scrapers/Hannaford/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const rxTouch = require("../../lib/RxTouch.js"); 3 | const moment = require("moment"); 4 | 5 | module.exports = async function GetAvailableAppointments(browser) { 6 | console.log(`${site.name} starting.`); 7 | const webData = await rxTouch.ScrapeRxTouch(browser, site, site.name, 5954); 8 | const individualLocationData = Object.values(webData).map((loc) => { 9 | return { 10 | name: `${site.name}`, 11 | street: loc.street, 12 | city: loc.city, 13 | zip: loc.zip, 14 | hasAvailability: !!Object.keys(loc.availability).length, 15 | extraData: loc.message, 16 | debug: loc.debug, 17 | availability: loc.availability, 18 | signUpLink: site.website, 19 | }; 20 | }); 21 | console.log(`${site.name} done.`); 22 | return { 23 | parentLocationName: "Hannaford", 24 | isChain: true, 25 | timestamp: moment().format(), 26 | individualLocationData, 27 | }; 28 | }; 29 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/index.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | 3 | let scrapers = []; 4 | 5 | // args override directory list allowing single site runs, e.g. `node scrapers_no_browser.js LynnTech` 6 | if (process.argv.length > 2 && process.env.NODE_ENV !== "test") { 7 | for (let i = 2; i < process.argv.length; i++) { 8 | const scraper = require(`./${process.argv[i]}`); 9 | scrapers.push({ 10 | run: scraper, 11 | name: process.argv[i], 12 | }); 13 | } 14 | } else { 15 | const ls = fs 16 | .readdirSync("./no-browser-site-scrapers", { withFileTypes: true }) 17 | .filter((item) => item.isDirectory()) 18 | .map((item) => item.name); 19 | 20 | ls.map((fileName) => { 21 | let scraper = require(`./${fileName}`); 22 | scrapers.push({ run: scraper, name: fileName }); 23 | }); 24 | } 25 | 26 | if (process.env.PROPRIETARY_NO_BROWSER_SITE_SCRAPERS_PATH) { 27 | const otherScrapers = require(process.env 28 | .PROPRIETARY_NO_BROWSER_SITE_SCRAPERS_PATH); 29 | scrapers.push(...otherScrapers); 30 | } 31 | 32 | module.exports = scrapers; 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Olivia Adams and Ora Innovations, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/UMassMemorial/config.js: -------------------------------------------------------------------------------- 1 | const signUpLink = 2 | "https://mychartonline.umassmemorial.org/mychart/openscheduling?specialty=15&hidespecialtysection=1"; 3 | 4 | const sites = [ 5 | { 6 | name: "UMass Memorial Marlborough", 7 | street: "157 Union Street", 8 | city: "Marlborough", 9 | zip: "01752", 10 | departmentID: "111029148", 11 | signUpLink, 12 | }, 13 | { 14 | name: "UMass Memorial Mercantile Center", 15 | street: "201 Commercial St.", 16 | city: "Worcester", 17 | zip: "01608", 18 | departmentID: "111029146", 19 | signUpLink, 20 | }, 21 | { 22 | name: "UMMHC HealthAlliance Clinton Hospital Leominster Campus", 23 | departmentID: "104001144", 24 | signUpLink, 25 | }, 26 | ]; 27 | 28 | const providers = [ 29 | "56394", 30 | "56395", 31 | "56396", 32 | "56475", 33 | "56476", 34 | "56526", 35 | "56527", 36 | "56528", 37 | "56529", 38 | "56530", 39 | "56531", 40 | "56532", 41 | "56533", 42 | ]; 43 | 44 | module.exports = { 45 | sites, 46 | providers, 47 | }; 48 | -------------------------------------------------------------------------------- /site-scrapers/EastBostonNHC/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "East Boston Neighborhood Health Center", 3 | website: 4 | "https://patient.lumahealth.io/survey?patientFormTemplate=601d6aec4f308f00128eb4cd&user=600f45213901d90012deb171", 5 | locations: [ 6 | { 7 | street: "1290 N Shore Road", 8 | city: "Revere", 9 | zip: "02151", 10 | // facility_id: "6011f3c1fa2b92009a1c0e26", // Leaving facility_id in case we want to query by location in the future. 11 | }, 12 | { 13 | street: "318 Broadway", 14 | city: "Chelsea", 15 | zip: "02150", 16 | // facility_id: "601a236ff7f880001333e993", 17 | }, 18 | { 19 | street: "120 Liverpool St", 20 | city: "Boston", 21 | zip: "02128", 22 | // facility_id: "6011f3c1fa2b92009a1c0e24", 23 | }, 24 | { 25 | street: "1601 Washington St", 26 | city: "Boston", 27 | zip: "02118", 28 | // facility_id: "6011f3c1fa2b92009a1c0e2a", 29 | }, 30 | ], 31 | }; 32 | 33 | module.exports = { 34 | site, 35 | }; 36 | -------------------------------------------------------------------------------- /test/bootstrap.js: -------------------------------------------------------------------------------- 1 | const chromium = require("chrome-aws-lambda"); 2 | const { addExtra } = require("puppeteer-extra"); 3 | const puppeteer = addExtra(chromium.puppeteer); 4 | const { expect } = require("chai"); 5 | const _ = require("lodash"); 6 | const globalVariables = _.pick(global, ["browser", "expect"]); 7 | const dotenv = require("dotenv"); 8 | dotenv.config(); 9 | 10 | // expose variables 11 | before(async function () { 12 | global.expect = expect; 13 | global.browser = process.env.DEVELOPMENT 14 | ? await puppeteer.launch({ 15 | executablePath: process.env.CHROMEPATH, 16 | headless: true, 17 | }) 18 | : await puppeteer.launch({ 19 | args: chromium.args, 20 | defaultViewport: chromium.defaultViewport, 21 | executablePath: await chromium.executablePath, 22 | headless: chromium.headless, 23 | ignoreHTTPSErrors: true, 24 | }); 25 | }); 26 | 27 | // close browser and reset global variables 28 | after(async function () { 29 | await browser.close(); 30 | global.browser = globalVariables.browser; 31 | global.expect = globalVariables.expect; 32 | }); 33 | -------------------------------------------------------------------------------- /test/SouthcoastHealthTest.js: -------------------------------------------------------------------------------- 1 | const { 2 | parseJson, 3 | } = require("../site-scrapers/SouthcoastHealth/responseParser"); 4 | const { expect } = require("chai"); 5 | 6 | describe("Southcoast Health :: test availability JSON", () => { 7 | it("should show availability", () => { 8 | const json = require("./SouthcoastHealth/sampleFallRiver.json"); 9 | 10 | const results = parseJson(json); 11 | 12 | expect(Object.keys(results).length).equals(3); 13 | 14 | // Count the number times the key "time" occurs in the JSON file. 15 | // That should equal the number of appointments (slots). 16 | const timesStrCount = JSON.stringify(json).match(/time/g).length; 17 | const totalSlots = Object.values(results).reduce((acc, value) => { 18 | acc += value.numberAvailableAppointments; 19 | return acc; 20 | }, 0); 21 | 22 | expect(totalSlots).equals(timesStrCount); 23 | }); 24 | 25 | it("should show no availability", () => { 26 | const json = require("./SouthcoastHealth/sampleNoAvailability.json"); 27 | 28 | const results = parseJson(json); 29 | expect(results).deep.equals({}); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /lib/zipRadiusFinder.js: -------------------------------------------------------------------------------- 1 | const maZips = require("../data/ma-zips.json"); 2 | 3 | // unit is miles 4 | module.exports = function zipRadiusFinder({ 5 | greaterThan, 6 | lessThan, 7 | originLoc, 8 | }) { 9 | const resultZips = []; 10 | if (!(greaterThan || lessThan)) { 11 | throw new Error("one of greaterThan/lessThan must be set (or both)."); 12 | } 13 | for (const [zip, loc] of Object.entries(maZips)) { 14 | const distance = getDistance( 15 | originLoc.latitude, 16 | originLoc.longitude, 17 | loc.latitude, 18 | loc.longitude 19 | ); 20 | if ( 21 | (!greaterThan || distance >= greaterThan) && 22 | (!lessThan || distance < lessThan) 23 | ) { 24 | resultZips.push(zip); 25 | } 26 | } 27 | return resultZips; 28 | }; 29 | 30 | /* from https://stackoverflow.com/a/21623206 */ 31 | function getDistance(lat1, lon1, lat2, lon2) { 32 | const p = 0.017453292519943295; // Math.PI / 180 33 | const c = Math.cos; 34 | const a = 35 | 0.5 - 36 | c((lat2 - lat1) * p) / 2 + 37 | (c(lat1 * p) * c(lat2 * p) * (1 - c((lon2 - lon1) * p))) / 2; 38 | 39 | return 7917.6 * Math.asin(Math.sqrt(a)); // 2 * R; R = 3958.8 mi 40 | } 41 | -------------------------------------------------------------------------------- /site-scrapers/SouthcoastHealth/responseParser.js: -------------------------------------------------------------------------------- 1 | /* 2 | Functionality here is unit testable. 3 | */ 4 | /** 5 | * Sample data format: 6 | {... 7 | "dateToSlots": { 8 | "2021-05-08": {}, 9 | "2021-05-09": {}, 10 | "2021-05-10": { 11 | "e01e1f56-0029-4256-be54-a8f2a33d6011": { 12 | "slots": [...] 13 | } 14 | }, ... 15 | * 16 | * @param {JSON} json 17 | * @returns if no slots -> {}; otherwise { MM-DD-YYYY: { numberAvailableAppointments: x, hasAvailability: y}, ... } 18 | */ 19 | function parseJson(json) { 20 | const dateToSlots = json.dateToSlots; 21 | 22 | if (!dateToSlots) { 23 | return {}; 24 | } 25 | const availabilityContainer = {}; 26 | // Otherwise, get dates with slots: keys are dates 27 | for (const [date, slotsObj] of Object.entries(dateToSlots)) { 28 | const slots = Object.values(slotsObj); 29 | 30 | if (slots.length > 0) { 31 | availabilityContainer[date] = { 32 | numberAvailableAppointments: slots[0].slots.length, 33 | hasAvailability: true, 34 | }; 35 | } 36 | } 37 | 38 | return availabilityContainer; 39 | } 40 | 41 | module.exports = { 42 | parseJson, 43 | }; 44 | -------------------------------------------------------------------------------- /one-off-scripts/20210423-cancel-reminder.js: -------------------------------------------------------------------------------- 1 | const dbUtils = require("../lib/db/utils"); 2 | const faunadb = require("faunadb"), 3 | fq = faunadb.query; 4 | const { sendTexts } = require("../alerts/send_texts_and_emails"); 5 | 6 | module.exports = run; 7 | 8 | async function run() { 9 | // find people who were marked cancelled on/before 9 AM this morning 10 | // but were non-existent OR were not marked cancelled on/before 6 PM yesterday 11 | const allSubscribers = await dbUtils 12 | .faunaQuery( 13 | fq.Map( 14 | fq.Paginate( 15 | fq.Match("subscriptionsByStatuses", [true, false]), 16 | { 17 | size: 5000, //10000, 18 | } 19 | ), 20 | fq.Lambda( 21 | "sub", 22 | fq.Select(["data", "phoneNumber"], fq.Get(fq.Var("sub"))) 23 | ) 24 | ) 25 | ) 26 | .then((res) => res.data) 27 | .catch(console.error); 28 | console.log(`found ${allSubscribers.length} subscribers.`); 29 | const msg = 30 | "Happy Friday! This is a gentle reminder to unsubscribe by texting STOP if you " + 31 | "no longer need our notifications. Thank you!"; 32 | await sendTexts(allSubscribers, msg); 33 | } 34 | -------------------------------------------------------------------------------- /lib/twitter.js: -------------------------------------------------------------------------------- 1 | const Twitter = require("twit"); 2 | const dotenv = require("dotenv"); 3 | dotenv.config(); 4 | let client; 5 | 6 | const USE_TWITTER = true; 7 | 8 | if (process.env.NODE_ENV === "production" && USE_TWITTER) { 9 | client = new Twitter({ 10 | consumer_key: process.env.TWITTER_API_KEY, 11 | consumer_secret: process.env.TWITTER_API_SECRET, 12 | access_token: process.env.TWITTER_ACCESS_TOKEN_KEY, 13 | access_token_secret: process.env.TWITTER_ACCESS_TOKEN_SECRET, 14 | }); 15 | } else { 16 | console.log( 17 | `SETTING UP FAKE TWITTER CLIENT - If you intended this to be real, set NODE_ENV to "production" instead of ${process.env.NODE_ENV}.` 18 | ); 19 | client = { 20 | post: (method, args) => { 21 | console.log( 22 | `Twitter client was called with ${JSON.stringify([ 23 | method, 24 | args, 25 | ])}, but it is set up as a test object so it did nothing.` 26 | ); 27 | }, 28 | }; 29 | } 30 | 31 | async function sendTweet(message) { 32 | if (message.length > 280) { 33 | throw new Error("Tweet too long! It needs to be under 280 characters."); 34 | } 35 | return client.post("statuses/update", { status: message }); 36 | } 37 | 38 | module.exports = { sendTweet }; 39 | -------------------------------------------------------------------------------- /site-scrapers/Harrington/config.js: -------------------------------------------------------------------------------- 1 | const townRestricted = { 2 | name: "Harrington Healthcare (Local Residents)", 3 | street: "153 Chestnut Street", 4 | city: "Southbridge", 5 | zip: "01550", 6 | extraData: 7 | "All vaccinations are performed at the Southbridge Community Center at 153 Chestnut St. Southbridge, MA 01550.", 8 | restrictions: 9 | "Registration is open to only those who live in the following towns: Auburn, Brimfield, Brookfield, (North, East, West) Charlton, Dudley, Holland, Leicester, Oxford, Southbridge, Spencer, Sturbridge, Sutton, Uxbridge, Wales, Warren, Webster", 10 | signUpLink: 11 | "https://app.acuityscheduling.com/schedule.php?owner=22192301&calendarID=5202050", 12 | }; 13 | const unRestricted = { 14 | name: "Harrington Healthcare (All Eligible Residents)", 15 | street: "153 Chestnut Street", 16 | city: "Southbridge", 17 | zip: "01550", 18 | extraData: 19 | "All vaccinations are performed at the Southbridge Community Center at 153 Chestnut St. Southbridge, MA 01550.", 20 | signUpLink: 21 | "https://app.acuityscheduling.com/schedule.php?owner=22192301&calendarID=5202038", 22 | }; 23 | 24 | const monthCount = 2; 25 | 26 | module.exports = { 27 | entity: "Harrington Healthcare", 28 | townRestricted, 29 | unRestricted, 30 | monthCount, 31 | }; 32 | -------------------------------------------------------------------------------- /site-scrapers/SouthcoastHealth/config.js: -------------------------------------------------------------------------------- 1 | const entityName = "Southcoast Health"; 2 | 3 | const sites = [ 4 | { 5 | name: "Southcoast Health (Fall River)", 6 | street: "20 Star Street", 7 | city: "Fall River", 8 | zip: "02724", 9 | signUpLink: 10 | "https://www.southcoast.org/covid-19-vaccine-scheduling/#/slot-search/covid/resource-type/98C5A8BE-25D1-4125-9AD5-1EE64AD164D2", 11 | }, 12 | { 13 | name: "Southcoast Health (North Dartmouth)", 14 | street: "375 Faunce Corner Road", 15 | city: "North Dartmouth", 16 | zip: "02747", 17 | signUpLink: 18 | "https://www.southcoast.org/covid-19-vaccine-scheduling/#/slot-search/covid/resource-type/98C5A8BE-25D1-4125-9AD5-1EE64AD164D2", 19 | }, 20 | { 21 | name: "Southcoast Health (Wareham)", 22 | street: "48 Marion Road", 23 | city: "Wareham", 24 | zip: "02571", 25 | signUpLink: 26 | "https://www.southcoast.org/covid-19-vaccine-scheduling/#/slot-search/covid/resource-type/98C5A8BE-25D1-4125-9AD5-1EE64AD164D2", 27 | }, 28 | ]; 29 | 30 | const siteUrl = 31 | "https://www.southcoast.org/covid-19-vaccine-scheduling/#/slot-search/covid/resource-type/98C5A8BE-25D1-4125-9AD5-1EE64AD164D2"; 32 | module.exports = { 33 | entityName, 34 | sites, 35 | siteUrl, 36 | }; 37 | -------------------------------------------------------------------------------- /site-scrapers/LowellGeneral/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Lowell General", 3 | signUpLink: "https://www.lowellgeneralvaccine.com/#schedule", 4 | startUrl: 5 | "https://www.lowellgeneralvaccine.com/schedule.html?wpforms%5Bfields%5D%5B2%5D=Massachusetts+residents+over+the+age+of+75+or+those+75%2B+with+a+Circle+Health+affiliated+PCP&wpforms%5Bfields%5D%5B5%5D%5B%5D=I+hereby+attest+under+penalty+of+perjury+and+to+the+best+of+my+knowledge+and+belief+that+I+%28or+the+individual+I+am+scheduling%29+belong+to+one+of+the+Massachusetts+Vaccination+Priority+Group+selected+above&wpforms%5Bfields%5D%5B6%5D%5B%5D=I+hereby+attest+under+penalty+of+perjury+that+I%2Fthey+reside%2C+work%2C+and%2For+study+in+the+State+of+Massachusetts&wpforms%5Bfields%5D%5B7%5D%5B%5D=I+understand+that+at+the+vaccination+site+I+may+be+asked+to+provide+proof+of+the+above.+Such+proof+can+include%3A+government+issued+ID%2C+work+badge%2C+employment+letter+or+current+pay+stub&wpforms%5Bid%5D=6400&wpforms%5Bauthor%5D=1&wpforms%5Bpost_id%5D=6037&wpforms%5Btoken%5D=dca7ffc8c0e098ddf465f4dc68841c1d", 6 | street: "Cross River Center - East 1001 Pawtucket Blvd", 7 | city: "Lowell", 8 | state: "MA", 9 | zip: "01854", 10 | noAppointments: /Vaccine appointments are full at this time due to high demand and our scheduling tool is now offline./, 11 | }; 12 | 13 | module.exports = { 14 | site, 15 | }; 16 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/ZocDoc/index.js: -------------------------------------------------------------------------------- 1 | const { scraperName, sites } = require("./config"); 2 | const helper = require("./zocdocBase"); 3 | const moment = require("moment"); 4 | 5 | module.exports = async function GetAvailableAppointments( 6 | _ignored, 7 | fetchService = liveFetchService() 8 | ) { 9 | console.log(`${scraperName} starting.`); 10 | const webData = await ScrapeWebsiteData(fetchService); 11 | console.log(`${scraperName} done.`); 12 | return { 13 | parentLocationName: "ZocDoc", 14 | timestamp: moment().format(), 15 | individualLocationData: webData, 16 | }; 17 | }; 18 | 19 | function liveFetchService() { 20 | return { 21 | async fetchAvailability() { 22 | return await helper.fetchAvailability(); 23 | }, 24 | }; 25 | } 26 | 27 | async function ScrapeWebsiteData(fetchService) { 28 | // Initialize results to no availability 29 | const results = []; 30 | 31 | const fetchedAvailability = await fetchService.fetchAvailability(); 32 | const allAvailability = helper.parseAvailability(fetchedAvailability); 33 | 34 | Object.entries(allAvailability).forEach((value) => { 35 | results.push({ 36 | ...sites[value[0]], 37 | ...value[1], 38 | hasAvailability: Object.keys(value[1].availability).length > 0, 39 | }); 40 | }); 41 | 42 | return results; 43 | } 44 | -------------------------------------------------------------------------------- /site-scrapers/StopAndShop/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const rxTouch = require("../../lib/RxTouch.js"); 3 | const moment = require("moment"); 4 | 5 | function TitleCase(str) { 6 | if (!str) return str; 7 | 8 | return str 9 | .trim() 10 | .toLowerCase() 11 | .split(" ") 12 | .map(function (word) { 13 | return !word ? word : word.replace(word[0], word[0].toUpperCase()); 14 | }) 15 | .join(" "); 16 | } 17 | 18 | module.exports = async function GetAvailableAppointments(browser) { 19 | console.log(`${site.name} starting.`); 20 | const webData = await rxTouch.ScrapeRxTouch( 21 | browser, 22 | site, 23 | "StopAndShop", 24 | 5957 25 | ); 26 | 27 | const individualLocationData = Object.values(webData).map((loc) => { 28 | return { 29 | name: `Stop & Shop`, 30 | street: TitleCase(loc.street), 31 | city: TitleCase(loc.city), 32 | zip: loc.zip, 33 | hasAvailability: !!Object.keys(loc.availability).length, 34 | extraData: loc.message, 35 | debug: loc.debug, 36 | availability: loc.availability, 37 | signUpLink: site.website, 38 | }; 39 | }); 40 | console.log(`${site.name} done.`); 41 | return { 42 | parentLocationName: "Stop & Shop", 43 | timestamp: moment().format(), 44 | isChain: true, 45 | individualLocationData, 46 | }; 47 | }; 48 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/LynnTech/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const mychart = require("../../lib/MyChartAPI"); 3 | const moment = require("moment"); 4 | 5 | const siteId = "13300632"; 6 | const vt = "1089"; 7 | const dept = "133001025"; 8 | 9 | module.exports = async function GetAvailableAppointments() { 10 | console.log(`${site.name} starting.`); 11 | const webData = await ScrapeWebsiteData(); 12 | console.log(`${site.name} done.`); 13 | return { 14 | parentLocationName: "Lynn Tech", 15 | timestamp: moment().format(), 16 | individualLocationData: [ 17 | { 18 | ...site, 19 | ...webData[dept], 20 | }, 21 | ], 22 | }; 23 | }; 24 | 25 | async function ScrapeWebsiteData() { 26 | // We need to go through the flow and use a request verification token 27 | const [ 28 | cookie, 29 | verificationToken, 30 | ] = await mychart.GetCookieAndVerificationToken( 31 | `https://mychartos.ochin.org/mychart/SignupAndSchedule/EmbeddedSchedule?id=${siteId}&vt=${vt}&dept=${dept}&view=plain&public=1&payor=-1,-2,-3,4655,4660,1292,4661,5369,5257,1624,4883&lang=english1089` 32 | ); 33 | 34 | return mychart.AddFutureWeeks( 35 | "mychartos.ochin.org", 36 | "/mychart/OpenScheduling/OpenScheduling/GetOpeningsForProvider", 37 | cookie, 38 | verificationToken, 39 | 10, 40 | mychart.CommonPostDataCallback([siteId], [dept], vt) 41 | ); 42 | } 43 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/SouthBostonCHC/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const mychart = require("../../lib/MyChartAPI"); 3 | const moment = require("moment"); 4 | 5 | const siteId = "1900119"; 6 | const vt = "1089"; 7 | const dept = "150001007"; 8 | 9 | module.exports = async function GetAvailableAppointments() { 10 | console.log(`${site.name} starting.`); 11 | const webData = await ScrapeWebsiteData(); 12 | console.log(`${site.name} done.`); 13 | return { 14 | parentLocationName: "South Boston Community Health Center", 15 | timestamp: moment().format(), 16 | individualLocationData: [ 17 | { 18 | ...site, 19 | ...webData[dept], 20 | }, 21 | ], 22 | }; 23 | }; 24 | 25 | async function ScrapeWebsiteData() { 26 | // We need to go through the flow and use a request verification token 27 | const [ 28 | cookie, 29 | verificationToken, 30 | ] = await mychart.GetCookieAndVerificationToken( 31 | `https://mychartos.ochin.org/mychart/SignupAndSchedule/EmbeddedSchedule?id=${siteId}&dept=${dept}&vt=${vt}&payor=-1,-2,-3,4653,1624,4660,4655,1292,4881,5543,5979,2209,5257,1026,1001,2998,3360,3502,4896,2731` 32 | ); 33 | 34 | return mychart.AddFutureWeeks( 35 | "mychartos.ochin.org", 36 | "/mychart/OpenScheduling/OpenScheduling/GetOpeningsForProvider", 37 | cookie, 38 | verificationToken, 39 | 10, 40 | mychart.CommonPostDataCallback([siteId], [dept], vt) 41 | ); 42 | } 43 | -------------------------------------------------------------------------------- /site-scrapers/UMassAmherst/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const moment = require("moment"); 3 | 4 | module.exports = async function GetAvailableAppointments(browser) { 5 | console.log(`${site.name} starting.`); 6 | const info = await ScrapeWebsiteData(browser); 7 | console.log(`${site.name} done.`); 8 | return { 9 | parentLocationName: "UMass Amherst", 10 | timestamp: moment().format(), 11 | individualLocationData: [ 12 | { 13 | ...site, 14 | ...info, 15 | }, 16 | ], 17 | }; 18 | }; 19 | 20 | async function ScrapeWebsiteData(browser) { 21 | const page = await browser.newPage(); 22 | await page.goto(site.signUpLink); 23 | // evidently the loading spinner doesn't always show up? so we'll let these silently fail if they time out. 24 | await page 25 | .waitForSelector(".loadingSpinner", { visible: true }) 26 | .catch(() => {}); 27 | await page 28 | .waitForSelector(".loadingSpinner", { hidden: true }) 29 | .catch(() => {}); 30 | // Wait for the buttons to show up 31 | await page.waitForSelector(".slds-button").catch(() => {}); 32 | 33 | const content = await page.content(); 34 | 35 | const result = { 36 | hasAvailability: 37 | content.indexOf( 38 | "Sorry there are no time slots available at the moment to book first and second dose appointments, please check back later." 39 | ) == -1, 40 | }; 41 | 42 | page.close(); 43 | return result; 44 | } 45 | -------------------------------------------------------------------------------- /test/PriceChopper.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const nock = require("nock"); 3 | const mock = require("mock-require"); 4 | const chai = require("chai"); 5 | chai.use(require("chai-subset")); 6 | const expect = chai.expect; 7 | 8 | describe("PriceChopper GetAvailabilities", () => { 9 | it("should return availabilities when there are availabilities", async () => { 10 | const scraper = require("../no-browser-site-scrapers/PriceChopper"); 11 | 12 | const apiResponse = [ 13 | { 14 | address1: "555 HUBBARD AVE", 15 | zipCode: "12345", 16 | city: "PITTSFIELD", 17 | visibleTimeSlots: [{ time: "2021-03-05T10:00Z" }], 18 | }, 19 | ]; 20 | 21 | const resultingAvailability = [ 22 | { 23 | hasAvailability: true, 24 | availability: { 25 | "3/5/2021": { 26 | hasAvailability: true, 27 | numberAvailableAppointments: 1, 28 | }, 29 | }, 30 | }, 31 | ]; 32 | 33 | nock("https://scrcxp.pdhi.com") 34 | .persist() 35 | .get(/ScreeningEvent/) 36 | .reply(200, JSON.stringify(apiResponse)); 37 | 38 | // run the test and assert that the result containss availability: 39 | const result = await scraper(); 40 | return expect(result.individualLocationData).to.containSubset( 41 | resultingAvailability 42 | ); 43 | }).timeout(60000); 44 | }); 45 | -------------------------------------------------------------------------------- /.github/workflows/nodejs.yaml: -------------------------------------------------------------------------------- 1 | name: Workflow 1 - Run Test Suite 2 | on: 3 | pull_request: 4 | branches: 5 | - master 6 | push: 7 | branches: 8 | - master 9 | jobs: 10 | build: 11 | runs-on: ubuntu-18.04 12 | strategy: 13 | matrix: 14 | node-version: [14.x] 15 | steps: 16 | - name: Install chromium browser 17 | run: | 18 | sudo apt-get -q -y update 19 | sudo apt-get -q -y install chromium-browser 20 | - name: Set timezone to America/New_York 21 | run: sudo timedatectl set-timezone America/New_York 22 | - uses: actions/checkout@v2 23 | - name: Use Node.js ${{ matrix.node-version }} 24 | uses: actions/setup-node@v1 25 | with: 26 | node-version: ${{ matrix.node-version }} 27 | - run: npm ci 28 | - name: Run test suite 29 | env: 30 | DEVELOPMENT: true 31 | CHROMEPATH: /usr/bin/chromium-browser 32 | FAUNA_DB: ${{ secrets.FAUNA_DB_DEV }} 33 | GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} 34 | run: | 35 | if [ -z "${{ secrets.FAUNA_DB_DEV }}" ] || [ -z "${{ secrets.GOOGLE_API_KEY }}" ] 36 | then 37 | echo "\$FAUNA_DB_DEV or \$GOOGLE_API_KEY secret does not exist, skipping Fauna tests" 38 | npm test -- --exclude **/db/utils_test.js --exclude **/db/scraper_test.js 39 | else 40 | echo "Necessary secrets exist, running all tests" 41 | npm test 42 | fi 43 | - run: npx prettier --check ./ 44 | - run: npm run verifyLockfile 45 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/CapeCodHealthcare/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const mychart = require("../../lib/MyChartAPI"); 3 | const moment = require("moment"); 4 | 5 | const siteId = "15342"; 6 | const vt = "696"; 7 | const dept = "10102135"; 8 | 9 | module.exports = async function GetAvailableAppointments() { 10 | console.log(`CapeCodHealthcare starting.`); 11 | const webData = await ScrapeWebsiteData(); 12 | console.log(`CapeCodHealthcare done.`); 13 | return { 14 | parentLocationName: "CapeCodHealthcare", 15 | timestamp: moment().format(), 16 | individualLocationData: [ 17 | { 18 | ...site, 19 | ...webData[dept], 20 | }, 21 | ], 22 | }; 23 | }; 24 | 25 | async function ScrapeWebsiteData() { 26 | // We need to go through the flow and use a request verification token 27 | const [ 28 | cookie, 29 | verificationToken, 30 | ] = await mychart.GetCookieAndVerificationToken( 31 | `https://mychart-openscheduling.et1149.epichosted.com/MyChart/OpenScheduling/SignupAndSchedule/EmbeddedSchedule?id=${siteId}&vt=${vt}&dept=${dept}&view=plain&public=1&payor=-1,-2,-3,4655,4660,1292,4661,5369,5257,1624,4883&lang=english1089` 32 | ); 33 | 34 | return mychart.AddFutureWeeks( 35 | "mychart-openscheduling.et1149.epichosted.com", 36 | "/mychart/OpenScheduling/OpenScheduling/GetOpeningsForProvider", 37 | cookie, 38 | verificationToken, 39 | 10, 40 | mychart.CommonPostDataCallback([siteId], [dept], vt) 41 | ); 42 | } 43 | -------------------------------------------------------------------------------- /site-scrapers/Hannaford/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Hannaford", 3 | website: 4 | "https://hannafordsched.rxtouch.com/rbssched/program/covid19/Patient/Advisory", 5 | locations: [ 6 | { 7 | street: "66 Drum Hill Rd.", 8 | city: "Chelmsford", 9 | zip: "01824", 10 | }, 11 | { 12 | street: "301 Pleasant St.", 13 | city: "Dracut", 14 | zip: "01826", 15 | }, 16 | { 17 | street: "118 Lancaster St.", 18 | city: "Leominster", 19 | zip: "01453", 20 | }, 21 | { 22 | street: "333 Mass Ave", 23 | city: "Lunenburg", 24 | zip: "01462", 25 | }, 26 | { 27 | street: "193 Boston Post Rd. West, Rte 20", 28 | city: "Marlborough", 29 | zip: "01752", 30 | }, 31 | { 32 | street: "8 Merchants Way", 33 | city: "Middleborough", 34 | zip: "02346", 35 | }, 36 | { 37 | street: "5 Gilbert St.", 38 | city: "North Brookfield", 39 | zip: "01535", 40 | }, 41 | { 42 | street: "255 Joseph E. Warner Blvd.", 43 | city: "Taunton", 44 | zip: "02780", 45 | }, 46 | { 47 | street: "158 No. Main St. Suite 3", 48 | city: "Uxbridge", 49 | zip: "01569", 50 | }, 51 | { 52 | street: "55 Russell St.", 53 | city: "Waltham", 54 | zip: "02453", 55 | }, 56 | ], 57 | }; 58 | 59 | module.exports = { 60 | site, 61 | }; 62 | -------------------------------------------------------------------------------- /site-scrapers/BaystateHealth/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const { sendSlackMsg } = require("../../lib/slack"); 3 | const s3 = require("../../lib/s3"); 4 | const moment = require("moment"); 5 | 6 | /* 7 | * This function calls ScrapeWebsiteData to gather availability data from the 8 | * site 9 | * 10 | * @param browser created when Puppeteer connects to a Chromium instance 11 | * @return JSON blob with { hasAppointments: Bool, totalAvailability: Int } 12 | */ 13 | module.exports = async function GetAvailableAppointments(browser) { 14 | console.log(`${site.name} starting.`); 15 | const webData = await ScrapeWebsiteData(browser); 16 | console.log(`${site.name} done.`); 17 | return { 18 | parentLocationName: "Baystate Health", 19 | timestamp: moment().format(), 20 | individualLocationData: [ 21 | { 22 | ...site, 23 | ...webData, 24 | }, 25 | ], 26 | }; 27 | }; 28 | 29 | async function ScrapeWebsiteData(browser) { 30 | const page = await browser.newPage(); 31 | await page.goto(site.signUpLink); 32 | const alertElement = await Promise.race([ 33 | page.waitForTimeout(3000).then(null), 34 | page.waitForSelector( 35 | "app-guest-covid-vaccine-register.ion-page > div.sky-bg > div.content-body > div.content-card > h3.text-align-center" 36 | ), 37 | ]); 38 | // Is the clinic open? 39 | const alert = await (alertElement 40 | ? alertElement.evaluate((node) => node.innerText) 41 | : false); 42 | let hasAvailability = !alert; 43 | 44 | page.close(); 45 | return { 46 | hasAvailability, 47 | }; 48 | } 49 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/MassGeneralBrigham/index.js: -------------------------------------------------------------------------------- 1 | const { entityName, startUrl, schedulePath, sites, vt } = require("./config"); 2 | const mychart = require("../../lib/MyChartAPI"); 3 | const moment = require("moment"); 4 | 5 | module.exports = async function GetAvailableAppointments() { 6 | console.log(`${entityName} starting.`); 7 | const webData = await ScrapeWebsiteData(sites); 8 | console.log(`${entityName} done.`); 9 | const results = []; 10 | const timestamp = moment().format(); 11 | 12 | for (const site of sites) { 13 | results.push({ 14 | ...site.public, 15 | ...webData[site.private.departmentID], 16 | }); 17 | } 18 | 19 | return { 20 | parentLocationName: `${entityName}`, 21 | isChain: true, 22 | timestamp, 23 | individualLocationData: results, 24 | }; 25 | }; 26 | 27 | async function ScrapeWebsiteData(sites) { 28 | const providerIDs = sites.flatMap((item) => item.private.providerId); 29 | const departmentIDs = sites.flatMap((item) => item.private.departmentID); 30 | 31 | // Request verification token which is needed to proceed onto fetching calendars 32 | const [ 33 | cookie, 34 | verificationToken, 35 | ] = await mychart.GetCookieAndVerificationToken( 36 | `${startUrl}?id=${providerIDs.join(",")}&dept=${departmentIDs.join( 37 | "," 38 | )}&vt=${vt}&lang=en-US` 39 | ); 40 | 41 | return mychart.AddFutureWeeks( 42 | "patientgateway.massgeneralbrigham.org", 43 | `${schedulePath()}`, 44 | cookie, 45 | verificationToken, 46 | 10, 47 | mychart.CommonPostDataCallback(providerIDs, departmentIDs, vt) 48 | ); 49 | } 50 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "macovidvaccines.com", 3 | "version": "1.0.0", 4 | "dependencies": { 5 | "@googlemaps/google-maps-services-js": "^3.1.16", 6 | "axios": "^0.21.1", 7 | "axios-retry": "^3.1.9", 8 | "bestzip": "^2.1.7", 9 | "chrome-aws-lambda": "^8.0.0", 10 | "deep-equal-in-any-order": "^1.0.28", 11 | "dotenv": "^8.2.0", 12 | "faunadb": "^4.1.1", 13 | "jsdom": "^16.4.0", 14 | "lodash": "^4.17.21", 15 | "moment": "^2.29.1", 16 | "node-fetch": "^2.6.1", 17 | "node-html-parser": "^2.1.0", 18 | "puppeteer": "npm:puppeteer-core@^8.0.0", 19 | "puppeteer-extra": "^3.1.18", 20 | "puppeteer-extra-plugin-recaptcha": "^3.3.5", 21 | "puppeteer-extra-plugin-stealth": "^2.7.6", 22 | "rimraf": "^3.0.2", 23 | "twit": "^2.2.11", 24 | "us-zips": "^4.0.2" 25 | }, 26 | "devDependencies": { 27 | "aws-sdk": "^2.840.0", 28 | "chai": "^4.3.0", 29 | "chai-as-promised": "^7.1.1", 30 | "chai-shallow-deep-equal": "^1.4.6", 31 | "chai-subset": "^1.6.0", 32 | "cross-env": "^7.0.3", 33 | "mocha": "^8.3.0", 34 | "mock-require": "^3.0.3", 35 | "nock": "^13.0.7", 36 | "prettier": "^2.2.1", 37 | "sinon": "^10.0.0" 38 | }, 39 | "engines": { 40 | "node": ">= 14.0.0", 41 | "npm": ">= 7.0.0" 42 | }, 43 | "scripts": { 44 | "predeploy": "rimraf lambda.zip && npm ci --production && bestzip lambda.zip ./*", 45 | "test": "cross-env NODE_ENV=test mocha --recursive --exit --timeout 10000", 46 | "verifyLockfile": "node -e \"if(require('./package-lock.json').lockfileVersion !== 2) {process.exit(1);};\"" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/HarborHealth/index.js: -------------------------------------------------------------------------------- 1 | const { sites } = require("./config"); 2 | const mychart = require("../../lib/MyChartAPI"); 3 | const moment = require("moment"); 4 | 5 | module.exports = async function GetAvailableAppointments() { 6 | console.log(`HarborHealth starting.`); 7 | const originalSites = sites; 8 | const finalSites = []; 9 | for (const site of originalSites) { 10 | const { siteId, vt, dept, ...restSite } = site; 11 | const webData = await ScrapeWebsiteData(siteId, vt, dept); 12 | finalSites.push({ 13 | ...webData[dept], 14 | /* adding site last because myChart is returning address 15 | * of the provider instead of the address of the clinic 16 | */ 17 | ...restSite, 18 | }); 19 | } 20 | console.log(`HarborHealth done.`); 21 | return { 22 | parentLocationName: "HarborHealth", 23 | timestamp: moment().format(), 24 | individualLocationData: finalSites, 25 | }; 26 | }; 27 | 28 | async function ScrapeWebsiteData(siteId, vt, dept) { 29 | // We need to go through the flow and use a request verification token 30 | const [ 31 | cookie, 32 | verificationToken, 33 | ] = await mychart.GetCookieAndVerificationToken( 34 | `https://mychartos.ochin.org/mychart/SignupAndSchedule/EmbeddedSchedule?id=${siteId}&vt=${vt}&dept=${dept}&view=plain&public=1&payor=-1,-2,-3,4655,4660,1292,4661,5369,5257,1624,4883&lang=english` 35 | ); 36 | 37 | return mychart.AddFutureWeeks( 38 | "mychartos.ochin.org", 39 | "/mychart/OpenScheduling/OpenScheduling/GetOpeningsForProvider", 40 | cookie, 41 | verificationToken, 42 | 10, 43 | mychart.CommonPostDataCallback([siteId], [dept], vt) 44 | ); 45 | } 46 | -------------------------------------------------------------------------------- /lib/slack.js: -------------------------------------------------------------------------------- 1 | const https = require("https"); 2 | 3 | const slackWebhookHost = "hooks.slack.com"; 4 | 5 | const channelConfig = { 6 | dev: { 7 | webhookPath: process.env.SLACKWEBHOOKDEVCHANNEL, 8 | }, 9 | bot: { 10 | webhookPath: process.env.SLACKWEBHOOKBOTCHANNEL, 11 | }, 12 | }; 13 | 14 | /** 15 | * Send Slack message via webhook 16 | * @param {string} channel 17 | * @param {string} text 18 | */ 19 | 20 | function sendSlackMsg(channel, text) { 21 | const postData = JSON.stringify({ 22 | text, 23 | }); 24 | 25 | const options = { 26 | hostname: slackWebhookHost, 27 | port: 443, 28 | path: channelConfig[channel].webhookPath, 29 | rejectUnauthorized: false, 30 | method: "POST", 31 | headers: { 32 | "Content-Type": "application/json", 33 | "Content-Length": postData.length, 34 | }, 35 | }; 36 | 37 | return new Promise((resolve) => { 38 | const req = https.request(options, (res) => { 39 | let body = ""; 40 | res.on("data", (chunk) => { 41 | body += chunk; 42 | }); 43 | res.on("end", () => { 44 | if (res.statusCode >= 200 && res.statusCode < 300) { 45 | resolve("Sent"); 46 | } else { 47 | console.error( 48 | `Error status code [${res.statusCode}] returned from schedule request: ${res.statusMessage}` 49 | ); 50 | resolve("Error"); 51 | } 52 | }); 53 | }); 54 | req.write(postData); 55 | req.on("error", (e) => { 56 | console.error("Error making scheduling request : " + e); 57 | }); 58 | req.end(); 59 | }); 60 | } 61 | 62 | module.exports = { 63 | sendSlackMsg, 64 | }; 65 | -------------------------------------------------------------------------------- /data/dataDefaulter.js: -------------------------------------------------------------------------------- 1 | /* mergeResults 2 | * 3 | * Merges cachedResults into currentResults. If secondsOfTolerance is set, 4 | * will only merge in cachedResults with a timestamp newer than 5 | * now - secondsOfTolerance. 6 | */ 7 | function mergeResults(currentResults, cachedResults, secondsOfTolerance) { 8 | if (!(cachedResults && cachedResults.length)) { 9 | return currentResults; 10 | } else { 11 | const combinedResults = []; 12 | const currentResultsMap = {}; 13 | currentResults.forEach((result) => { 14 | combinedResults.push(result); 15 | currentResultsMap[generateKey(result)] = 1; 16 | }); 17 | 18 | cachedResults.forEach((cachedResult) => { 19 | // ignore any cached results that don't have a timestamp 20 | if ( 21 | cachedResult.timestamp && 22 | !currentResultsMap[generateKey(cachedResult)] 23 | ) { 24 | if (secondsOfTolerance) { 25 | const lowerTimeBound = 26 | new Date() - secondsOfTolerance * 1000; 27 | if (cachedResult.timestamp >= lowerTimeBound) { 28 | combinedResults.push(cachedResult); 29 | } 30 | } else { 31 | combinedResults.push(cachedResult); 32 | } 33 | } 34 | }); 35 | 36 | return combinedResults; 37 | } 38 | } 39 | 40 | function generateKey(entry) { 41 | let uniqueIdentifier = ""; 42 | ["name", "street", "city", "zip"].forEach((key) => { 43 | if (entry[key]) { 44 | uniqueIdentifier += `${entry[key] 45 | .toLowerCase() 46 | .replace(/[^\w]/g, "")}|`; 47 | } 48 | }); 49 | 50 | return uniqueIdentifier; 51 | } 52 | 53 | module.exports.mergeResults = mergeResults; 54 | module.exports.generateKey = generateKey; 55 | -------------------------------------------------------------------------------- /site-scrapers/Walgreens/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Walgreens", 3 | website: "https://www.walgreens.com/findcare/vaccination/covid-19/", 4 | locations: [ 5 | { street: "1010 Broadway", city: "Chelsea", zip: "02150" }, 6 | { street: "107 High St", city: "Danvers", zip: "01923" }, 7 | { 8 | street: "757 Gallivan Blvd", 9 | city: "Dorchester", 10 | zip: "02122", 11 | }, 12 | { street: "405 Broadway", city: "Everett", zip: "02149" }, 13 | { 14 | street: "220 Huttleston Ave", 15 | city: "Fairhaven", 16 | zip: "02719", 17 | }, 18 | { 19 | street: "369 Plymouth Ave", 20 | city: "Fall River", 21 | zip: "02721", 22 | }, 23 | { street: "232 Main St", city: "Gardner", zip: "01440" }, 24 | { 25 | street: "68 South Main St", 26 | city: "Haverhill", 27 | zip: "01835", 28 | }, 29 | { street: "1145 Main St", city: "Holden", zip: "01520" }, 30 | { street: "520 W. Main St", city: "Hyannis", zip: "02601" }, 31 | { street: "25 Park St", city: "Lee", zip: "01238" }, 32 | { street: "21 South St", city: "Mashpee", zip: "02649" }, 33 | { street: "825 Morton St", city: "Boston", zip: "02126" }, 34 | { street: "90 River St", city: "Mattapan", zip: "02126" }, 35 | { 36 | street: "37 Cheshire Road", 37 | city: "Pittsfield", 38 | zip: "01201", 39 | }, 40 | { street: "430 Broadway", city: "Revere", zip: "02151" }, 41 | { 42 | street: "1890 Columbus Ave", 43 | city: "Roxbury", 44 | zip: "02119", 45 | }, 46 | { street: "416 Warren St", city: "Roxbury", zip: "02119" }, 47 | { street: "166 Walnut St", city: "Saugus", zip: "01906" }, 48 | { street: "296 Buffinton St", city: "Somerset", zip: "02726" }, 49 | ], 50 | }; 51 | 52 | module.exports = { 53 | site, 54 | }; 55 | -------------------------------------------------------------------------------- /site-scrapers/FamilyPracticeGroup/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const { sendSlackMsg } = require("../../lib/slack"); 3 | const s3 = require("../../lib/s3"); 4 | const moment = require("moment"); 5 | 6 | const noAppointmentMatchString = 7 | "All appointment times are currently reserved."; 8 | 9 | module.exports = async function GetAvailableAppointments(browser) { 10 | console.log(`${site.name} starting.`); 11 | const data = await ScrapeWebsiteData(browser); 12 | console.log(`${site.name} done.`); 13 | const { website, ...restSite } = site; 14 | return { 15 | parentLocationName: "Arlington Family Practice Group", 16 | timestamp: moment().format(), 17 | individualLocationData: [ 18 | { 19 | ...restSite, 20 | signUpLink: site.website, 21 | ...data, 22 | }, 23 | ], 24 | }; 25 | }; 26 | 27 | async function waitForLoadComplete(page, loaderSelector) { 28 | await page.waitForSelector(loaderSelector, { visible: true }); 29 | await page.waitForSelector(loaderSelector, { hidden: true }); 30 | } 31 | 32 | async function ScrapeWebsiteData(browser) { 33 | return { hasAvailability: false }; 34 | // const page = await browser.newPage(); 35 | // await page.goto(site.website); 36 | // await page.waitForSelector("#nextBtn", { visible: true }); 37 | // await page.click("#nextBtn"); 38 | // await page.waitForSelector("#serviceTitle", { visible: true }); 39 | // await page.waitForSelector("#nextBtn", { visible: true }); 40 | // await page.waitForTimeout(300); 41 | // await page.click("#nextBtn"); 42 | // await page.waitForSelector("#screeningQuestionPassBtn", { visible: true }); 43 | // await page.click("#screeningQuestionPassBtn"); 44 | // await waitForLoadComplete(page, ".schedulerPanelLoading"); 45 | 46 | // const content = await (await page.$(".schedulerPanelBody")).evaluate( 47 | // (node) => node.innerText 48 | // ); 49 | // const hasAvailability = content.indexOf(noAppointmentMatchString) === -1; 50 | // return { hasAvailability }; 51 | } 52 | -------------------------------------------------------------------------------- /site-scrapers/Walgreens/dataFormatter.js: -------------------------------------------------------------------------------- 1 | const { generateKey } = require("../../data/dataDefaulter"); 2 | const { toTitleCase } = require("../../lib/stringUtil"); 3 | const moment = require("moment"); 4 | const fetch = require("node-fetch"); 5 | 6 | function formatData(data, website) { 7 | return data.map((entry) => { 8 | let hasAvailability = false; 9 | const availability = {}; 10 | entry.appointmentAvailability.forEach((daySlot) => { 11 | const date = moment(daySlot.date).local().startOf("day"); 12 | const midnightToday = moment().local().startOf("day"); 13 | if (date.isSame(midnightToday)) { 14 | const todaySlots = daySlot.slots.filter((slot) => 15 | moment(slot, "HH:mm a").isAfter(moment().local()) 16 | ); 17 | const numSlots = todaySlots.length; 18 | availability[date.format("M/D/YYYY")] = { 19 | hasAvailability: !!numSlots, 20 | numberAvailableAppointments: numSlots, 21 | }; 22 | hasAvailability = hasAvailability || !!numSlots; 23 | } else if (date.isAfter(midnightToday)) { 24 | const numSlots = daySlot.slots.length; 25 | availability[date.format("M/D/YYYY")] = { 26 | hasAvailability: !!numSlots, 27 | numberAvailableAppointments: numSlots, 28 | }; 29 | hasAvailability = hasAvailability || !!numSlots; 30 | } 31 | }); 32 | return { 33 | name: `Walgreens (${toTitleCase(entry.address.city)})`, // NOTE: change to "entry.name" if we use the commented-out code above 34 | street: toTitleCase( 35 | entry.address.line1 + " " + entry.address.line2 36 | ).trim(), 37 | city: toTitleCase(entry.address.city), 38 | zip: entry.address.zip, 39 | signUpLink: website, 40 | hasAvailability, 41 | availability: availability, 42 | }; 43 | }); 44 | } 45 | 46 | module.exports = { 47 | formatData, 48 | }; 49 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/UMassMemorial/index.js: -------------------------------------------------------------------------------- 1 | const { sites, providers } = require("./config"); 2 | const mychart = require("../../lib/MyChartAPI"); 3 | const moment = require("moment"); 4 | 5 | const departmentIDs = sites.map((site) => site.departmentID); 6 | 7 | module.exports = async function GetAvailableAppointments() { 8 | console.log("UMassMemorial starting."); 9 | const webData = await ScrapeWebsiteData(); 10 | const results = []; 11 | const timestamp = moment().format(); 12 | 13 | for (const site of sites) { 14 | const { departmentID, ...restSite } = site; 15 | results.push({ 16 | ...webData[departmentID], 17 | ...restSite, 18 | }); 19 | } 20 | console.log("UMassMemorial done."); 21 | return { 22 | parentLocationName: "UMass Memorial", 23 | isChain: true, 24 | timestamp, 25 | individualLocationData: results, 26 | }; 27 | }; 28 | 29 | async function ScrapeWebsiteData() { 30 | // We need to go through the flow and use a request verification token 31 | const [ 32 | cookie, 33 | verificationToken, 34 | ] = await mychart.GetCookieAndVerificationToken( 35 | "https://mychartonline.umassmemorial.org/mychart/openscheduling?specialty=15&hidespecialtysection=1" 36 | ); 37 | return mychart.AddFutureWeeks( 38 | "mychartonline.umassmemorial.org", 39 | "/MyChart/OpenScheduling/OpenScheduling/GetScheduleDays", 40 | cookie, 41 | verificationToken, 42 | 10, 43 | PostDataCallback 44 | ); 45 | } 46 | 47 | /** 48 | * mychart.AddFutureWeeks calls this function to get the data it should POST to the API. 49 | */ 50 | function PostDataCallback(startDateFormatted) { 51 | const Departments = mychart.CommonFilters.Departments(departmentIDs); 52 | const Providers = mychart.CommonFilters.Providers(providers); 53 | const { DaysOfWeek, TimesOfDay } = mychart.CommonFilters; 54 | 55 | const filters = JSON.stringify({ 56 | Providers, 57 | Departments, 58 | DaysOfWeek, 59 | TimesOfDay, 60 | }); 61 | return `view=grouped&specList=15&vtList=5060&start=${startDateFormatted}&filters=${encodeURIComponent( 62 | filters 63 | )}`; 64 | } 65 | -------------------------------------------------------------------------------- /site-scrapers/ReverePopup/index.js: -------------------------------------------------------------------------------- 1 | // Unsure how often this clinic will pop up but seems to be weekly for now 2 | const { site } = require("./config"); 3 | const moment = require("moment"); 4 | 5 | module.exports = async function GetAvailableAppointments(browser) { 6 | console.log(`${site.name} starting.`); 7 | const info = await ScrapeWebsiteData(browser); 8 | console.log(`${site.name} done.`); 9 | return { 10 | parentLocationName: "Revere Popup Clinic", 11 | timestamp: moment().format(), 12 | individualLocationData: [ 13 | { 14 | ...site, 15 | ...info, 16 | }, 17 | ], 18 | }; 19 | }; 20 | 21 | async function ScrapeWebsiteData(browser) { 22 | const page = await browser.newPage(); 23 | const results = { 24 | hasAvailability: false, 25 | availability: {}, 26 | }; 27 | await page.goto(site.signUpLink); 28 | // Only 1 h1 on page, will have text like "Sign Up for Vaccinations - Rumney Marsh Academy on 04/30/2021" 29 | try { 30 | await page.waitForSelector("h1", { visible: true }); 31 | const date = await page.$eval( 32 | "h1", 33 | (h1) => h1.innerText.match(/[0-9]+\/[0-9]+\/[0-9]+/)[0] 34 | ); 35 | 36 | // the first tr on the page is unrelated which is why you have to be specific 37 | const appointments = await page.$$eval("tr[data-parent]", (trs) => 38 | trs.reduce((acc, tr) => { 39 | // text will look like "06:09 pm\n\n20 appointments available" 40 | const text = tr.innerText; 41 | const numberAvailableAppointments = text.match( 42 | /([0-9]+) appointments available/ 43 | ); 44 | if (numberAvailableAppointments) { 45 | return Number(numberAvailableAppointments[1]) + acc; 46 | } 47 | return acc; 48 | }, 0) 49 | ); 50 | 51 | page.close(); 52 | if (appointments) { 53 | results["hasAvailability"] = true; 54 | results["availability"][date] = { 55 | numberAvailableAppointments: appointments, 56 | hasAvailability: true, 57 | }; 58 | return results; 59 | } else { 60 | return results; 61 | } 62 | } catch (error) { 63 | console.log("Error in Revere PopUp clinic:", error); 64 | return results; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/Albertsons/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const https = require("https"); 3 | const fetch = require("node-fetch"); 4 | const moment = require("moment"); 5 | 6 | module.exports = async function GetAvailableAppointments() { 7 | console.log(`${site.name} starting.`); 8 | 9 | const url = `${site.nationalStoresJson}?v=${new Date().valueOf()}`; 10 | 11 | const nationalLocations = await fetch(url).then((res) => res.json()); 12 | 13 | const massLocations = nationalLocations.filter( 14 | (location) => location.region == "Massachusetts" 15 | ); 16 | 17 | console.log(`${site.name} done.`); 18 | 19 | return { 20 | parentLocationName: "Shaw's/Star Market", 21 | isChain: true, 22 | timestamp: moment().format(), 23 | individualLocationData: massLocations.map((location) => { 24 | // Raw address is like: (Star Market 4587 - Pfizer and Moderna Availability||45 William T Morrissey Blvd, Dorchester, MA, 02125) 25 | // The format seems to be very consistent nationally, not to mention locally in MA. So we 26 | // pull the specific parts out of the string. 27 | const rawAddress = location.address; 28 | const trimmedAddress = rawAddress.replace(/^\(|\)$/, ""); // Trim parens 29 | let [name, addressRest] = trimmedAddress.split(" - "); 30 | let [vaccine, longAddress] = addressRest.split("||"); 31 | if (addressRest.indexOf("||") === -1) { 32 | longAddress = vaccine; 33 | vaccine = undefined; 34 | } 35 | const [street, city, state, zip] = longAddress.split(", "); 36 | const storeName = name.match(/([^\d]*) /g)[0].trim(); 37 | // const storeNumber = name.substring(storeName.length).trim(); 38 | 39 | const extraData = vaccine 40 | ? { extraData: { "Vaccinations offered": vaccine } } 41 | : undefined; 42 | 43 | const retval = { 44 | name: storeName, 45 | street: street, 46 | city: city, 47 | state: state, 48 | zip: zip, 49 | hasAvailability: location.availability === "yes", 50 | ...extraData, 51 | signUpLink: location.coach_url, 52 | latitude: location.lat, 53 | longitude: location.long, 54 | }; 55 | return retval; 56 | }), 57 | }; 58 | }; 59 | -------------------------------------------------------------------------------- /getGeocode.js: -------------------------------------------------------------------------------- 1 | const dotenv = require("dotenv"); 2 | //note: this only works locally; in Lambda we use environment variables set manually 3 | dotenv.config(); 4 | 5 | const { Client } = require("@googlemaps/google-maps-services-js"); 6 | const { generateKey } = require("./data/dataDefaulter"); 7 | const axios = require("axios"); 8 | const axiosRetry = require("axios-retry"); 9 | 10 | axiosRetry(axios, { retries: 5, retryDelay: axiosRetry.exponentialDelay }); 11 | 12 | const getGeocode = async (name, street, zip) => { 13 | const address = `${name},MA,${street},${zip}`; 14 | const client = new Client(); 15 | try { 16 | const resp = await client.geocode( 17 | { 18 | params: { 19 | address, 20 | key: process.env.GOOGLE_API_KEY, 21 | }, 22 | }, 23 | axios 24 | ); 25 | return resp.data; 26 | } catch (e) { 27 | console.error(e.response.data); 28 | } 29 | }; 30 | 31 | const getAllCoordinates = async (locations, cachedResults = {}) => { 32 | const existingLocations = cachedResults.reduce((acc, location) => { 33 | const { latitude, longitude } = location; 34 | if (latitude && longitude) { 35 | acc[generateKey(location)] = { 36 | latitude, 37 | longitude, 38 | }; 39 | return acc; 40 | } else { 41 | return acc; 42 | } 43 | }, {}); 44 | 45 | const coordinateData = await Promise.all( 46 | locations.map(async (location) => { 47 | const { name = "", street = "", zip = "" } = location; 48 | const locationInd = generateKey(location); 49 | 50 | if (existingLocations[locationInd]) { 51 | return { ...location, ...existingLocations[locationInd] }; 52 | } else { 53 | const locationData = await getGeocode(name, street, zip); 54 | 55 | if (locationData) { 56 | return { 57 | ...location, 58 | latitude: 59 | locationData?.results[0].geometry.location.lat, 60 | longitude: 61 | locationData?.results[0].geometry.location.lng, 62 | }; 63 | } else return location; 64 | } 65 | }) 66 | ); 67 | return coordinateData; 68 | }; 69 | 70 | exports.getAllCoordinates = getAllCoordinates; 71 | exports.getGeocode = getGeocode; 72 | -------------------------------------------------------------------------------- /site-scrapers/LowellGeneral/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const { 3 | buildTimeSlotsUrl, 4 | getStationTimeslots, 5 | parseAvailability, 6 | groupAppointmentsByDate, 7 | getNodeId, 8 | } = require("./functions"); 9 | const moment = require("moment"); 10 | 11 | module.exports = async function GetAvailableAppointments(browser) { 12 | console.log(`${site.name} starting.`); 13 | const webData = await ScrapeWebsiteData(browser); 14 | console.log(`${site.name} done.`); 15 | return { 16 | parentLocationName: "Lowell General", 17 | timestamp: moment().format(), 18 | individualLocationData: [webData], 19 | }; 20 | }; 21 | 22 | async function ScrapeWebsiteData(browser) { 23 | const page = await browser.newPage(); 24 | await page.goto(site.startUrl, { waitUntil: "networkidle0" }); 25 | const { noAppointments, startUrl, ...restSite } = site; 26 | const noAppointmentsMatch = (await page.content()).match(noAppointments); 27 | let hasAvailability = false; 28 | let availability = {}; 29 | if (noAppointmentsMatch) { 30 | // This is so we avoid timing out when there aren't any appointments 31 | return { 32 | hasAvailability, 33 | availability, 34 | ...restSite, 35 | }; 36 | } 37 | const stationSelector = "[id*='time_slots_for_doctor_']"; 38 | await page.waitForSelector(stationSelector); 39 | let stations = await page.$$(stationSelector); 40 | for (const station of stations) { 41 | const stationId = await getNodeId(station); 42 | const url = buildTimeSlotsUrl(stationId); 43 | const xhrRes = await getStationTimeslots(page, url); 44 | const parsedData = parseAvailability(xhrRes); 45 | // Only set availability if still false. 46 | // If one station has availability, 47 | // then hasAvailability should stay true. 48 | if (!hasAvailability) { 49 | hasAvailability = parsedData.hasAvailability; 50 | } 51 | availability = groupAppointmentsByDate( 52 | availability, 53 | parsedData.availability 54 | ); 55 | } 56 | // Sort the keys since we get dates out of order. 57 | availability = Object.keys(availability) 58 | .sort() 59 | .reduce((obj, key) => { 60 | obj[key] = availability[key]; 61 | return obj; 62 | }, {}); 63 | 64 | return { 65 | hasAvailability, 66 | availability, 67 | ...site, 68 | timestamp: new Date(), 69 | }; 70 | } 71 | -------------------------------------------------------------------------------- /alerts/send_texts_and_emails.js: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk"); 2 | const determineRecipients = require("./determine_recipients"); 3 | const dotenv = require("dotenv"); 4 | dotenv.config(); 5 | 6 | const SMS_PER_SECOND = 20; 7 | 8 | if (process.env.NODE_ENV !== "production") { 9 | AWS.config.credentials = new AWS.SharedIniFileCredentials({ 10 | profile: "macovidvaccines", 11 | }); 12 | AWS.config.update({ region: "us-east-1" }); 13 | } 14 | 15 | const pinpoint = new AWS.Pinpoint(); 16 | 17 | module.exports = { 18 | handler, 19 | sendTexts, 20 | }; 21 | 22 | async function handler({ locations, numberAppointmentsFound, message }) { 23 | const subscribers = await determineRecipients.determineRecipients({ 24 | locations, 25 | numberAvailable: numberAppointmentsFound, 26 | }); 27 | return sendTexts( 28 | subscribers.textRecipients.map((subscriber) => subscriber.phoneNumber), 29 | message 30 | ); 31 | } 32 | 33 | async function sendTexts(phoneNumbers, message) { 34 | const phoneNumbersCopy = [...phoneNumbers]; 35 | while (phoneNumbersCopy.length) { 36 | const currentPhoneNumbers = phoneNumbersCopy.splice( 37 | 0, 38 | Math.min(SMS_PER_SECOND, phoneNumbersCopy.length) 39 | ); 40 | console.log(`message: ${message}`); 41 | console.log(`sending texts to ${JSON.stringify(currentPhoneNumbers)}`); 42 | await new Promise((resolve, reject) => 43 | pinpoint.sendMessages( 44 | { 45 | ApplicationId: process.env.PINPOINT_APPLICATION_ID, 46 | MessageRequest: { 47 | Addresses: currentPhoneNumbers.reduce((obj, curNum) => { 48 | obj[`+1${curNum}`] = { ChannelType: "SMS" }; 49 | return obj; 50 | }, {}), 51 | MessageConfiguration: { 52 | SMSMessage: { 53 | Body: message, 54 | MessageType: "PROMOTIONAL", 55 | OriginationNumber: 56 | process.env.PINPOINT_ORIGINATION_NUMBER, 57 | }, 58 | }, 59 | }, 60 | }, 61 | (err, data) => { 62 | if (err) { 63 | reject(err); 64 | } else { 65 | resolve(data); 66 | } 67 | } 68 | ) 69 | ); 70 | if (phoneNumbersCopy.length) { 71 | await new Promise((resolve) => setTimeout(resolve, 1000)); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /site-scrapers/Wegmans/index.js: -------------------------------------------------------------------------------- 1 | const { site, paths } = require("./config"); 2 | const moment = require("moment"); 3 | const s3 = require("../../lib/s3"); 4 | const { sendSlackMsg } = require("../../lib/slack"); 5 | 6 | module.exports = async function GetAvailableAppointments(browser) { 7 | console.log(`${site.name} starting.`); 8 | const webData = await ScrapeWebsiteData(browser); 9 | console.log(`${site.name} done.`); 10 | return { 11 | parentLocationName: "Wegmans", 12 | isChain: true, 13 | timestamp: moment().format(), 14 | individualLocationData: [ 15 | { 16 | ...site, 17 | ...webData, 18 | }, 19 | ], 20 | }; 21 | }; 22 | 23 | async function ScrapeWebsiteData(browser) { 24 | const page = await browser.newPage(); 25 | await page.goto(site.signUpLink, { waitUntil: "networkidle0" }); 26 | await page.waitForSelector("#frameForm"); 27 | 28 | const frameForm = await page.$("#frameForm"); 29 | const botUrl = await frameForm.evaluate((node) => 30 | node.getAttribute("action") 31 | ); 32 | await page.goto(botUrl, { waitUntil: "networkidle0" }); 33 | 34 | await page.waitForXPath(paths.getStartedBtn); 35 | const getStartedBtns = await page.$x(paths.getStartedBtn); 36 | getStartedBtns[0].click(); 37 | await page.waitForXPath(paths.massOption); 38 | const massLinks = await page.$x(paths.massOption); 39 | massLinks[1].click(); 40 | await page.waitForTimeout(1000); 41 | const scheduleBtns = await page.$x(paths.scheduleBtn); 42 | scheduleBtns[1].click(); 43 | 44 | // Wait for schedule chat bot response 45 | const lastMessageText = await new Promise((resolve) => { 46 | const interval = setInterval(async () => { 47 | // wait for count of chat bot responses to be at least 4 48 | const botMessages = await page.$x(paths.botMessage); 49 | const count = botMessages.length; 50 | const lastMessage = botMessages.pop(); 51 | const messageText = await lastMessage.evaluate( 52 | (node) => node.innerText 53 | ); 54 | if (count >= 4) { 55 | clearInterval(interval); 56 | resolve(messageText); 57 | } 58 | }, 500); 59 | }); 60 | 61 | let hasAvailability = false; 62 | let availability = {}; 63 | 64 | const noAppointments = lastMessageText.includes(paths.noAppointments); 65 | if (!noAppointments) { 66 | const msg = `${site.name} - possible appointments`; 67 | console.log(msg); 68 | await s3.savePageContent(site.name, page); 69 | await sendSlackMsg("bot", msg); 70 | } 71 | 72 | return { 73 | hasAvailability, 74 | availability, 75 | }; 76 | } 77 | -------------------------------------------------------------------------------- /one-off-scripts/20210420-send-accidentally-cancelled-text.js: -------------------------------------------------------------------------------- 1 | const dbUtils = require("../lib/db/utils"); 2 | const faunadb = require("faunadb"), 3 | fq = faunadb.query; 4 | const { sendTexts } = require("../alerts/send_texts_and_emails"); 5 | 6 | module.exports = run; 7 | 8 | async function run() { 9 | // find people who were marked cancelled on/before 9 AM this morning 10 | // but were non-existent OR were not marked cancelled on/before 6 PM yesterday 11 | const start_ts = fq.Time("2021-04-19T22:00:00Z"); 12 | const end_ts = fq.Time("2021-04-20T13:00:00Z"); 13 | const cancelledSubscribers = await dbUtils 14 | .faunaQuery( 15 | fq.Map( 16 | fq.Filter( 17 | fq.Paginate(fq.Documents(fq.Collection("subscriptions")), { 18 | size: 5000, 19 | }), 20 | fq.Lambda( 21 | "sub", 22 | fq.And( 23 | fq.And( 24 | fq.Exists(fq.Var("sub"), end_ts), 25 | fq.Equals( 26 | true, 27 | fq.Select( 28 | ["data", "cancelled"], 29 | fq.Get(fq.Var("sub"), end_ts) 30 | ) 31 | ) 32 | ), 33 | fq.Or( 34 | fq.Not(fq.Exists(fq.Var("sub"), start_ts)), 35 | fq.Equals( 36 | false, 37 | fq.Select( 38 | ["data", "cancelled"], 39 | fq.Get(fq.Var("sub"), start_ts) 40 | ) 41 | ) 42 | ) 43 | ) 44 | ) 45 | ), 46 | fq.Lambda("x", fq.Get(fq.Var("x"))) 47 | ) 48 | ) 49 | .catch(console.error); 50 | console.log(`found ${cancelledSubscribers.data.length} candidates.`); 51 | const nums = cancelledSubscribers.data 52 | .map((sub) => sub.data.phoneNumber) 53 | .sort(); 54 | const msg = 55 | "This morning we fixed an issue where replying HELP unsubscribed people from alerts. " + 56 | "We are contacting you because you may have been affected. " + 57 | "To re-enroll in alerts, fill out the form again on macovidvaccines.com. " + 58 | "If you intended to unsubscribe, no action is necessary and we will not contact you again. " + 59 | "We apologize for the inconvenience!"; 60 | await sendTexts(nums, msg); 61 | } 62 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/Atrius/index.js: -------------------------------------------------------------------------------- 1 | const https = require("https"); 2 | const { site } = require("./config"); 3 | const mychart = require("../../lib/MyChartAPI.js"); 4 | const moment = require("moment"); 5 | 6 | const dept = "12701803"; 7 | 8 | module.exports = async function GetAvailableAppointments() { 9 | console.log("Atrius starting."); 10 | const webData = await ScrapeWebsiteData(); 11 | console.log("Atrius done."); 12 | const { dphLink, website, ...atriusSubObject } = site; 13 | return { 14 | parentLocationName: "Atrius", 15 | timestamp: moment().format(), 16 | individualLocationData: [ 17 | { 18 | ...atriusSubObject, 19 | ...webData, 20 | }, 21 | ], 22 | }; 23 | }; 24 | 25 | function urlRedirect(url, options) { 26 | return new Promise((resolve) => { 27 | let response = ""; 28 | https.get(url, options, (res) => { 29 | let body = ""; 30 | res.on("data", (chunk) => (body += chunk)); 31 | res.on("end", () => { 32 | response = res.headers ? res.headers.location : null; 33 | resolve(response); 34 | }); 35 | }); 36 | }); 37 | } 38 | 39 | async function ScrapeWebsiteData() { 40 | const checkSlots = await urlRedirect(site.website, {}); 41 | if (checkSlots && checkSlots.match("No_Slots")) { 42 | console.log( 43 | `Atrius redirecting to no slots, ${checkSlots}, assuming failure!` 44 | ); 45 | return { 46 | hasAvailability: false, 47 | availability: {}, //this line is optional 48 | }; 49 | } 50 | 51 | // We need to go through the flow and use a request verification token 52 | const [ 53 | cookie, 54 | verificationToken, 55 | ] = await mychart.GetCookieAndVerificationToken(site.dphLink); 56 | 57 | // Setup the return object. 58 | const results = mychart.AddFutureWeeks( 59 | "myhealth.atriushealth.org", 60 | "/OpenScheduling/OpenScheduling/GetScheduleDays", 61 | cookie, 62 | verificationToken, 63 | 10, 64 | PostDataCallback 65 | ); 66 | // object returned from AddFutureWeeks maps dept ID -> availability info 67 | // here, we want to just return availability info for relevant dept. 68 | return results[dept]; 69 | } 70 | 71 | /** 72 | * mychart.AddFutureWeeks calls this function to get the data it should POST to the API. 73 | */ 74 | function PostDataCallback(startDateFormatted) { 75 | const { TimesOfDay } = mychart.CommonFilters; 76 | return `view=grouped&specList=121&vtList=1424&start=${startDateFormatted}&filters=${encodeURIComponent( 77 | JSON.stringify({ 78 | Providers: {}, 79 | Departments: {}, 80 | DaysOfWeek: {}, 81 | TimesOfDay, 82 | }) 83 | )}`; 84 | } 85 | -------------------------------------------------------------------------------- /test/AtriusTest.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const nock = require("nock"); 3 | const sinon = require("sinon"); 4 | const chai = require("chai"); 5 | chai.use(require("chai-as-promised")); 6 | const expect = chai.expect; 7 | 8 | describe("Atrius GetAvailabilities", async () => { 9 | it("should return no availabilities when there is a redirect", async () => { 10 | const atrius = require("./../no-browser-site-scrapers/Atrius"); 11 | // mock out the redirect that occurs when there Atrius doesn't want to show any slots. 12 | nock("https://myhealth.atriushealth.org") 13 | .get("/fr/") 14 | .reply(301, "", { location: "No_Slots" }); 15 | // run the test and assert that the result looks like: 16 | /* 17 | { 18 | "hasAvailability": false, 19 | "availability: {} 20 | } 21 | */ 22 | await expect(atrius().then((res) => res.individualLocationData[0])) 23 | .to.eventually.deep.include({ 24 | hasAvailability: false, 25 | }) 26 | .and.nested.property("availability") 27 | .deep.equal({}); 28 | nock.cleanAll(); 29 | }); 30 | 31 | it("should return availabilities when there are some.", async () => { 32 | // mock that there is no redirect 33 | nock("https://myhealth.atriushealth.org") 34 | .get("/fr/") 35 | .reply(200, "OK response", { location: "/dph/" }); 36 | 37 | const resultingAvailability = { 38 | hasAvailability: true, 39 | availability: { 40 | "2/21/2021": { 41 | numberAvailableAppointments: 3, 42 | hasAvailability: true, 43 | }, 44 | }, 45 | }; 46 | 47 | // mock the mychart api 48 | const myChartApi = require("../lib/MyChartAPI"); 49 | sinon 50 | .stub(myChartApi, "GetCookieAndVerificationToken") 51 | .callsFake(() => ["theCookie", "theVerificationToken"]); 52 | sinon 53 | .stub(myChartApi, "AddFutureWeeks") 54 | .callsFake(() => ({ 12701803: resultingAvailability })); 55 | // specify the require here after the mock was created. 56 | const atrius = require("./../no-browser-site-scrapers/Atrius"); 57 | // run the test and assert that the result looks like: 58 | /* 59 | { 60 | "hasAvailability": true, 61 | "availability: { 62 | "2/21/2021": { 63 | numberAvailableAppointments: 3, 64 | hasAvailability: true, 65 | }, 66 | } 67 | } 68 | */ 69 | const result = atrius().then((res) => res.individualLocationData[0]); 70 | await expect(result).to.eventually.include(resultingAvailability); 71 | nock.cleanAll(); 72 | }); 73 | }); 74 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/GreaterLawrenceFHC/config.js: -------------------------------------------------------------------------------- 1 | const provider = "Greater Lawrence Family Health Center"; 2 | 3 | const sites = [ 4 | { 5 | public: { 6 | name: "Greater Lawrence Family Health Center (Essex Street)", 7 | street: "700 Essex St.", 8 | zip: "01841", 9 | restrictions: 10 | "Greater Lawrence Family Health Center patients only / Pacientes solamente", 11 | signUpLink: 12 | "https://glfhccovid19iz.as.me/schedule.php?location=700+Essex+Street%2C+Lawrence+MA", 13 | }, 14 | private: { 15 | scrapeUrl: 16 | "https://glfhccovid19iz.as.me/schedule.php?action=showCalendar&fulldate=1&owner=21956779&template=weekly&location=700+Essex+Street%2C+Lawrence+MA", 17 | 18 | calendar: "5075244", 19 | }, 20 | }, 21 | { 22 | public: { 23 | name: "Greater Lawrence Family Health Center (Pelham Street)", 24 | street: "147 Pelham St.", 25 | city: "Methuen", 26 | zip: "01844", 27 | signUpLink: 28 | "https://glfhccovid19iz.as.me/schedule.php?calendarID=5114836", 29 | }, 30 | private: { 31 | scrapeUrl: 32 | "https://glfhccovid19iz.as.me/schedule.php?action=showCalendar&fulldate=1&owner=21956779&template=weekly", 33 | calendar: "5114836", 34 | }, 35 | }, 36 | { 37 | public: { 38 | name: "Greater Lawrence Family Health Center (Central Plaza)", 39 | street: "2 Water St.", 40 | city: "Haverhill", 41 | zip: "01830", 42 | signUpLink: 43 | "https://glfhccovid19iz.as.me/schedule.php?location=Central+Plaza%2C+2+Water+Street%2C+Haverhill+MA", 44 | }, 45 | private: { 46 | scrapeUrl: 47 | "https://glfhccovid19iz.as.me/schedule.php?action=showCalendar&fulldate=1&owner=21956779&template=weekly&location=Central+Plaza%2C+2+Water+Street%2C+Haverhill+MA", 48 | 49 | calendar: "5236989", 50 | }, 51 | }, 52 | { 53 | public: { 54 | name: "Northern Essex Community College", 55 | street: "45 Franklin St.", 56 | city: "Lawrence", 57 | zip: "01840", 58 | restrictions: 59 | "Lawrence Residents only / Residentes de Lawrence solamente", 60 | signUpLink: 61 | "https://glfhccovid19iz.as.me/?location=45%20Franklin%20Street%2C%20Lawrence%20MA", 62 | }, 63 | private: { 64 | scrapeUrl: 65 | "https://glfhccovid19iz.as.me/schedule.php?action=showCalendar&fulldate=1&owner=21956779&template=weekly&location=45+Franklin+Street%2C+Lawrence+MA", 66 | calendar: "5341082", 67 | }, 68 | }, 69 | ]; 70 | 71 | module.exports = { 72 | provider, 73 | sites, 74 | }; 75 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/ZocDoc/config.js: -------------------------------------------------------------------------------- 1 | const scraperName = "ZocDoc"; 2 | 3 | /** 4 | * sites is essentially a map used for lookup by providerID (e.g., "pr_fSHH-Tyvm0SZvoK3pfH8tx|lo_EMLPse6C60qr6_M2rJmilx"). 5 | * The availability result objects returned by fetch() contain a providerID property, and this 6 | * is used to lookup the site details held here. 7 | */ 8 | const sites = { 9 | "pr_fSHH-Tyvm0SZvoK3pfH8tx|lo_EMLPse6C60qr6_M2rJmilx": { 10 | name: "Tufts Medical Center", 11 | street: "279 Tremont St", 12 | city: "Boston", 13 | zip: "02116", 14 | signUpLink: "https://www.tuftsmcvaccine.org", 15 | }, 16 | "pr_BDBebslqJU2vrCAvVMhYeh|lo_zDtK7NZWO0S_rgUqjxD1hB": { 17 | name: "Holtzman Medical Group - Mount Ida Campus", 18 | street: "777 Dedham St", 19 | city: "Newton", 20 | zip: "02459", 21 | signUpLink: "https://www.holtzmanmedical.org/covid19", 22 | }, 23 | "pr_CUmBnwtlz0C16bif5EU0IR|lo_X6zHHncQnkqyLp-rvpi1_R": { 24 | name: "AFC Urgent Care Springfield", 25 | street: "415 Cooley St, Unit 3", 26 | city: "Springfield", 27 | zip: "01128", 28 | signUpLink: 29 | "https://afcurgentcarespringfield.com/covid-vaccination-registration/", 30 | extraData: 31 | "Appointments for the vaccine are only available at AFC Urgent Care West Springfield at this time. ", 32 | }, 33 | "pr_4Vg_3ZeLY0aHJJxsCU-WhB|lo_VA_6br22m02Iu57vrHWtaB": { 34 | name: "AFC Urgent Care West Springfield", 35 | street: "18 Union St", 36 | city: "West Springfield", 37 | zip: "01089", 38 | signUpLink: 39 | "https://afcurgentcarespringfield.com/covid-vaccination-registration/", 40 | }, 41 | "pr_VUnpWUtg1k2WFBMK8IhZkx|lo_PhFQcSZjdUKZUHp63gDcmx": { 42 | name: "AFC Urgent Care Dedham", 43 | street: "370 Providence Hwy", 44 | city: "Dedham", 45 | zip: "02026", 46 | signUpLink: 47 | "https://afcurgentcarededham.com/covid-19-vaccine-registration/", 48 | }, 49 | "pr_iXjD9x2P-0OrLNoIknFr8R|lo_xmpTUhghfUC2n6cs3ZGHhh": { 50 | name: "AFC Urgent Care Saugus", 51 | street: "371 Broadway", 52 | city: "Saugus", 53 | zip: "01906", 54 | signUpLink: "https://afcurgentcaresaugus.com/covid-19-vaccination/", 55 | }, 56 | "pr_pEgrY3r5qEuYKsKvc4Kavx|lo_f3k2t812AUa9NTIYJpbuKx": { 57 | name: "AFC Urgent Care Worcester", 58 | street: "117A Stafford St", 59 | city: "Worcester", 60 | zip: "01603", 61 | signUpLink: "https://afcurgentcareworcester.com/", 62 | }, 63 | "pr_TeD-JuoydUKqszEn2ATb8h|lo_jbrpfIgELEWWL2j5d3t6Sh": { 64 | name: "AFC Urgent Care New Bedford", 65 | street: "119 Coggeshall St", 66 | city: "New Bedford", 67 | zip: "02746", 68 | signUpLink: "https://afcurgentcarenewbedford.com/", 69 | }, 70 | }; 71 | 72 | module.exports = { 73 | scraperName, 74 | sites, 75 | }; 76 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/BostonMedicalCenter/index.js: -------------------------------------------------------------------------------- 1 | const { entityName, settingsUrl, signUpLink } = require("./config"); 2 | const mychart = require("../../lib/MyChartAPI"); 3 | const moment = require("moment"); 4 | const { default: fetch } = require("node-fetch"); 5 | 6 | module.exports = async function GetAvailableAppointments() { 7 | console.log(`${entityName} staring.`); 8 | const webData = await ScrapeWebsiteData(); 9 | console.log(`${entityName} done.`); 10 | 11 | const results = Object.values(webData); 12 | for (const result of results) { 13 | result["signUpLink"] = signUpLink; 14 | } 15 | const timestamp = moment().format(); 16 | 17 | return { 18 | parentLocationName: "Boston Medical Center", 19 | isChain: true, 20 | timestamp, 21 | individualLocationData: results, 22 | }; 23 | }; 24 | 25 | async function ScrapeWebsiteData() { 26 | const data = await getSiteData(); 27 | 28 | // Request verification token to use in further access 29 | const [ 30 | cookie, 31 | verificationToken, 32 | ] = await mychart.GetCookieAndVerificationToken( 33 | `https://mychartscheduling.bmc.org/mychartscheduling/SignupAndSchedule/EmbeddedSchedule?id=${data.providerIds.join( 34 | "," 35 | )}&dept=${data.departmentIds.join(",")}&vt=${data.vt}&lang=en-US` 36 | ); 37 | 38 | return mychart.AddFutureWeeks( 39 | "mychartscheduling.bmc.org", 40 | `/MyChartscheduling/OpenScheduling/OpenScheduling/GetOpeningsForProvider?noCache=${Math.random()}`, 41 | cookie, 42 | verificationToken, 43 | 10, 44 | mychart.CommonPostDataCallback( 45 | data.providerIds, 46 | data.departmentIds, 47 | data.vt 48 | ) 49 | ); 50 | } 51 | 52 | /** 53 | * Get all possible provider and respective department IDs, and the vt (visit type) for BMC. 54 | * 55 | * Kudos to the folks at VaccineTimes 56 | * (https://github.com/ginkgobioworks/vaccinetime/blob/main/lib/sites/my_chart.rb) 57 | * who somehow discovered this settings URL. 58 | * 59 | * @returns 60 | */ 61 | async function getSiteData() { 62 | const text = await fetch(`${settingsUrl()}`) 63 | .then((res) => res.text()) 64 | .then((text) => { 65 | return text; 66 | }) 67 | .catch((error) => 68 | console.error(`failed to get BMC providers from setting: ${error}`) 69 | ); 70 | 71 | /* 72 | Looking for content in the following: 73 | 74 | url: "https://mychartscheduling.bmc.org/mychartscheduling/SignupAndSchedule/EmbeddedSchedule?id=10033909,10033319,10033364,10033367,10033370,10033706,10033083,10033373&dept=10098252,10098245,10098242,10098243,10098244,10105138,10108801,10098241&vt=2008", 75 | */ 76 | const relevantText = text.match(/EmbeddedSchedule.+/)[0]; 77 | const ids = relevantText.match(/id=([\d,]+)&/)[1].split(","); 78 | const deptIds = relevantText.match(/dept=([\d,]+)&/)[1].split(","); 79 | const vt = relevantText.match(/vt=(\d+)/)[1]; 80 | 81 | return { providerIds: ids, departmentIds: deptIds, vt: vt }; 82 | } 83 | -------------------------------------------------------------------------------- /test/GreaterLawrenceFHCTest.js: -------------------------------------------------------------------------------- 1 | const scraper = require("../no-browser-site-scrapers/GreaterLawrenceFHC/index"); 2 | 3 | const { expect } = require("chai"); 4 | const moment = require("moment"); 5 | const file = require("../lib/file"); 6 | 7 | /** Generator to feed filenames sequentially to the "with availability" test. */ 8 | function* filenames() { 9 | yield "noAvailability.html"; 10 | yield "noAvailability.html"; 11 | yield "sampleAvailability.html"; 12 | yield "noAvailability.html"; 13 | yield "noAvailability.html"; 14 | } 15 | 16 | describe("GLFHC availability test using scraper and saved HTML", function () { 17 | const filenameGenerator = filenames(); 18 | 19 | const testFetchService = { 20 | async fetchAvailability(/*site*/) { 21 | return loadTestHtmlFromFile( 22 | "GreaterLawrenceFHC", 23 | filenameGenerator.next().value 24 | ); 25 | }, 26 | }; 27 | const beforeTime = moment(); 28 | 29 | it("should provide availability for one site, and the results objects structure should conform", async function () { 30 | const results = await scraper(false, testFetchService); 31 | 32 | const expected = [false, false, true, false]; 33 | const hasAvailability = Object.values( 34 | results.individualLocationData 35 | ).map((result) => result.hasAvailability); 36 | const afterTime = moment(); 37 | 38 | expect(hasAvailability).is.deep.equal(expected); 39 | 40 | const expectedSlotCounts = [0, 0, 53, 0]; 41 | const slotCounts = Object.values(results.individualLocationData).map( 42 | (result) => 43 | Object.values(result.availability) 44 | .map((value) => value.numberAvailableAppointments) 45 | .reduce(function (total, number) { 46 | return total + number; 47 | }, 0) 48 | ); 49 | 50 | expect(expectedSlotCounts).is.deep.equal(slotCounts); 51 | /* 52 | Structure conformance expectations: 53 | 54 | - All the timestamps are expected to be between before 55 | and after when the scraper was executed 56 | - Each site's results object must have a property named "hasAvailability" 57 | */ 58 | results.individualLocationData.forEach((result) => { 59 | expect(moment(result.timestamp).isBetween(beforeTime, afterTime)); 60 | expect(result.hasAvailability).is.not.undefined; 61 | }); 62 | 63 | // console.log(`${JSON.stringify(results)}`); 64 | 65 | if (process.env.DEVELOPMENT) { 66 | file.write( 67 | `${process.cwd()}/out_no_browser.json`, 68 | `${JSON.stringify(results, null, " ")}` 69 | ); 70 | } 71 | }); 72 | }); 73 | 74 | /* utilities */ 75 | 76 | /** 77 | * Loads the saved HTML of the website. 78 | * 79 | * @param {String} filename The filename only, include its extension 80 | */ 81 | function loadTestHtmlFromFile(testFolderName, filename) { 82 | const fs = require("fs"); 83 | const path = `${process.cwd()}/test/${testFolderName}/${filename}`; 84 | return fs.readFileSync(path, "utf8"); 85 | } 86 | -------------------------------------------------------------------------------- /site-scrapers/Harrington/harrington-notes.txt: -------------------------------------------------------------------------------- 1 | An alternative to using Puppeteer, which seems to be quite flaky, is 2 | the following. 3 | 4 | /** **/ 5 | 6 | const fetch = require("node-fetch"); 7 | const htmlParser = require("node-html-parser"); 8 | 9 | 10 | async function getDailyAvailabilityCountsForMonth(page) { 11 | function reformatDate(date) { 12 | const dateObj = new Date(date + "T00:00:00"); 13 | return new Intl.DateTimeFormat("en-US").format(dateObj); 14 | } 15 | 16 | const noTimesAvailable = await page.$("#no-times-available-message"); 17 | const alertDanger = await page.$("#alert-danger"); 18 | if (noTimesAvailable || alertDanger) { 19 | return new Map(); 20 | } 21 | let dailySlotCountsMap; // keyed by date, value accumulates slot counts per date. 22 | const cookies = await page.cookies(); 23 | console.log(`cookies: ${JSON.stringify(cookies)}`); 24 | 25 | /* 26 | The fetch cookie property includes only device and PHPSESSID. 27 | 28 | 29 | */ 30 | 31 | const moment = require("moment"); 32 | const midnightToday = moment().local().startOf("day"); 33 | console.log(`${midnightToday.format("YYYY-MM-DD")}`); 34 | 35 | const html = await fetch( 36 | "https://app.acuityscheduling.com/schedule.php?action=showCalendar&fulldate=1&owner=22192301&template=weekly", 37 | { 38 | headers: { 39 | accept: "*/*", 40 | "accept-language": "en-US,en;q=0.9", 41 | "content-type": 42 | "application/x-www-form-urlencoded; charset=UTF-8", 43 | "sec-ch-ua": 44 | '"Google Chrome";v="89", "Chromium";v="89", ";Not A Brand";v="99"', 45 | "sec-ch-ua-mobile": "?0", 46 | "sec-fetch-dest": "empty", 47 | "sec-fetch-mode": "cors", 48 | "sec-fetch-site": "same-origin", 49 | "x-requested-with": "XMLHttpRequest", 50 | cookie: 51 | "device_id=b7edec36-9f66-42ce-b93d-1981d9aee39e; PHPSESSID=731oholt85l2a8m2ccdttmi9bk", 52 | }, 53 | referrer: 54 | "https://app.acuityscheduling.com/schedule.php?owner=22192301&calendarID=5202050", 55 | referrerPolicy: "same-origin", 56 | body: [ 57 | "type=20926295", 58 | "calendar=5202050", 59 | `month=${midnightToday}`, 60 | "skip=true", 61 | "options%5Bnextprev%5D%5B2021-04-26%5D=2021-03-27", 62 | "options%5BnumDays%5D=3", 63 | "ignoreAppointment=", 64 | "appointmentType=", 65 | "calendarID=5202050", 66 | ].join("&"), 67 | method: "POST", 68 | mode: "cors", 69 | } 70 | ) 71 | .then((response) => response.text()) 72 | .then((text) => { 73 | return text; 74 | }) 75 | .catch((error) => { 76 | console.error("Error:", error); 77 | }); 78 | console.log(`html returned from fetch: ${html}`); 79 | const parsed = htmlParser.parse(html); 80 | const notTimesMessage = parsed.querySelector("#no-times-available-message"); 81 | console.log(`no times message: ${notTimesMessage.innerHTML}`); 82 | } 83 | -------------------------------------------------------------------------------- /.github/workflows/prod_deploy.yaml: -------------------------------------------------------------------------------- 1 | name: Workflow 2 - SAM Validate, Build, Deploy 2 | on: 3 | workflow_run: 4 | workflows: ["Workflow 1 - Run Test Suite"] 5 | branches: [master] 6 | types: 7 | - completed 8 | 9 | jobs: 10 | sam-validate-build-deploy: 11 | if: ${{ github.repository_owner == 'livgust' }} 12 | runs-on: ubuntu-latest 13 | outputs: 14 | env-name: ${{ steps.env-name.outputs.environment }} 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Install Dependencies with Apt Get 18 | run: | 19 | sudo apt-get update 20 | sudo apt-get install python3.8 jq -y 21 | - name: Configure AWS credentials 22 | id: creds 23 | uses: aws-actions/configure-aws-credentials@v1 24 | with: 25 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 26 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 27 | aws-region: ${{ secrets.AWS_REGION }} 28 | - name: Configure variables 29 | shell: bash 30 | id: vars 31 | env: 32 | REPO: ${{ github.repository }} 33 | HASH: ${{ github.sha }} 34 | REF: ${{ github.ref }} 35 | run: | 36 | # Set variables 37 | BRANCH=${REF#refs/heads/} 38 | REPOSITORY=`echo $REPO | tr "/" "-"` 39 | ENVIRONMENT=$BRANCH-$REPOSITORY-${{ secrets.AWS_REGION }} 40 | # In this step we are setting variables and persistenting them 41 | # into the environment so that they can be utilized in other steps 42 | echo "::set-output name=branch::$BRANCH" 43 | echo "::set-output name=repository::$REPOSITORY" 44 | echo "::set-output name=environment::$ENVIRONMENT" 45 | # Output variables to ensure their values are set correctly when run 46 | echo "The region is ${{ secrets.AWS_REGION }}" 47 | echo "The repository is $REPOSITORY" 48 | echo "The environment is $ENVIRONMENT" 49 | echo "The branch is $BRANCH" 50 | - name: SAM Build 51 | run: | 52 | sam build 53 | - name: SAM Validate 54 | run: | 55 | sam validate -t ./.aws-sam/build/template.yaml 56 | - name: SAM Deploy 57 | run: | 58 | # Run SAM Deploy 59 | sam deploy --config-env prod --no-fail-on-empty-changeset --no-confirm-changeset \ 60 | --parameter-overrides GoogleApiKey=${{ secrets.GOOGLE_API_KEY }} \ 61 | WalgreensEmail=${{ secrets.WALGREENS_EMAIL }} \ 62 | WalgreensPassword=${{ secrets.WALGREENS_PASSWORD }} \ 63 | WalgreensChallenge=${{ secrets.WALGREENS_CHALLENGE }} \ 64 | FaunaApiKey=${{ secrets.FAUNA_DB_PROD }} \ 65 | NodeEnv=production \ 66 | BucketName=${{ secrets.AWSS3BUCKETNAME }} \ 67 | RecaptchaToken=${{ secrets.RECAPTCHA_TOKEN }} \ 68 | SlackWebhook=${{ secrets.SLACK_WEBHOOK_BOT_CHANNEL }} \ 69 | TwitterApiKey=${{ secrets.TWITTER_API_KEY }} \ 70 | TwitterApiSecret=${{ secrets.TWITTER_API_SECRET }} \ 71 | TwitterAccessTokenKey=${{ secrets.TWITTER_ACCESS_TOKEN_KEY }} \ 72 | TwitterAccessTokenSecret=${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }} \ 73 | PinpointApplicationId=${{ secrets.PINPOINT_APPLICATION_ID }} \ 74 | PinpointOriginationNumber=${{ secrets.PINPOINT_ORIGINATION_NUMBER }} \ 75 | Environment=prod 76 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/HealthMart/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const https = require("https"); 3 | const crypto = require("crypto"); 4 | const moment = require("moment"); 5 | const { toTitleCase } = require("../../lib/stringUtil"); 6 | 7 | module.exports = async function GetAvailableAppointments() { 8 | console.log(`${site.name} starting.`); 9 | const webData = await ScrapeWebsiteData(); 10 | const individualLocationData = Object.values(webData).map( 11 | (responseLocation) => { 12 | let hasAvailability = false; 13 | const availability = {}; 14 | if (responseLocation && responseLocation.visibleTimeSlots) { 15 | hasAvailability = !!responseLocation.visibleTimeSlots.length; 16 | for (const timeSlot of responseLocation.visibleTimeSlots) { 17 | //strip time so we group by date, but then force midnight local time zone to avoid UTC dateline issues 18 | const date = new Date( 19 | `${timeSlot.time.split("T")[0]}T00:00` 20 | ); 21 | const formattedDate = `${ 22 | date.getMonth() + 1 23 | }/${date.getDate()}/${date.getFullYear()}`; 24 | if (!availability[formattedDate]) { 25 | availability[formattedDate] = { 26 | hasAvailability: true, 27 | numberAvailableAppointments: 0, 28 | }; 29 | } 30 | availability[ 31 | formattedDate 32 | ].numberAvailableAppointments += 1; 33 | } 34 | } 35 | return { 36 | name: responseLocation.name, 37 | street: toTitleCase(responseLocation.address1), 38 | city: toTitleCase(responseLocation.city), 39 | state: responseLocation.state, 40 | zip: responseLocation.zipCode, 41 | hasAvailability, 42 | availability, 43 | signUpLink: site.signUpLink, 44 | }; 45 | } 46 | ); 47 | console.log(`${site.name} done.`); 48 | return { 49 | parentLocationName: "Health Mart Pharmacy", 50 | isChain: true, 51 | individualLocationData, 52 | timestamp: moment().format(), 53 | }; 54 | }; 55 | 56 | function md5HashString(string) { 57 | return crypto.createHash("md5").update(string).digest("hex"); 58 | } 59 | 60 | async function ScrapeWebsiteData() { 61 | const rawData = {}; 62 | for (const zip of site.zips) { 63 | const url = [site.websiteRoot, zip].join("/") + "?state=MA&userAge=21"; 64 | const getUrl = new Promise((resolve) => { 65 | let response = ""; 66 | https.get(url, { rejectUnauthorized: false }, (res) => { 67 | let body = ""; 68 | res.on("data", (chunk) => (body += chunk)); 69 | res.on("end", () => { 70 | response = JSON.parse(body); 71 | resolve(response); 72 | }); 73 | }); 74 | }); 75 | const responseJson = await getUrl; 76 | responseJson.map((responseLoc) => { 77 | const responseLocHash = md5HashString( 78 | responseLoc.address1 + responseLoc.city 79 | ); 80 | rawData[responseLocHash] = responseLoc; 81 | }); 82 | } 83 | return rawData; 84 | } 85 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/PriceChopper/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const https = require("https"); 3 | const crypto = require("crypto"); 4 | const moment = require("moment"); 5 | const { toTitleCase } = require("../../lib/stringUtil"); 6 | 7 | module.exports = async function GetAvailableAppointments() { 8 | console.log(`${site.name} starting.`); 9 | const webData = await ScrapeWebsiteData(); 10 | const individualLocationData = Object.values(webData).map( 11 | (responseLocation) => { 12 | let hasAvailability = false; 13 | const availability = {}; 14 | if (responseLocation && responseLocation.visibleTimeSlots) { 15 | hasAvailability = !!responseLocation.visibleTimeSlots.length; 16 | for (const timeSlot of responseLocation.visibleTimeSlots) { 17 | //strip time so we group by date, but then force midnight local time zone to avoid UTC dateline issues 18 | const date = new Date( 19 | `${timeSlot.time.split("T")[0]}T00:00` 20 | ); 21 | const formattedDate = `${ 22 | date.getMonth() + 1 23 | }/${date.getDate()}/${date.getFullYear()}`; 24 | if (!availability[formattedDate]) { 25 | availability[formattedDate] = { 26 | hasAvailability: true, 27 | numberAvailableAppointments: 0, 28 | }; 29 | } 30 | availability[ 31 | formattedDate 32 | ].numberAvailableAppointments += 1; 33 | } 34 | } 35 | return { 36 | name: `Price Chopper (${toTitleCase(responseLocation.city)})`, 37 | street: toTitleCase(responseLocation.address1), 38 | city: toTitleCase(responseLocation.city), 39 | state: responseLocation.state, 40 | zip: responseLocation.zipCode, 41 | hasAvailability, 42 | availability, 43 | signUpLink: site.signUpLink, 44 | }; 45 | } 46 | ); 47 | console.log(`${site.name} done.`); 48 | return { 49 | parentLocationName: "Price Chopper", 50 | isChain: true, 51 | individualLocationData, 52 | timestamp: moment().format(), 53 | }; 54 | }; 55 | 56 | function md5HashString(string) { 57 | return crypto.createHash("md5").update(string).digest("hex"); 58 | } 59 | 60 | async function ScrapeWebsiteData() { 61 | const rawData = {}; 62 | for (const zip of site.zips) { 63 | const url = [site.websiteRoot, zip].join("/") + "?state=MA&userAge=21"; 64 | const getUrl = new Promise((resolve) => { 65 | let response = ""; 66 | https.get(url, { rejectUnauthorized: false }, (res) => { 67 | let body = ""; 68 | res.on("data", (chunk) => (body += chunk)); 69 | res.on("end", () => { 70 | response = JSON.parse(body); 71 | resolve(response); 72 | }); 73 | }); 74 | }); 75 | const responseJson = await getUrl; 76 | responseJson.map((responseLoc) => { 77 | const responseLocHash = md5HashString( 78 | responseLoc.address1 + responseLoc.city 79 | ); 80 | rawData[responseLocHash] = responseLoc; 81 | }); 82 | } 83 | return rawData; 84 | } 85 | -------------------------------------------------------------------------------- /test/alerts/determine_recipients.js: -------------------------------------------------------------------------------- 1 | const sinon = require("sinon"); 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const signUp = require("../../alerts/sign_up"); 5 | const pinpointWorkflow = require("../../alerts/pinpoint/new_subscriber"); 6 | const determineRecipients = require("../../alerts/determine_recipients"); 7 | const maZips = require("../../data/ma-zips.json"); 8 | 9 | async function addSubscription(phone, zip) { 10 | if (!pinpointWorkflow.validateNumberAndAddSubscriber.restore?.sinon) { 11 | sinon 12 | .stub(pinpointWorkflow, "validateNumberAndAddSubscriber") 13 | .returns(Promise.resolve()); 14 | } 15 | return signUp 16 | .addOrUpdateSubscription({ 17 | phoneNumber: phone || "8578675309", 18 | zip: zip || "01880", 19 | radius: 10, 20 | }) 21 | .then(() => 22 | signUp.activateSubscription({ phoneNumber: phone || "8578675309" }) 23 | ); 24 | } 25 | 26 | afterEach(() => { 27 | sinon.restore(); 28 | }); 29 | 30 | (process.env.FAUNA_DB ? describe : describe.skip)( 31 | "findSubscribersWithZips", 32 | () => { 33 | it("finds a recipient with a given zip", async () => { 34 | await addSubscription(); 35 | expect( 36 | await determineRecipients.findSubscribersWithZips(["01880"]) 37 | ).to.include.deep.members([ 38 | { 39 | phoneNumber: "8578675309", 40 | zip: "01880", 41 | radius: 10, 42 | active: true, 43 | cancelled: false, 44 | }, 45 | ]); 46 | }); 47 | } 48 | ); 49 | 50 | (process.env.FAUNA_DB ? describe : describe.skip)("determineRecipients", () => { 51 | afterEach(() => { 52 | sinon.restore(); 53 | }); 54 | 55 | it("finds easy recipient", async () => { 56 | await addSubscription(); 57 | expect( 58 | ( 59 | await determineRecipients.determineRecipients({ 60 | locations: [maZips["01880"]], 61 | numberAvailable: 1, 62 | }) 63 | ).textRecipients 64 | ).to.include.deep.members([ 65 | { 66 | phoneNumber: "8578675309", 67 | zip: "01880", 68 | radius: 10, 69 | active: true, 70 | cancelled: false, 71 | }, 72 | ]); 73 | }); 74 | 75 | it("works with multiple locations", async () => { 76 | await addSubscription("1234567890", "01880"); 77 | await addSubscription("0123456789", "01238"); 78 | expect( 79 | ( 80 | await determineRecipients.determineRecipients({ 81 | locations: [maZips["01880"], maZips["01238"]], 82 | numberAvailable: 1, 83 | }) 84 | ).textRecipients 85 | ).to.include.deep.members([ 86 | { 87 | phoneNumber: "1234567890", 88 | zip: "01880", 89 | radius: 10, 90 | active: true, 91 | cancelled: false, 92 | }, 93 | { 94 | phoneNumber: "0123456789", 95 | zip: "01238", 96 | radius: 10, 97 | active: true, 98 | cancelled: false, 99 | }, 100 | ]); 101 | }); 102 | }); 103 | -------------------------------------------------------------------------------- /lib/db/utils.js: -------------------------------------------------------------------------------- 1 | const dotenv = require("dotenv"); 2 | dotenv.config(); 3 | 4 | const faunadb = require("faunadb"), 5 | fq = faunadb.query; 6 | 7 | const DEBUG = false; // set to false to disable debugging 8 | function debugLog(...args) { 9 | if (DEBUG) { 10 | console.log(...args); 11 | } 12 | } 13 | 14 | const client = new faunadb.Client({ secret: process.env.FAUNA_DB }); 15 | 16 | async function faunaQuery(query) { 17 | try { 18 | debugLog(`trying to execute query ${JSON.stringify(query)}`); 19 | const res = await client.query(query); 20 | debugLog( 21 | `successfully executed query ${JSON.stringify( 22 | query 23 | )}, got res ${JSON.stringify(res)}` 24 | ); 25 | return res; 26 | } catch (error) { 27 | console.error(`for query ${JSON.stringify(query)}, got error ${error}`); 28 | console.error(error.description); 29 | throw error; 30 | } 31 | } 32 | async function generateId() { 33 | return faunaQuery(fq.NewId()); 34 | } 35 | /* 36 | Basic CRUD operations. 37 | */ 38 | async function retrieveItemByRefId(collectionName, refId) { 39 | return faunaQuery( 40 | fq.Get(fq.Ref(fq.Collection(collectionName), refId)) 41 | ).then((res) => { 42 | debugLog( 43 | `querying ${collectionName} collection with refId ${refId} and got result ${JSON.stringify( 44 | res 45 | )}` 46 | ); 47 | return res; 48 | }); 49 | } 50 | 51 | /* returns null if the object doesn't exist */ 52 | async function retrieveItemByRefIdIfExists(collectionName, refId) { 53 | return faunaQuery( 54 | fq.If( 55 | fq.Exists(fq.Ref(fq.Collection(collectionName), refId)), 56 | fq.Get(fq.Ref(fq.Collection(collectionName), refId)), 57 | null 58 | ) 59 | ); 60 | } 61 | 62 | async function retrieveItemsByRefIds(collectionName, refIds) { 63 | const queries = refIds.map((refId) => 64 | fq.Get(fq.Ref(fq.Collection(collectionName), refId)) 65 | ); 66 | return faunaQuery(queries).then((res) => { 67 | debugLog( 68 | `querying ${collectionName} collection with refIds ${refIds} and got result ${JSON.stringify( 69 | res 70 | )}` 71 | ); 72 | return res; 73 | }); 74 | } 75 | 76 | async function checkItemExistsByRefId(collectionName, refId) { 77 | return faunaQuery(fq.Exists(fq.Ref(fq.Collection(collectionName), refId))); 78 | } 79 | 80 | async function checkItemsExistByRefIds(collectionName, refIds) { 81 | const queries = refIds.map((refId) => 82 | fq.Exists(fq.Ref(fq.Collection(collectionName), refId)) 83 | ); 84 | return faunaQuery(queries); 85 | } 86 | 87 | async function deleteItemByRefId(collectionName, refId) { 88 | return faunaQuery(fq.Delete(fq.Ref(fq.Collection(collectionName), refId))); 89 | } 90 | 91 | async function deleteItemsByRefIds(collectionName, refIds) { 92 | const queries = refIds.map((refId) => 93 | fq.Delete(fq.Ref(fq.Collection(collectionName), refId)) 94 | ); 95 | return faunaQuery(queries); 96 | } 97 | module.exports = { 98 | /*********/ 99 | faunaQuery, 100 | /*********/ 101 | checkItemExistsByRefId, 102 | checkItemsExistByRefIds, 103 | debugLog, 104 | deleteItemByRefId, 105 | deleteItemsByRefIds, 106 | generateId, 107 | retrieveItemByRefId, 108 | retrieveItemsByRefIds, 109 | retrieveItemByRefIdIfExists, 110 | }; 111 | -------------------------------------------------------------------------------- /site-scrapers/StopAndShop/config.js: -------------------------------------------------------------------------------- 1 | const site = { 2 | name: "Stop & Shop", 3 | website: 4 | "https://stopandshopsched.rxtouch.com/rbssched/program/covid19/Patient/Advisory", 5 | locations: [ 6 | { city: "Abington", zip: "02351" }, 7 | { city: "Amesbury", zip: "01913" }, 8 | { city: "Arlington", zip: "02476" }, 9 | { city: "Attleboro", zip: "02703" }, 10 | { city: "Bedford", zip: "01730" }, 11 | { city: "Belchertown", zip: "01007" }, 12 | { city: "Bourne", zip: "02532" }, 13 | { city: "Braintree", zip: "02184" }, 14 | { city: "Brockton", zip: "02301" }, 15 | { city: "Chicopee", zip: "01020" }, 16 | { city: "Cohasset", zip: "02025" }, 17 | { city: "Dartmouth", zip: "02747" }, 18 | { city: "Dedham", zip: "02026" }, 19 | { city: "Dorchester", zip: "02125" }, 20 | { city: "Dorchester", zip: "02121" }, 21 | { city: "Dorchester", zip: "02122" }, 22 | { city: "East Longmeadow", zip: "01028" }, 23 | { city: "Edgartown", zip: "02539" }, 24 | { city: "Fairhaven", zip: "02719" }, 25 | { city: "Fall River", zip: "02721" }, 26 | { city: "Falmouth", zip: "02536" }, 27 | { city: "Feeding Hills", zip: "01030" }, 28 | { city: "Foxboro", zip: "02035" }, 29 | { city: "Framingham", zip: "01701" }, 30 | { city: "Franklin", zip: "02038" }, 31 | { city: "Greenfield", zip: "01301" }, 32 | { city: "Hadley", zip: "01035" }, 33 | { city: "Halifax", zip: "02338" }, 34 | { city: "Harwich", zip: "02645" }, 35 | { city: "Hingham", zip: "02043" }, 36 | { city: "Holyoke", zip: "01040" }, 37 | { city: "Hyannis", zip: "02601" }, 38 | { city: "Jamaica Plain", zip: "02130" }, 39 | { city: "Kingston", zip: "02364" }, 40 | { city: "Lynn", zip: "01904" }, 41 | { city: "Malden", zip: "02148" }, 42 | { city: "Mansfield", zip: "02048" }, 43 | { city: "Mashpee", zip: "02649" }, 44 | { city: "Medford", zip: "02155" }, 45 | { city: "Milford", zip: "01757" }, 46 | { city: "Natick", zip: "01760" }, 47 | { city: "North Adams", zip: "01247" }, 48 | { city: "North Andover", zip: "01845" }, 49 | { city: "New Bedford", zip: "02745" }, 50 | { city: "New Bedford", zip: "02740" }, 51 | { city: "North Attleboro", zip: "02760" }, 52 | { city: "Northampton", zip: "01060" }, 53 | { city: "Norwell", zip: "02061" }, 54 | { city: "Orleans", zip: "02653" }, 55 | { city: "Pembroke", zip: "02359" }, 56 | { city: "Pittsfield", zip: "01201" }, 57 | { city: "Plymouth", zip: "02360" }, 58 | { city: "Provincetown", zip: "02657" }, 59 | { city: "Quincy", zip: "02169" }, 60 | { city: "Quincy", zip: "02171" }, 61 | { city: "Revere", zip: "02151" }, 62 | { city: "Sandwich", zip: "02563" }, 63 | { city: "Saugus", zip: "01906" }, 64 | { city: "Seekonk", zip: "02771" }, 65 | { city: "Somerset", zip: "02725" }, 66 | { city: "Springfield", zip: "01104" }, 67 | { city: "Springfield", zip: "01129" }, 68 | { city: "Springfield", zip: "01119" }, 69 | { city: "Swampscott", zip: "01907" }, 70 | { city: "W. Springfield", zip: "01089" }, 71 | { city: "Westborough", zip: "01581" }, 72 | { city: "Westfield", zip: "01085" }, 73 | { city: "Woburn", zip: "01801" }, 74 | { city: "Worcester", zip: "01605" }, 75 | { city: "Worcester", zip: "01604" }, 76 | ], 77 | }; 78 | 79 | module.exports = { 80 | site, 81 | }; 82 | -------------------------------------------------------------------------------- /test/WalgreensDataFormatter.js: -------------------------------------------------------------------------------- 1 | const { formatData } = require("./../site-scrapers/Walgreens/dataFormatter"); 2 | const chai = require("chai"); 3 | const moment = require("moment"); 4 | const expect = chai.expect; 5 | 6 | const now = moment().local(); 7 | const yesterday = moment().subtract(1, "days").local(); 8 | const tomorrow = moment().add(1, "days").local(); 9 | 10 | function removeTimestamps(results) { 11 | return results.map((entry) => { 12 | const { timestamp, ...rest } = entry; 13 | return rest; 14 | }); 15 | } 16 | 17 | const fakeWebsite = "www.example.com/sign-up"; 18 | const exampleEntry = { 19 | locationId: "9148a7ed-9dd4-4062-9cb5-46277d9b5b3d", 20 | name: "Walgreen Drug Store", 21 | partnerLocationId: "5879", 22 | description: "", 23 | logoURL: 24 | "/images/adaptive/pharmacy/healthcenter/health-navigator/Walgreens_logo_2X.png", 25 | distance: 12.94, 26 | position: { latitude: 34.063943, longitude: -118.291847 }, 27 | address: { 28 | line1: "3201 W 6TH ST", 29 | line2: "", 30 | city: "LOS ANGELES", 31 | state: "CA", 32 | country: "US", 33 | zip: "90020", 34 | }, 35 | categories: [ 36 | { 37 | code: "2", 38 | display: "Immunizations", 39 | services: [{ code: "99", display: "COVID-19 Vaccine" }], 40 | }, 41 | ], 42 | orgId: "Organization/35860656-84da-43fd-b66f-47e81b483e3b", 43 | phone: [ 44 | { type: "StorePrimary", number: "213-251-0179" }, 45 | { type: "StoreSecondary", number: "" }, 46 | { type: "Pharmacy", number: "213-251-0179" }, 47 | ], 48 | fhirLocationId: "9148a7ed-9dd4-4062-9cb5-46277d9b5b3d", 49 | storenumber: "5879", 50 | appointmentAvailability: [ 51 | { 52 | date: yesterday.format("YYYY-MM-DD"), 53 | day: yesterday.format("dddd"), 54 | slots: ["12:00 pm"], 55 | }, 56 | { 57 | date: now.format("YYYY-MM-DD"), 58 | day: now.dddd, 59 | slots: [now.format("HH:mm a")], 60 | }, 61 | { 62 | date: tomorrow.format("YYYY-MM-DD"), 63 | day: tomorrow.format("dddd"), 64 | slots: ["09:30 am", "09:45 am", "10:00 am"], 65 | }, 66 | ], 67 | }; 68 | 69 | describe("WalgreensDataFormatter formatData", () => { 70 | const { signUpLink, hasAvailability, availability, ...result } = formatData( 71 | [exampleEntry], 72 | fakeWebsite 73 | )[0]; 74 | 75 | it("formats the name and address", () => { 76 | expect(removeTimestamps([result])).to.be.deep.equal([ 77 | { 78 | name: "Walgreens (Los Angeles)", 79 | street: "3201 W 6th St", 80 | city: "Los Angeles", 81 | zip: "90020", 82 | }, 83 | ]); 84 | }); 85 | 86 | it("filters out and formats availability", () => { 87 | const expectedAvailability = {}; 88 | expectedAvailability[now.format("M/D/YYYY")] = { 89 | hasAvailability: false, 90 | numberAvailableAppointments: 0, 91 | }; 92 | expectedAvailability[tomorrow.format("M/D/YYYY")] = { 93 | hasAvailability: true, 94 | numberAvailableAppointments: 3, 95 | }; 96 | 97 | expect(hasAvailability).to.be.true; 98 | expect(availability).to.be.deep.equal(expectedAvailability); 99 | }); 100 | 101 | it("passes the sign up link", () => { 102 | expect(signUpLink).to.be.equal(fakeWebsite); 103 | }); 104 | }); 105 | -------------------------------------------------------------------------------- /test/metrics.js: -------------------------------------------------------------------------------- 1 | const { getTotalNumberOfAppointments } = require("../lib/metrics"); 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | 5 | describe("metrics getTotalNumberOfAppointments", () => { 6 | it("works for null result", () => { 7 | expect(getTotalNumberOfAppointments(null)).to.be.equal(0); 8 | }); 9 | 10 | it("works for single result with no date availability", () => { 11 | expect( 12 | getTotalNumberOfAppointments({ hasAvailability: true }) 13 | ).to.be.equal(1); 14 | }); 15 | 16 | it("works for single result with date availability", () => { 17 | expect( 18 | getTotalNumberOfAppointments({ 19 | availability: { 20 | "2021-01-01": { numberAvailableAppointments: 5 }, 21 | "2021-01-02": { numberAvailableAppointments: 3 }, 22 | }, 23 | hasAvailability: true, 24 | signUpLink: "www.example.com", 25 | }) 26 | ).to.be.equal(8); 27 | }); 28 | 29 | it("works for array with date availability", () => { 30 | expect( 31 | getTotalNumberOfAppointments([ 32 | { 33 | availability: { 34 | "2021-01-01": { numberAvailableAppointments: 5 }, 35 | "2021-01-02": { numberAvailableAppointments: 3 }, 36 | }, 37 | hasAvailability: true, 38 | signUpLink: "www.example.com", 39 | }, 40 | { 41 | availability: { 42 | "2021-01-01": { numberAvailableAppointments: 10 }, 43 | "2021-01-02": { numberAvailableAppointments: 20 }, 44 | }, 45 | hasAvailability: true, 46 | signUpLink: "www.example.com", 47 | }, 48 | ]) 49 | ).to.be.equal(38); 50 | }); 51 | 52 | it("ignores when no sign-up link is present", () => { 53 | expect( 54 | getTotalNumberOfAppointments([ 55 | { 56 | availability: { 57 | "2021-01-01": { numberAvailableAppointments: 5 }, 58 | "2021-01-02": { numberAvailableAppointments: 3 }, 59 | }, 60 | hasAvailability: true, 61 | signUpLink: null, 62 | }, 63 | ]) 64 | ).to.be.equal(0); 65 | expect( 66 | getTotalNumberOfAppointments([ 67 | { 68 | availability: { 69 | "2021-01-01": { 70 | numberAvailableAppointments: 5, 71 | signUpLink: "www.example.com", 72 | }, 73 | "2021-01-02": { 74 | numberAvailableAppointments: 3, 75 | signUpLink: null, 76 | }, 77 | }, 78 | hasAvailability: true, 79 | }, 80 | ]) 81 | ).to.be.equal(5); 82 | }); 83 | 84 | it("ignores when hasAvailability is false", () => { 85 | expect( 86 | getTotalNumberOfAppointments([ 87 | { 88 | availability: { 89 | "2021-01-01": { numberAvailableAppointments: 5 }, 90 | "2021-01-02": { numberAvailableAppointments: 3 }, 91 | }, 92 | hasAvailability: false, 93 | signUpLink: "www.example.com", 94 | }, 95 | ]) 96 | ).to.be.equal(0); 97 | }); 98 | }); 99 | -------------------------------------------------------------------------------- /test/alerts/sign_up.js: -------------------------------------------------------------------------------- 1 | const sinon = require("sinon"); 2 | const pinpointWorkflow = require("../../alerts/pinpoint/new_subscriber"); 3 | const signUp = require("../../alerts/sign_up"); 4 | const chai = require("chai"); 5 | const expect = chai.expect; 6 | const moment = require("moment"); 7 | const { faunaQuery } = require("../../lib/db/utils"); 8 | 9 | (process.env.FAUNA_DB ? describe : describe.skip)("getSubscription", () => { 10 | it("errors if you send both phone and email", async () => { 11 | let e; 12 | try { 13 | await signUp.getSubscription({ 14 | phoneNumber: "8578675309", 15 | email: "email@test.com", 16 | }); 17 | } catch (error) { 18 | e = error; 19 | } 20 | expect(e).to.be.instanceOf(Error); 21 | }); 22 | }); 23 | (process.env.FAUNA_DB ? describe : describe.skip)("addSubscription", () => { 24 | it("sets and retrieves a phone number subscription", async () => { 25 | sinon 26 | .stub(pinpointWorkflow, "validateNumberAndAddSubscriber") 27 | .returns(Promise.resolve()); 28 | await signUp.addOrUpdateSubscription({ 29 | phoneNumber: "8578675309", 30 | zip: "12345", 31 | radius: 100, 32 | }); 33 | expect( 34 | await signUp.getSubscription({ phoneNumber: "8578675309" }) 35 | ).to.deep.equal({ 36 | phoneNumber: "8578675309", 37 | zip: "12345", 38 | radius: 100, 39 | active: false, 40 | cancelled: false, 41 | }); 42 | }); 43 | it("sets and retrieves an email subscription", async () => { 44 | await signUp.addOrUpdateSubscription({ 45 | email: "test@example.com", 46 | zip: "12345", 47 | radius: 100, 48 | }); 49 | expect( 50 | await signUp.getSubscription({ email: "test@example.com" }) 51 | ).to.deep.equal({ 52 | email: "test@example.com", 53 | zip: "12345", 54 | radius: 100, 55 | active: false, 56 | cancelled: false, 57 | }); 58 | }); 59 | }); 60 | 61 | (process.env.FAUNA_DB ? describe : describe.skip)( 62 | "activateSubscripition", 63 | () => { 64 | it("sets active to true", async () => { 65 | await signUp.addOrUpdateSubscription({ 66 | email: "test@example.com", 67 | zip: "12345", 68 | radius: 100, 69 | }); 70 | await signUp.activateSubscription({ email: "test@example.com" }); 71 | expect( 72 | await signUp.getSubscription({ email: "test@example.com" }) 73 | ).to.deep.equal({ 74 | email: "test@example.com", 75 | zip: "12345", 76 | radius: 100, 77 | active: true, 78 | cancelled: false, 79 | }); 80 | }); 81 | } 82 | ); 83 | 84 | (process.env.FAUNA_DB ? describe : describe.skip)("cancelSubscripition", () => { 85 | it("sets cancelled to true", async () => { 86 | await signUp.addOrUpdateSubscription({ 87 | email: "test@example.com", 88 | zip: "12345", 89 | radius: 100, 90 | }); 91 | await signUp.cancelSubscription({ email: "test@example.com" }); 92 | expect( 93 | await signUp.getSubscription({ email: "test@example.com" }) 94 | ).to.deep.equal({ 95 | email: "test@example.com", 96 | zip: "12345", 97 | radius: 100, 98 | active: false, 99 | cancelled: true, 100 | }); 101 | }); 102 | }); 103 | -------------------------------------------------------------------------------- /site-scrapers/MercyMedicalCenter/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const moment = require("moment"); 3 | const s3 = require("../../lib/s3"); 4 | const { sendSlackMsg } = require("../../lib/slack"); 5 | const html_parser = require("node-html-parser"); 6 | const { JSDOM } = require("jsdom"); 7 | 8 | module.exports = async function GetAvailableAppointments(browser) { 9 | console.log(`${site.name} starting.`); 10 | const webData = await ScrapeWebsiteData(browser); 11 | console.log(`${site.name} done.`); 12 | const { noAppointments, timeslotsUrl, ...restSite } = site; 13 | return { 14 | parentLocationName: "Mercy Medical Center", 15 | timestamp: moment().format(), 16 | individualLocationData: [ 17 | { 18 | ...restSite, 19 | ...webData, 20 | }, 21 | ], 22 | }; 23 | }; 24 | 25 | async function jQueryPost(page, url, data) { 26 | return await page.evaluate( 27 | async (url, data) => { 28 | return await new Promise((resolve) => { 29 | $.post(url, data, function (data) { 30 | resolve(data); 31 | }); 32 | }); 33 | }, 34 | url, 35 | data 36 | ); 37 | } 38 | 39 | async function getTimeSlotsForDate(page, ScheduleDay) { 40 | const url = site.timeslotsUrl; 41 | const data = { 42 | ScheduleDay, 43 | }; 44 | return await jQueryPost(page, url, data); 45 | } 46 | 47 | async function ScrapeWebsiteData(browser) { 48 | const page = await browser.newPage(); 49 | await page.goto(site.signUpLink, { waitUntil: "networkidle0" }); 50 | await page.waitForSelector("form"); 51 | await page.evaluate(() => { 52 | document.querySelector("button[name='SiteName']").click(); 53 | }); 54 | // Wait for form to post 55 | await page.waitForSelector("button[onclick*='ScheduleDay']"); 56 | const daysToCheck = 11; // we could check more but it looks like they only allow 11 days right now. 57 | 58 | let hasAvailability = false; 59 | let availability = {}; 60 | 61 | for (let i = 0; i < daysToCheck; i++) { 62 | const date = moment().local().add(i, "days").format("MM/DD/YYYY"); 63 | const xhrRes = await getTimeSlotsForDate(page, date); 64 | // xhrRes is html in this case so we can just string match regex 65 | // but we also match any html so we know we get a good response 66 | const someHTMLRegex = /div/; 67 | const someHTML = xhrRes.match(someHTMLRegex); 68 | const noAppointments = xhrRes.match(site.noAppointments); 69 | if (someHTML && !noAppointments) { 70 | const dom = new JSDOM(xhrRes); 71 | const reserveButtons = dom.window.document.querySelectorAll( 72 | "button[onclick*='ReserveAppt']" 73 | ); 74 | 75 | const numAvailableAppts = await reserveButtons.length; 76 | 77 | if (numAvailableAppts) { 78 | hasAvailability = true; 79 | } else { 80 | console.log(`There could be appointments: ${xhrRes}`); 81 | const msg = `${site.name} - possible appointments on ${date}`; 82 | await sendSlackMsg("bot", msg); 83 | } 84 | 85 | if (!availability[date]) { 86 | availability[date] = { 87 | hasAvailability: true, 88 | numberAvailableAppointments: 0, 89 | }; 90 | availability[ 91 | date 92 | ].numberAvailableAppointments += numAvailableAppts; 93 | } 94 | } 95 | } 96 | 97 | page.close(); 98 | return { 99 | hasAvailability, 100 | availability, 101 | }; 102 | } 103 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/RevereCataldo/index.js: -------------------------------------------------------------------------------- 1 | // This clinic runs every Sunday, the URL to hit would need to be updated weekly 2 | const https = require("https"); 3 | const moment = require("moment"); 4 | 5 | async function GetAvailableAppointments() { 6 | console.log("Revere Cataldo starting"); 7 | const recurrentCataldoClinic = await ScrapeWebsiteData(); 8 | console.log("Revere Cataldo completed"); 9 | return { 10 | parentLocationName: "Revere Cataldo Clinic", 11 | timestamp: moment().format(), 12 | individualLocationData: [recurrentCataldoClinic], 13 | }; 14 | } 15 | 16 | async function ScrapeWebsiteData() { 17 | // This would need to be updated regularly, there are emails you can sign up 18 | // to receive from the city @Revere.org that would have the relevant url every week 19 | const url = 20 | "https://home.color.com/api/v1/vaccination_appointments/availability?calendar=7f6e2ddb-08b1-4471-8e15-8ef27897e1ed&collection_site=Revere&dob=1997-04-02"; 21 | const availabilityPromise = new Promise((resolve) => { 22 | https 23 | .get(url, (res) => { 24 | let body = ""; 25 | res.on("data", (chunk) => { 26 | body += chunk; 27 | }); 28 | res.on("end", () => { 29 | resolve(body); 30 | }); 31 | }) 32 | .on("error", (e) => { 33 | console.error( 34 | `Error making request for Revere Cataldo popup: + ${e}` 35 | ); 36 | }) 37 | .end(); 38 | }); 39 | 40 | const availabilityResponse = await availabilityPromise; 41 | const availability = JSON.parse(availabilityResponse).results; 42 | const results = { 43 | hasAvailability: false, 44 | availability: {}, 45 | name: "Rumney Marsh Academy - Cataldo Clinic", 46 | street: "140 American Legion Hwy", 47 | city: "Revere", 48 | zip: "02151", 49 | extraData: 50 | "Pfizer. Pre-registration tent will be on City Hall Lawn this week, Wed- Fri 11-3pm", 51 | // This might need to change weekly as well 52 | signUpLink: "https://www.cic-health.com/revere/rumneymarsh", 53 | }; 54 | // logic borrowed from the Color scraper 55 | availability.reduce((memo, currentValue) => { 56 | /* The availability returns an array of appointments like this: 57 | { 58 | "start": "2021-02-22T14:00:00+00:00", 59 | "end": "2021-02-22T14:04:00+00:00", 60 | "capacity": 1, 61 | "remaining_spaces": -1 62 | } 63 | */ 64 | let remainingSpaces = currentValue["remaining_spaces"]; 65 | if (remainingSpaces > 0) { 66 | const appointmentDateGMT = new Date(currentValue["start"]); 67 | let appointmentDateET = appointmentDateGMT.toLocaleString("en-US", { 68 | timeZone: "America/New_York", 69 | }); 70 | appointmentDateET = appointmentDateET.substring( 71 | 0, 72 | appointmentDateET.indexOf(",") 73 | ); 74 | let dateAvailability = memo["availability"][appointmentDateET]; 75 | if (!dateAvailability) { 76 | dateAvailability = { 77 | numberAvailableAppointments: 0, 78 | hasAvailability: false, 79 | }; 80 | memo["availability"][appointmentDateET] = dateAvailability; 81 | } 82 | dateAvailability["numberAvailableAppointments"] += remainingSpaces; 83 | dateAvailability["hasAvailability"] = true; 84 | memo["hasAvailability"] = true; 85 | } 86 | return memo; 87 | }, results); 88 | 89 | return results; 90 | } 91 | 92 | module.exports = GetAvailableAppointments; 93 | -------------------------------------------------------------------------------- /site-scrapers/MercyMedicalCenter/fakeMercyResponse.html: -------------------------------------------------------------------------------- 1 | ` 2 |
3 |
4 | 14 |
15 | 20 |
21 | 31 |
32 |
33 | 34 |
35 |
36 |

37 |
38 | Please select a date and time to book your initial vaccine 39 | appointment. 40 | These appointments are only for a first dose of a COVID 42 | vaccination series. 44 | Individuals should generally return to the site where they 45 | received their first dose to receive their second dose. 46 |
47 |

48 |
49 |
50 |
51 |
52 |

53 |
54 | Second doses will be booked automatically with our clinic based 55 | off the vaccine you receive during your first visit. You will be 56 | notified of your 2nd appointment date/time by the clinic staff 57 | when your first dose is administered. 58 |
59 |

60 |
61 |
62 | 63 |
64 |
65 | 71 |

72 |
73 | Friday March 12, 2021 74 | 81 |
82 |

83 |
84 |
85 | 86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 | 11:15 AM - 11:30 AM 94 |
95 |
96 |
97 |
98 | 107 |
108 |
109 |
110 |
111 |
112 |
113 | `; 114 |
115 | -------------------------------------------------------------------------------- /test/ZocDocTest.js: -------------------------------------------------------------------------------- 1 | const scraper = require("../no-browser-site-scrapers/ZocDoc/index"); 2 | const helper = require("../no-browser-site-scrapers/ZocDoc/zocdocBase"); 3 | const generator = require("../no-browser-site-scrapers/ZocDoc/zocdoc-config-generator"); 4 | const { expect } = require("chai"); 5 | const moment = require("moment"); 6 | const file = require("../lib/file"); 7 | const config = require("../no-browser-site-scrapers/ZocDoc/config"); 8 | 9 | const { someAvailability } = require("./ZocDoc/sample-availability"); 10 | 11 | const { locationData } = require("./ZocDoc/location-data"); 12 | 13 | /** 14 | * This test the config generator utility. Not a part of scraping but gathering 15 | * info from fetching provider data from ZocDoc. Automates site info creation. 16 | */ 17 | describe("ZocDoc Config generator test", function () { 18 | it("should provide id, name and location for each site", async function () { 19 | const providerDetailsJson = locationData; // await generator.fetchProviderDetails(); 20 | const providerDetails = generator.parseProviderDetails( 21 | providerDetailsJson 22 | ); 23 | 24 | // Object.values(providerDetails).forEach((provider) => 25 | // console.log(`${JSON.stringify(provider)}`) 26 | // ); 27 | 28 | expect(Object.entries(providerDetails).length).equals(7); 29 | }); 30 | }); 31 | 32 | describe("ZocDoc Provider availability test using scraper and canned data", function () { 33 | const testFetchService = { 34 | async fetchAvailability() { 35 | return someAvailability; 36 | }, 37 | }; 38 | const beforeTime = moment(); 39 | 40 | it("should provide availability for each site, and the results objects structure should conform", async function () { 41 | const results = await scraper(false, testFetchService); 42 | 43 | const expected = [ 44 | true, 45 | false, 46 | false, 47 | false, 48 | false, 49 | false, 50 | false, 51 | false, 52 | ]; 53 | const hasAvailability = Object.values( 54 | results.individualLocationData 55 | ).map((result) => result.hasAvailability); 56 | const afterTime = moment(); 57 | 58 | expect(hasAvailability).is.deep.equal(expected); 59 | 60 | /* 61 | Structure conformance expectations: 62 | 63 | - All the timestamps are expected to be between before 64 | and after when the scraper was executed 65 | - Each site's results object must have a property named "hasAvailability" 66 | */ 67 | results.individualLocationData.forEach((result) => { 68 | expect(moment(result.timestamp).isBetween(beforeTime, afterTime)); 69 | expect(result.hasAvailability).is.not.undefined; 70 | }); 71 | 72 | // console.log(`${JSON.stringify(results)}`); 73 | 74 | if (process.env.DEVELOPMENT) { 75 | file.write( 76 | `${process.cwd()}/out_no_browser.json`, 77 | `${JSON.stringify(results, null, " ")}` 78 | ); 79 | } 80 | }); 81 | }); 82 | 83 | describe("ZocDoc Testing zocdocBase with live data", function () { 84 | it("should provide availability for each site listed in config.js", async function () { 85 | const providerAvailabilityJson = await helper.fetchAvailability(); 86 | 87 | const providerAvailability = helper.parseAvailability( 88 | providerAvailabilityJson 89 | ); 90 | 91 | // Object.entries(providerAvailability).forEach((provider) => 92 | // console.log(`${JSON.stringify(provider)}`) 93 | // ); 94 | 95 | expect(Object.keys(providerAvailability).length).equals( 96 | Object.keys(config.sites).length 97 | ); 98 | }); 99 | }); 100 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/Curative/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const https = require("https"); 3 | const moment = require("moment"); 4 | 5 | module.exports = async function GetAvailableAppointments() { 6 | console.log(`${site.name} starting.`); 7 | const locationIDs = site.locations.map((loc) => loc.id); 8 | 9 | const rawData = {}; 10 | for (const id of locationIDs) { 11 | const p = new Promise((resolve, reject) => { 12 | let response = ""; 13 | https.get(site.website + id, (res) => { 14 | let body = ""; 15 | res.on("data", (chunk) => { 16 | body += chunk; 17 | }); 18 | res.on("end", () => { 19 | try { 20 | response = JSON.parse(body); 21 | resolve(response); 22 | } catch (e) { 23 | console.error("JSON.parse failed: " + e); 24 | reject(e); 25 | } 26 | }); 27 | }); 28 | }); 29 | rawData[id] = await p; 30 | } 31 | 32 | const individualLocationData = site.locations.map((loc) => { 33 | const data = rawData[loc.id]; 34 | const results = { 35 | name: data.name, 36 | street: ( 37 | data.street_address_1 + 38 | " " + 39 | data.street_address_2 40 | ).trim(), 41 | city: data.city, 42 | zip: data.postal_code, 43 | signUpLink: site.linkWebsite + loc.id, 44 | massVax: !!loc.massVax, // This is a MassVax site that only allows preregistration 45 | hasAvailability: false, 46 | availability: {}, //date (MM/DD/YYYY) => hasAvailability, numberAvailableAppointments 47 | }; 48 | // If this is a massVax site that is invite-only, then we don't 49 | // need availability data. 50 | if (!results.massVax) { 51 | data.appointment_windows.forEach((appointment) => { 52 | const dateRegexp = /(?[0-9]{4})-(?[0-9]{2})-(?[0-9]{2})/; 53 | const { year, month, day } = appointment.start_time.match( 54 | dateRegexp 55 | ).groups; 56 | const date = `${month}/${day}/${year}`; 57 | let newNumberAvailable = 58 | appointment.status !== "Disabled" 59 | ? appointment.slots_available 60 | : 0; 61 | 62 | if (newNumberAvailable) { 63 | results.hasAvailability = true; 64 | } 65 | 66 | if (results.availability[date]) { 67 | newNumberAvailable += 68 | results.availability[date].numberAvailableAppointments; 69 | } 70 | 71 | // Only add date keys if there are appointments 72 | if (newNumberAvailable) { 73 | results.availability[date] = { 74 | numberAvailableAppointments: newNumberAvailable, 75 | hasAvailability: true, 76 | }; 77 | } 78 | }); 79 | 80 | if ( 81 | data.hasOwnProperty("visible_in_search") && 82 | !data.visible_in_search 83 | ) { 84 | // Commented out the following line because there is currently a 85 | // waiting room for people to join for an event that starts at 8:30am. 86 | // We should revisit this later after the appointments are gone. 87 | //results.hasAvailability = false; 88 | } 89 | } 90 | return results; 91 | }); 92 | console.log(`${site.name} done.`); 93 | return { 94 | parentLocationName: "Curative", 95 | timestamp: moment().format(), 96 | individualLocationData, 97 | }; 98 | }; 99 | -------------------------------------------------------------------------------- /test/Color.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const { formatResponse } = require("./../no-browser-site-scrapers/Color"); 3 | const chai = require("chai"); 4 | const expect = chai.expect; 5 | 6 | describe("Color Transformations", () => { 7 | it("should return no availabilities when there is -1", () => { 8 | // mock out the availability request 9 | const response = `{"results": [{ 10 | "start": "2021-02-22T14:00:00+00:00", 11 | "end": "2021-02-22T14:04:00+00:00", 12 | "capacity": 1, 13 | "remaining_spaces":-1 14 | }]}`; 15 | expect(formatResponse("TestSiteName", response, false)) 16 | .to.deep.include({ 17 | hasAvailability: false, 18 | }) 19 | .and.nested.property("availability") 20 | .deep.equal({}); 21 | }); 22 | // Skipping this test as we're hard-coding no availability due to mandatory pre-registration 23 | it("should return one availability when there is one", () => { 24 | // mock out the http request that returns the token 25 | const response = `{"results":[{ 26 | "start": "2021-02-22T14:00:00+00:00", 27 | "end": "2021-02-22T14:04:00+00:00", 28 | "capacity": 1, 29 | "remaining_spaces":1 30 | }]}`; 31 | // run the test 32 | /** 33 | * Expect the result to look like: 34 | * { 35 | * "hasAvailability": true, 36 | * "availability": { 37 | * "2/22/2021": { 38 | * hasAvailability: true, 39 | * numberAvailableAppointments: 1 40 | * ] 41 | * } 42 | * } 43 | */ 44 | expect(formatResponse("TestSiteName", response, false)).to.deep.include( 45 | { 46 | hasAvailability: true, 47 | availability: { 48 | "2/22/2021": { 49 | hasAvailability: true, 50 | numberAvailableAppointments: 1, 51 | }, 52 | }, 53 | } 54 | ); 55 | }); 56 | // Skipping this test as we're hard-coding no availability due to mandatory pre-registration 57 | it("should return multiple date availabilities", () => { 58 | // mock out the http request that returns the token 59 | const response = `{"results":[{ 60 | "start": "2021-02-22T14:00:00+00:00", 61 | "end": "2021-02-22T14:04:00+00:00", 62 | "capacity": 1, 63 | "remaining_spaces":1 64 | }, 65 | { 66 | "start": "2021-02-22T14:04:00+00:00", 67 | "end": "2021-02-22T14:08:00+00:00", 68 | "capacity": 1, 69 | "remaining_spaces":1 70 | }, 71 | { 72 | "start": "2021-02-23T14:04:00+00:00", 73 | "end": "2021-02-23T14:08:00+00:00", 74 | "capacity": 1, 75 | "remaining_spaces":1 76 | }]}`; 77 | // run the test 78 | /** 79 | * Expect the result to look like: 80 | * { 81 | * "hasAvailability": true, 82 | * "availability": { 83 | * "2/22/2021": { 84 | * hasAvailability: true, 85 | * numberAvailableAppointments: 2 86 | * }, 87 | * "2/23/2021": { 88 | * hasAvailability: true, 89 | * numberAvailableAppointments: 1 90 | * }, 91 | * } 92 | */ 93 | expect(formatResponse("TestSiteName", response, false)).to.deep.include( 94 | { 95 | hasAvailability: true, 96 | availability: { 97 | "2/22/2021": { 98 | hasAvailability: true, 99 | numberAvailableAppointments: 2, 100 | }, 101 | "2/23/2021": { 102 | hasAvailability: true, 103 | numberAvailableAppointments: 1, 104 | }, 105 | }, 106 | } 107 | ); 108 | }); 109 | }); 110 | -------------------------------------------------------------------------------- /site-scrapers/SouthcoastHealth/index.js: -------------------------------------------------------------------------------- 1 | const { siteUrl, sites, entityName } = require("./config"); 2 | const { parseJson } = require("./responseParser.js"); 3 | const moment = require("moment"); 4 | 5 | module.exports = async function GetAvailableAppointments(browser) { 6 | console.log(`${entityName} starting.`); 7 | const siteData = await ScrapeWebsiteData(browser, sites); 8 | console.log(`${entityName} done.`); 9 | return { 10 | parentLocationName: entityName, 11 | timestamp: moment().format(), 12 | individualLocationData: siteData, 13 | }; 14 | }; 15 | 16 | /** 17 | * 18 | * @param {*} browser 19 | * @param {*} pageService 20 | * @param {*} sites 21 | * @returns array of site details and availability 22 | */ 23 | async function ScrapeWebsiteData(browser, sites) { 24 | const page = await browser.newPage(); 25 | 26 | const submitButtonSelector = "button.btn.btn-default.btn-lg.center-block"; 27 | await Promise.all([ 28 | page.goto(siteUrl), 29 | await page.waitForSelector(submitButtonSelector, { 30 | visible: true, 31 | timeout: 15000, 32 | }), 33 | ]); 34 | 35 | await page.evaluate((submitButtonSelector) => { 36 | document.querySelector(submitButtonSelector).click(); 37 | }, submitButtonSelector); 38 | 39 | await page.waitForSelector("#oas-scheduler"); 40 | 41 | const siteData = []; 42 | 43 | for (const site of sites) { 44 | const availabilityContainer = await getAvailabilityForSite(page, site); 45 | 46 | siteData.push({ 47 | ...site, 48 | availability: availabilityContainer, 49 | hasAvailability: Object.keys(availabilityContainer).length > 0, 50 | }); 51 | } 52 | 53 | page.close(); 54 | return siteData; 55 | } 56 | 57 | async function getAvailabilityForSite(page, site) { 58 | const responseJson = await fetchDataForSite(page, site); 59 | const availabilityContainer = parseJson(responseJson); 60 | return availabilityContainer; 61 | } 62 | 63 | async function fetchDataForSite(page, site) { 64 | const location = site.city; 65 | 66 | // 2021-05-08T02:34:12.058Z 67 | const startDateStr = moment().toISOString(); 68 | // 2021-05-21T02:34:12.058Z 13 days 69 | const endDateStr = moment().add(13, "days").toISOString(); 70 | 71 | const response = await page.evaluate( 72 | async (location, startDateStr, endDateStr) => { 73 | const json = await fetch( 74 | "https://southcoastapps.southcoast.org/OnlineAppointmentSchedulingApi/api/resourceTypes/slots/search", 75 | { 76 | headers: { 77 | accept: "application/json, text/plain, */*", 78 | "accept-language": "en-US,en;q=0.9", 79 | "content-type": "application/json", 80 | "sec-ch-ua": 81 | '" Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"', 82 | "sec-ch-ua-mobile": "?0", 83 | "sec-fetch-dest": "empty", 84 | "sec-fetch-mode": "cors", 85 | "sec-fetch-site": "same-site", 86 | sessiontoken: "5d886340-71e9-4bd2-9449-80d5eb3263e9", 87 | }, 88 | referrer: "https://www.southcoast.org/", 89 | referrerPolicy: "strict-origin-when-cross-origin", 90 | body: `{"ProviderCriteria":{"SpecialtyID":null,"ConcentrationID":null},"ResourceTypeId":"98C5A8BE-25D1-4125-9AD5-1EE64AD164D2","StartDate":"${startDateStr}","EndDate":"${endDateStr}","Location":"${location}"}`, 91 | method: "POST", 92 | mode: "cors", 93 | credentials: "omit", 94 | } 95 | ) 96 | .then((res) => res.json()) 97 | .then((json) => { 98 | return json; 99 | }) 100 | .catch((error) => 101 | console.log(`error fetching site data: ${error}`) 102 | ); 103 | return json; 104 | }, 105 | location, 106 | startDateStr, 107 | endDateStr 108 | ); 109 | 110 | return response; 111 | } 112 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/CVSPharmacy/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const { toTitleCase } = require("../../lib/stringUtil"); 3 | const https = require("https"); 4 | const moment = require("moment"); 5 | 6 | module.exports = async function GetAvailableAppointments() { 7 | console.log(`${site.name} starting.`); 8 | const webData = await ScrapeWebsiteData(); 9 | // Javascript is not good at timezones. CVS's timestamp arrives in 10 | // Mountain time ("America/Denver"), and we need to convert it to 11 | // UTC. Since the offset changes twice a year, we need to 12 | // calculate the current offset, and then parse CVS's currentTime 13 | // using it. 14 | // The best we can do is render the time in two timezones, parse those as dates 15 | // and subtract them, and convert them from milliseconds to hours. 16 | const now = new Date(); // 2021-02-20T04:52:15.444Z 17 | const offsetMountain = 18 | ["America/Denver", "UTC"] 19 | .map((z) => 20 | Date.parse(now.toLocaleString("en-US", { timeZone: z })) 21 | ) 22 | .reduce((a, b) => b - a) / 23 | (3600 * 1000); 24 | // This would fail if offsetMountain were 2 digits, but it will only ever be 6 or 7. 25 | const timestamp = moment( 26 | `${webData.responsePayloadData.currentTime}-0${offsetMountain}:00` 27 | ).format(); 28 | const individualLocationData = webData.responsePayloadData.data.MA.map( 29 | (responseLocation) => { 30 | // Prior to Feb 22 or so, CVS's JSON returned: 31 | // { 32 | // "totalAvailable": "0", 33 | // "city": "CAMBRIDGE", 34 | // "state": "MA", 35 | // "pctAvailable": "0.00%", 36 | // "status": "Fully Booked" 37 | // }, 38 | // 39 | // But as of Feb. 25 it returns: 40 | // { 41 | // "city": "CAMBRIDGE", 42 | // "state": "MA", 43 | // "status": "Fully Booked" 44 | // }, 45 | // It's not clear whether we can expect the other fields to return when there is 46 | // availability. 47 | // Also, we had previously seen cases where numeric availability was "0" but status 48 | // was "Available" and it's not apparent what that really meant (skew? bugs?). 49 | const totalAvailability = 50 | responseLocation.totalAvailable && 51 | parseInt(responseLocation.totalAvailable); 52 | const city = toTitleCase(responseLocation.city); 53 | const retval = { 54 | city: city, 55 | name: `${site.name} (${city})`, 56 | hasAvailability: responseLocation.status !== "Fully Booked", 57 | availability: {}, 58 | siteTimestamp: timestamp, 59 | signUpLink: site.website, 60 | }; 61 | if (totalAvailability) { 62 | retval.totalAvailability = totalAvailability; 63 | } 64 | return retval; 65 | } 66 | ); 67 | console.log(`${site.name} done.`); 68 | return { 69 | parentLocationName: "CVS Pharmacy", 70 | isChain: true, 71 | individualLocationData, 72 | timestamp: moment().format(), 73 | }; 74 | }; 75 | 76 | function urlContent(url, options) { 77 | return new Promise((resolve) => { 78 | let response = ""; 79 | https.get(url, options, (res) => { 80 | let body = ""; 81 | res.on("data", (chunk) => (body += chunk)); 82 | res.on("end", () => { 83 | response = body; 84 | resolve(response); 85 | }); 86 | }); 87 | }); 88 | } 89 | 90 | async function ScrapeWebsiteData() { 91 | // Simply retrieving 92 | // https://www.cvs.com/immunizations/covid-19-vaccine.vaccine-status.ma.json?vaccineinfo 93 | // returns potentially stale data that varies based on the Akamai Edgekey server that you access. 94 | // Append a cachebusting 95 | // &nonce=&nonce=1613934207668 96 | // to bypass Akamai caching. 97 | const url = `${site.massJson}&nonce=${new Date().valueOf()}`; 98 | const options = { headers: { Referer: site.website } }; 99 | return JSON.parse(await urlContent(url, options)); 100 | } 101 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/SouthLawrence/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const moment = require("moment"); 3 | const fetch = require("node-fetch"); 4 | const { sendSlackMsg } = require("../../lib/slack"); 5 | const s3 = require("../../lib/s3"); 6 | 7 | // Set to true for verbose debug messages. 8 | const DEBUG = false; 9 | 10 | function Debug(...args) { 11 | if (DEBUG) { 12 | console.log(...args); 13 | } 14 | } 15 | 16 | module.exports = async function GetAvailableAppointments() { 17 | console.log("SouthLawrence starting."); 18 | 19 | const { website, ...restLawrence } = site; 20 | 21 | const websiteData = await ScrapeWebsiteData(website); 22 | Debug("websiteData", websiteData); 23 | 24 | console.log("SouthLawrence done."); 25 | return { 26 | parentLocationName: "South Lawrence", 27 | timestamp: moment().format(), 28 | individualLocationData: [ 29 | { 30 | ...restLawrence, 31 | ...websiteData, 32 | }, 33 | ], 34 | }; 35 | }; 36 | 37 | async function ScrapeWebsiteData(website) { 38 | Debug("website", website); 39 | // GET the website to scrape the calendar and type codes. 40 | const calendarHtml = await fetch(website) 41 | .then((res) => res.text()) 42 | .then((body) => { 43 | Debug(body); 44 | const { type, calendar } = ParseCovidVaccineCalendar(body); 45 | // POST to the ShowCalendar action (at the same URL) to 46 | // retrieve the calendar HTML 47 | return type && calendar 48 | ? ShowCalendar(website, type, calendar) 49 | : ""; 50 | }); 51 | 52 | // Look for a certain element ID to determine if no times are available. 53 | Debug("calendarHtml", calendarHtml); 54 | let hasAvailability = null; 55 | if (calendarHtml.includes("no-times-available-message")) { 56 | hasAvailability = false; 57 | } else { 58 | if (process.env.NODE_ENV === "production") { 59 | await s3.saveHTMLContent(site.name, calendarHtml); 60 | await sendSlackMsg( 61 | "bot", 62 | "Appointments maybe found at South Lawrence" 63 | ); 64 | } 65 | } 66 | // TODO: there are no examples of available slots at this time, so 67 | // we can only confirm there is no availability 68 | return { 69 | hasAvailability: hasAvailability, 70 | availability: {}, 71 | }; 72 | } 73 | 74 | function ParseCovidVaccineCalendar(body) { 75 | // Scrape the type and calendar query parameters from the javascript array, 76 | // looking for the type and calendar of the COVID vaccine, first dose. 77 | // javascript looks like: 78 | // typeToCalendars[19267408] = [[4896962, '1st Dose - COVID Vaccine', '', '', ''] ]; 79 | // The calendar name displayed on the website changes over time. 80 | // Pick the best one. 81 | const regexes = [ 82 | /typeToCalendars\[(?\d+)\] = \[\[(?\d+), '1st Dose - COVID Vaccine/, 83 | /typeToCalendars\[(?\d+)\] = \[\[(?\d+), 'STAND-BY LIST\s+-\sS[.] Lawrence East School/, 84 | ]; 85 | for (let i = 0; i < regexes.length; i++) { 86 | const match = body.match(regexes[i]); 87 | if (match) { 88 | Debug("match", match && match.groups); 89 | return match.groups; 90 | } 91 | } 92 | Debug("no match"); 93 | return {}; 94 | } 95 | 96 | function ShowCalendar(website, type, calendar) { 97 | Debug("website", website, "type", type, "calendar", calendar); 98 | return new Promise((resolve) => { 99 | fetch( 100 | website + 101 | "?action=showCalendar&fulldate=1&owner=21579739&template=weekly", 102 | { 103 | headers: { 104 | "content-type": 105 | "application/x-www-form-urlencoded; charset=UTF-8", 106 | }, 107 | body: `type={type}&calendar={calendar}&skip=true&options%5Bqty%5D=1&options%5BnumDays%5D=5&ignoreAppointment=&appointmentType=&calendarID=`, 108 | method: "POST", 109 | } 110 | ) 111 | .then((res) => res.text()) 112 | .then((body) => { 113 | resolve(body); 114 | }); 115 | }); 116 | } 117 | -------------------------------------------------------------------------------- /alerts/determine_recipients.js: -------------------------------------------------------------------------------- 1 | const { zip } = require("lodash"); 2 | const dbUtils = require("../lib/db/utils"); 3 | const faunadb = require("faunadb"), 4 | fq = faunadb.query; 5 | const zipRadiusFinder = require("../lib/zipRadiusFinder"); 6 | 7 | module.exports = { determineRecipients, findSubscribersWithZips }; 8 | 9 | async function determineRecipients({ locations, numberAvailable }) { 10 | console.log( 11 | `finding recipients for ${JSON.stringify( 12 | locations, 13 | null, 14 | 2 15 | )} with ${numberAvailable} slots.` 16 | ); 17 | const radiusIncrements = [3, 5, 10, 15, 20, 30, 40, 50, 100, null]; 18 | let numberRecipients = 0; 19 | const textRecipients = []; 20 | const emailRecipients = []; 21 | for (const radiusIndex in radiusIncrements) { 22 | for (const location of locations) { 23 | const zips = zipRadiusFinder({ 24 | lessThan: radiusIncrements[radiusIndex], 25 | greaterThan: radiusIndex 26 | ? radiusIncrements[radiusIndex - 1] 27 | : null, 28 | originLoc: location, 29 | }); 30 | let subscribers = await findSubscribersWithZips(zips); 31 | // First, filter out anybody with a radius tighter than the distance 32 | // to the site. NOTE that for this simplification to work, we must 33 | // ensure that our radiusIncrements be a superset of the radii we allow 34 | // users to enter when they subscribe. 35 | // Also filter out people that are already in our subscription list. 36 | console.log(`found ${subscribers.length} subscribers`); 37 | subscribers = subscribers.filter( 38 | (subscriber) => 39 | !parseInt(radiusIndex) || 40 | subscriber.radius > radiusIncrements[radiusIndex - 1] || 41 | (subscriber.phoneNumber && 42 | textRecipients.indexOf( 43 | (rec) => rec.phoneNumber === subscriber.phoneNumber 44 | ) > -1) || 45 | (subscriber.email && 46 | emailRecipients.indexOf( 47 | (rec) => rec.email === subscriber.email 48 | ) > -1) 49 | ); 50 | console.log(`filtered down to ${subscribers.length} subscribers`); 51 | // Then, sort them out into text or email. 52 | subscribers.forEach((subscriber) => { 53 | if (subscriber.phoneNumber) { 54 | textRecipients.push(subscriber); 55 | } else if (subscriber.email) { 56 | emailRecipients.push(subscriber); 57 | } else { 58 | console.error( 59 | `found subscriber ${JSON.stringify( 60 | subscriber 61 | )} without phone # or email!` 62 | ); 63 | } 64 | }); 65 | numberRecipients += subscribers.length; 66 | } 67 | if (numberRecipients >= 4 * numberAvailable) { 68 | console.log(`notifying ${numberRecipients} recipients.`); 69 | return { 70 | emailRecipients, 71 | textRecipients, 72 | }; 73 | } 74 | } 75 | // if we go through all radii and still have found fewer subscribers than 76 | // twice the number of appointments, return everybody we found. 77 | console.log(`notifying ${numberRecipients} recipients.`); 78 | return { emailRecipients, textRecipients }; 79 | } 80 | 81 | async function findSubscribersWithZips(zips) { 82 | console.log(`finding subscribers with the following ZIP codes: ${zips}`); 83 | if (!zips || !zips.length) { 84 | return []; 85 | } 86 | return dbUtils.faunaQuery( 87 | fq.Select( 88 | "data", 89 | fq.Map( 90 | fq.Paginate( 91 | fq.Union( 92 | ...zips.map((zip) => 93 | fq.Match( 94 | fq.Index("subscriptionsByZipWithStatuses"), 95 | [zip, true, false] // active subscriptions with a zip 96 | ) 97 | ) 98 | ), 99 | { size: 5000 } 100 | ), 101 | fq.Lambda((x) => fq.Select("data", fq.Get(x))) 102 | ) 103 | ) 104 | ); 105 | } 106 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/Color/index.js: -------------------------------------------------------------------------------- 1 | const https = require("https"); 2 | const sites = require("./config"); 3 | const moment = require("moment"); 4 | 5 | const token = "bcd282a6fe22e6fc47e14be11a35b33fe1bc"; 6 | async function GetAvailableAppointments() { 7 | console.log("Color locations starting"); 8 | const originalSites = sites; 9 | const finalSites = []; 10 | for (const site of originalSites) { 11 | const { siteUrl, calendarUrl, ...rest } = site; 12 | const webData = await ScrapeWebsiteData( 13 | rest.name, 14 | siteUrl, 15 | calendarUrl, 16 | rest.massVax 17 | ); 18 | finalSites.push({ 19 | ...rest, 20 | ...webData, 21 | }); 22 | } 23 | console.log("Color locations complete"); 24 | return { 25 | parentLocationName: "Color", 26 | timestamp: moment().format(), 27 | individualLocationData: finalSites, 28 | }; 29 | } 30 | 31 | async function ScrapeWebsiteData(siteName, siteUrl, calendarUrl, massVax) { 32 | const availabilityUrl = 33 | `https://home.color.com/api/v1/vaccination_appointments/availability?claim_token=${token}&collection_site=${siteUrl}` + 34 | (calendarUrl ? `&calendar=${calendarUrl}` : ""); 35 | const availabilityPromise = new Promise((resolve) => { 36 | https 37 | .get(availabilityUrl, (res) => { 38 | let body = ""; 39 | res.on("data", (chunk) => { 40 | body += chunk; 41 | }); 42 | res.on("end", () => { 43 | resolve(body); 44 | }); 45 | }) 46 | .on("error", (e) => { 47 | console.error( 48 | `Error making token request for ${siteName}: + ${e}` 49 | ); 50 | }) 51 | .end(); 52 | }); 53 | 54 | const availabilityResponse = await availabilityPromise; 55 | return formatResponse(siteName, availabilityResponse, massVax); 56 | } 57 | 58 | function formatResponse(siteName, availabilityResponse, massVax) { 59 | const availability = JSON.parse(availabilityResponse).results; 60 | const results = { 61 | hasAvailability: false, 62 | availability: {}, 63 | }; 64 | 65 | if (!availability) { 66 | console.log("COLOR invalid availability for " + siteName + ". "); 67 | console.log(availabilityResponse); 68 | } 69 | 70 | // If this is a massVax site that is invite-only, then we don't 71 | // need availability data. 72 | if (!massVax && !!availability) { 73 | // Collect availability count by date 74 | availability.reduce((memo, currentValue) => { 75 | /* The availability returns an array of appointments like this: 76 | { 77 | "start": "2021-02-22T14:00:00+00:00", 78 | "end": "2021-02-22T14:04:00+00:00", 79 | "capacity": 1, 80 | "remaining_spaces": -1 81 | 82 | } 83 | */ 84 | let remainingSpaces = currentValue["remaining_spaces"]; 85 | if (remainingSpaces > 0) { 86 | const appointmentDateGMT = new Date(currentValue["start"]); 87 | let appointmentDateET = appointmentDateGMT.toLocaleString( 88 | "en-US", 89 | { 90 | timeZone: "America/New_York", 91 | } 92 | ); 93 | appointmentDateET = appointmentDateET.substring( 94 | 0, 95 | appointmentDateET.indexOf(",") 96 | ); 97 | let dateAvailability = memo["availability"][appointmentDateET]; 98 | if (!dateAvailability) { 99 | dateAvailability = { 100 | numberAvailableAppointments: 0, 101 | hasAvailability: false, 102 | }; 103 | memo["availability"][appointmentDateET] = dateAvailability; 104 | } 105 | dateAvailability[ 106 | "numberAvailableAppointments" 107 | ] += remainingSpaces; 108 | dateAvailability["hasAvailability"] = true; 109 | memo["hasAvailability"] = true; 110 | } 111 | return memo; 112 | }, results); 113 | } 114 | return results; 115 | } 116 | 117 | //ES5 way of doing named & default exports 118 | const Color = (module.exports = GetAvailableAppointments); 119 | Color.formatResponse = formatResponse; 120 | -------------------------------------------------------------------------------- /lib/s3.js: -------------------------------------------------------------------------------- 1 | const file = require("./file"); 2 | const AWS = require("aws-sdk"); 3 | const moment = require("moment"); 4 | 5 | /** 6 | * 7 | * @returns {AWS.s3} 8 | */ 9 | 10 | function init() { 11 | return new AWS.S3({ 12 | accessKeyId: process.env.AWSACCESSKEYID, 13 | secretAccessKey: process.env.AWSSECRETACCESSKEY, 14 | }); 15 | } 16 | 17 | /** 18 | * 19 | * @param {string} bucketPath - absolute path without leading or trailing slash ex. debug 20 | * @param {string} fileName - required 21 | * @param {string} fileContents - optional 22 | * @returns {object} s3.upload response data 23 | */ 24 | 25 | async function uploadFile(bucketPath, fileName, fileContents) { 26 | if (!bucketPath) { 27 | console.log("Missing bucketPath"); 28 | return; 29 | } 30 | if (!fileName) { 31 | console.log("Missing fileName"); 32 | return; 33 | } 34 | 35 | const s3 = init(); 36 | let s3Key; 37 | 38 | if (bucketPath === "/") { 39 | s3Key = `${fileName}`; 40 | } else { 41 | s3Key = `${bucketPath}/${fileName}`; 42 | } 43 | // s3.upload can read in the file but it might be useful at some point to have a global (file.read) 44 | const finalFileContents = fileContents || file.read(fileName); 45 | const params = { 46 | Bucket: process.env.AWSS3BUCKETNAME, 47 | Key: s3Key, 48 | Body: finalFileContents, 49 | }; 50 | 51 | if (!process.env.DEVELOPMENT) { 52 | const uploadResponse = await new Promise((resolve, reject) => { 53 | s3.upload(params, function (err, data) { 54 | if (err) { 55 | reject(err); 56 | } 57 | console.log(`File uploaded successfully. ${data.Location}`); 58 | resolve(data); 59 | }); 60 | }); 61 | 62 | return uploadResponse; 63 | } 64 | return fileName; 65 | } 66 | 67 | /** 68 | * 69 | * We remove the colons for filesystem integrity. 70 | * @returns {string} 71 | */ 72 | 73 | function getTimestampForFile() { 74 | return moment().utc().format("YYYY-MM-DDTHHmmss[Z]"); 75 | } 76 | 77 | /** 78 | * @param {string} siteName eg. Lowell General 79 | * @param {string} html - content from the page 80 | * @returns {object} with htmlFileName 81 | */ 82 | 83 | async function saveHTMLContent(siteName, html) { 84 | const timestamp = getTimestampForFile(); 85 | const htmlFileName = `${siteName}-${timestamp}.html`; 86 | const s3Dir = "debug"; 87 | if (process.env.DEVELOPMENT) { 88 | file.write(htmlFileName, html); 89 | } else { 90 | await uploadFile(s3Dir, htmlFileName, html); 91 | } 92 | return { htmlFileName }; 93 | } 94 | 95 | /** 96 | * @param {string} siteName eg. Lowell General 97 | * @param {object} page - puppeteer browser page 98 | * @returns {object} with htmlFileName , screenshotFileName 99 | */ 100 | 101 | async function savePageContent(siteName, page) { 102 | // It appears puppeteer does not have a generic page.waitForNetworkIdle 103 | // so for now we just assume all XHR requests will complete within 104 | // a reasonable timeframe, for now 10s. 105 | await page.waitForTimeout(10000); 106 | const html = await page.content(); 107 | const timestamp = getTimestampForFile(); 108 | const htmlFileName = `${siteName}-${timestamp}.html`; 109 | const screenshotFileName = `${siteName}-${timestamp}.png`; 110 | const s3Dir = "debug"; 111 | if (process.env.DEVELOPMENT) { 112 | file.write(htmlFileName, html); 113 | await page.screenshot({ path: screenshotFileName, fullPage: true }); 114 | } else { 115 | const pngDataBuffer = await page.screenshot(); 116 | await uploadFile(s3Dir, screenshotFileName, pngDataBuffer); 117 | await uploadFile(s3Dir, htmlFileName, html); 118 | } 119 | return { htmlFileName, screenshotFileName }; 120 | } 121 | 122 | /** 123 | * 124 | * @param {string} webData 125 | * @param {string} timestamp to use for the cache file name 126 | * @returns {object} s3.upload response data 127 | */ 128 | 129 | async function saveWebData(webData, timestamp) { 130 | const webDataFileName = `data.json`; 131 | const webCacheFileName = `data-${timestamp}.json`; 132 | if (process.env.DEVELOPMENT) { 133 | file.write(webDataFileName, webData); 134 | file.write(webCacheFileName, webData); 135 | } 136 | const s3Dir = "/"; 137 | const upload = await uploadFile(s3Dir, webDataFileName, webData); 138 | const upload2 = await uploadFile(s3Dir, webCacheFileName, webData); 139 | return { upload, upload2 }; 140 | } 141 | 142 | module.exports = { 143 | init, 144 | uploadFile, 145 | getTimestampForFile, 146 | savePageContent, 147 | saveWebData, 148 | }; 149 | -------------------------------------------------------------------------------- /alerts/pinpoint/new_subscriber.js: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk"); 2 | 3 | if (process.env.NODE_ENV !== "production") { 4 | AWS.config.credentials = new AWS.SharedIniFileCredentials({ 5 | profile: "macovidvaccines", 6 | }); 7 | AWS.config.update({ region: "us-east-1" }); 8 | } 9 | 10 | const pinpoint = new AWS.Pinpoint(); 11 | 12 | module.exports = { 13 | validateNumberAndAddSubscriber, 14 | }; 15 | 16 | async function validateNumberAndAddSubscriber(phoneNumber) { 17 | let destinationNumber = phoneNumber; 18 | if (destinationNumber.length === 10) { 19 | destinationNumber = "+1" + destinationNumber; 20 | } 21 | var params = { 22 | NumberValidateRequest: { 23 | IsoCountryCode: "US", 24 | PhoneNumber: destinationNumber, 25 | }, 26 | }; 27 | return new Promise((resolve, reject) => 28 | pinpoint.phoneNumberValidate(params, async function (err, data) { 29 | if (err) { 30 | console.error( 31 | `phone number validation failed with error: ${err}` 32 | ); 33 | reject(err); 34 | } else { 35 | if (data["NumberValidateResponse"]["PhoneTypeCode"] === 0) { 36 | await createEndpoint(data); 37 | } else { 38 | console.error( 39 | `${destinationNumber} cannot receive text messages.` 40 | ); 41 | } 42 | resolve(data); 43 | } 44 | }) 45 | ); 46 | } 47 | 48 | async function createEndpoint(data) { 49 | var destinationNumber = 50 | data["NumberValidateResponse"]["CleansedPhoneNumberE164"]; 51 | var endpointId = data["NumberValidateResponse"][ 52 | "CleansedPhoneNumberE164" 53 | ].substring(1); 54 | 55 | var params = { 56 | ApplicationId: process.env.PINPOINT_APPLICATION_ID, 57 | // The Endpoint ID is equal to the cleansed phone number minus the leading 58 | // plus sign. This makes it easier to easily update the endpoint later. 59 | EndpointId: endpointId, 60 | EndpointRequest: { 61 | ChannelType: "SMS", 62 | Address: destinationNumber, 63 | OptOut: "NONE", 64 | Location: { 65 | PostalCode: data["NumberValidateResponse"]["ZipCode"], 66 | City: data["NumberValidateResponse"]["City"], 67 | Country: data["NumberValidateResponse"]["CountryCodeIso2"], 68 | }, 69 | Demographic: { 70 | Timezone: data["NumberValidateResponse"]["Timezone"], 71 | }, 72 | }, 73 | }; 74 | return new Promise((resolve, reject) => 75 | pinpoint.updateEndpoint(params, async function (err, data) { 76 | if (err) { 77 | console.error(`endpoint could not be added: ${err}`); 78 | reject(err); 79 | } else { 80 | console.log("updateEndpoint res"); 81 | console.dir(data, { depth: null }); 82 | await sendConfirmation(endpointId); 83 | resolve(data); 84 | } 85 | }) 86 | ); 87 | } 88 | 89 | async function sendConfirmation(destinationNumber) { 90 | var params = { 91 | ApplicationId: process.env.PINPOINT_APPLICATION_ID, 92 | MessageRequest: { 93 | Addresses: { 94 | [destinationNumber]: { 95 | ChannelType: "SMS", 96 | }, 97 | }, 98 | MessageConfiguration: { 99 | SMSMessage: { 100 | Body: 101 | "Reply YES to confirm enrollment in COVID vaccine availability updates from macovidvaccines.com. Standard messaging & data rates may apply.", 102 | MessageType: "TRANSACTIONAL", 103 | OriginationNumber: process.env.PINPOINT_ORIGINATION_NUMBER, 104 | }, 105 | }, 106 | }, 107 | }; 108 | 109 | return new Promise((resolve, reject) => 110 | pinpoint.sendMessages(params, function (err, data) { 111 | // If something goes wrong, print an error message. 112 | if (err) { 113 | console.error(`could not send confirmation message: ${err}`); 114 | reject(err); 115 | // Otherwise, show the unique ID for the message. 116 | } else { 117 | resolve( 118 | "Message sent! " + 119 | data["MessageResponse"]["Result"][destinationNumber][ 120 | "StatusMessage" 121 | ] 122 | ); 123 | } 124 | }) 125 | ); 126 | } 127 | -------------------------------------------------------------------------------- /test/HeywoodHealthcareTest.js: -------------------------------------------------------------------------------- 1 | const scraper = require("../no-browser-site-scrapers/HeywoodHealthcare/index"); 2 | 3 | const { expect } = require("chai"); 4 | const moment = require("moment"); 5 | const file = require("../lib/file"); 6 | 7 | describe("HeywoodHealthcare availability test using scraper and saved HTML", function () { 8 | const beforeTime = moment(); 9 | 10 | it("should show no availability, and the results objects structure should conform", async function () { 11 | const results = await scraper(false, noAvailabilityFetchService); 12 | 13 | const afterTime = moment(); 14 | 15 | const expectedAvailability = false; 16 | expect(results.individualLocationData[0].hasAvailability).is.deep.equal( 17 | expectedAvailability 18 | ); 19 | 20 | expect(results.individualLocationData[0].availability).is.deep.equal( 21 | {} 22 | ); 23 | /* 24 | Structure conformance expectations: 25 | 26 | - All the timestamps are expected to be between before 27 | and after when the scraper was executed 28 | - Each site's results object must have a property named "hasAvailability" 29 | */ 30 | expect( 31 | moment(results.individualLocationData[0].timestamp).isBetween( 32 | beforeTime, 33 | afterTime 34 | ) 35 | ); 36 | 37 | // console.log(`${JSON.stringify(results)}`); 38 | 39 | if (process.env.DEVELOPMENT) { 40 | file.write( 41 | `${process.cwd()}/out_no_browser.json`, 42 | `${JSON.stringify(results, null, " ")}` 43 | ); 44 | } 45 | }); 46 | 47 | it("should show several slots", async function () { 48 | const results = await scraper(false, availabilityFetchService); 49 | 50 | const afterTime = moment(); 51 | 52 | const expectedAvailability = true; 53 | expect(results.individualLocationData[0].hasAvailability).is.deep.equal( 54 | expectedAvailability 55 | ); 56 | 57 | const date = Object.keys( 58 | results.individualLocationData[0].availability 59 | )[0]; 60 | 61 | expect( 62 | results.individualLocationData[0].availability[date] 63 | .numberAvailableAppointments 64 | ).equals(116); 65 | /* 66 | Structure conformance expectations: 67 | 68 | - All the timestamps are expected to be between before 69 | and after when the scraper was executed 70 | - Each site's results object must have a property named "hasAvailability" 71 | */ 72 | expect( 73 | moment(results.individualLocationData[0].timestamp).isBetween( 74 | beforeTime, 75 | afterTime 76 | ) 77 | ); 78 | 79 | // console.log(`${JSON.stringify(results)}`); 80 | 81 | if (process.env.DEVELOPMENT) { 82 | file.write( 83 | `${process.cwd()}/out_no_browser.json`, 84 | `${JSON.stringify(results, null, " ")}` 85 | ); 86 | } 87 | }); 88 | }); 89 | 90 | /* utilities */ 91 | 92 | /** Generator to feed filenames sequentially to the "with availability" test. */ 93 | function* noAvailabilityFilenames() { 94 | yield "firstPage-noSlots-hasMoreTimes-scripts-removed.html"; 95 | yield "secondPage-noSlots-scripts-removed.html"; 96 | } 97 | const noAvailabilityFilenameGenerator = noAvailabilityFilenames(); 98 | 99 | const noAvailabilityFetchService = { 100 | async fetchAvailability(/*site*/) { 101 | return loadTestHtmlFromFile( 102 | "HeywoodHealthcare", 103 | noAvailabilityFilenameGenerator.next().value 104 | ); 105 | }, 106 | }; 107 | 108 | /** Generator to feed filenames sequentially to the "with availability" test. */ 109 | function* availabilityFilenames() { 110 | yield "firstPage-noSlots-hasMoreTimes-scripts-removed.html"; 111 | yield "secondPage-lots0fSlots-scripts-removed.html"; 112 | yield "thirdPage-someSlots-noMoreTimes-scripts-removed.html"; 113 | } 114 | const availabilityFilenameGenerator = availabilityFilenames(); 115 | 116 | const availabilityFetchService = { 117 | async fetchAvailability(/*site*/) { 118 | return loadTestHtmlFromFile( 119 | "HeywoodHealthcare", 120 | availabilityFilenameGenerator.next().value 121 | ); 122 | }, 123 | }; 124 | 125 | /** 126 | * Loads the saved HTML of the website. 127 | * 128 | * @param {String} filename The filename only, include its extension 129 | */ 130 | function loadTestHtmlFromFile(testFolderName, filename) { 131 | const fs = require("fs"); 132 | const path = `${process.cwd()}/test/${testFolderName}/${filename}`; 133 | return fs.readFileSync(path, "utf8"); 134 | } 135 | -------------------------------------------------------------------------------- /site-scrapers/Walgreens/index.js: -------------------------------------------------------------------------------- 1 | const { site } = require("./config"); 2 | const usZips = require("us-zips"); 3 | const moment = require("moment"); 4 | const dataFormatter = require("./dataFormatter"); 5 | 6 | module.exports = async function GetAvailableAppointments(browser) { 7 | console.log(`${site.name} starting.`); 8 | const webData = await ScrapeWebsiteData(browser); 9 | console.log(`${site.name} done.`); 10 | return { 11 | parentLocationName: "Walgreens", 12 | isChain: true, 13 | timestamp: moment().format(), 14 | individualLocationData: dataFormatter.formatData(webData, site.website), 15 | }; 16 | }; 17 | 18 | async function waitAndClick(page, selector) { 19 | await page.waitForSelector(selector); 20 | await page.click(selector); 21 | return; 22 | } 23 | 24 | async function ScrapeWebsiteData(browser) { 25 | const page = await browser.newPage(); 26 | await page.goto(site.website); 27 | // note: this might happen too quickly - might not show up yet 28 | const needToLogIn = await Promise.race([ 29 | page.waitForSelector("#pf-dropdown-signin").then(() => true), 30 | page.waitForSelector("#pf-acc-signout").then(() => false), 31 | ]); 32 | 33 | if (needToLogIn) { 34 | await page.evaluate(() => 35 | document.querySelector(".sign-in-register a").click() 36 | ); 37 | await page.waitForSelector("#user_name"); 38 | await page.type("#user_name", process.env.WALGREENS_EMAIL); 39 | await page.waitForTimeout(1000); 40 | await page.type("#user_password", process.env.WALGREENS_PASSWORD); 41 | await page.waitForSelector("#submit_btn:not([disabled])"); 42 | await page.click("#submit_btn"); 43 | await page.waitForNavigation().then( 44 | () => {}, 45 | (err) => page.screenshot({ path: "walgreens.png" }) //if you get here, it's almost definitely because login failed. 46 | ); 47 | } 48 | 49 | const isChallenge = await Promise.race([ 50 | page.waitForSelector("#radio-security").then(() => true), 51 | page.waitForSelector("#pf-acc-signout").then(() => false), 52 | ]); 53 | 54 | // SECURITY CHALLENGE 55 | if (isChallenge) { 56 | await page.click("#radio-security"); 57 | await page.click("#optionContinue"); 58 | await page.waitForSelector("#secQues"); 59 | await page.type("#secQues", process.env.WALGREENS_CHALLENGE); 60 | await page.click("#validate_security_answer"); 61 | await page.waitForNavigation(); 62 | } 63 | const availableLocations = {}; 64 | 65 | const uniqueZips = [...new Set(site.locations.map((site) => site.zip))]; 66 | 67 | // SEARCH PAGE 68 | for (const zip of uniqueZips) { 69 | const { latitude, longitude } = usZips[zip]; 70 | const todayString = new moment().local().format("YYYY-MM-DD"); 71 | 72 | let postResponse = await page.evaluate( 73 | (latitude, longitude, todayString) => 74 | new Promise((resolve) => { 75 | fetch( 76 | "https://www.walgreens.com/hcschedulersvc/svc/v2/immunizationLocations/timeslots", 77 | { 78 | method: "POST", 79 | headers: { 80 | "Content-Type": "application/json", 81 | }, 82 | body: JSON.stringify({ 83 | position: { 84 | latitude: latitude, 85 | longitude: longitude, 86 | }, 87 | vaccine: { productId: "" }, 88 | appointmentAvailability: { 89 | startDateTime: todayString, 90 | }, 91 | radius: 25, 92 | size: 25, 93 | serviceId: "99", 94 | state: "MA", 95 | }), 96 | } 97 | ) 98 | .then((response) => response.json()) 99 | .then((data) => { 100 | resolve(data); // fetch won't reject 404 or 500 101 | }) 102 | .catch((error) => { 103 | resolve(error); // only happens on network failure 104 | }); 105 | }), 106 | latitude, 107 | longitude, 108 | todayString 109 | ); 110 | console.log(postResponse); 111 | if (postResponse && postResponse.locations) { 112 | postResponse.locations.forEach((location) => { 113 | availableLocations[location.partnerLocationId] = location; 114 | }); 115 | } 116 | } 117 | page.close(); 118 | return Object.values(availableLocations); 119 | } 120 | -------------------------------------------------------------------------------- /test/dataDefaulter.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const dataDefaulter = require("./../data/dataDefaulter"); 3 | const moment = require("moment"); 4 | 5 | // NOTE: Timestamps don't really matter in these tests, as long as they exist. 6 | 7 | const realisticTestData = [ 8 | { 9 | name: "Atrius Health", 10 | street: "100 Second Avenue ", 11 | city: "Needham", 12 | zip: "02494", 13 | website: "https://myhealth.atriushealth.org/fr/", 14 | dphLink: "https://myhealth.atriushealth.org/DPH", 15 | signUpLink: "https://myhealth.atriushealth.org/fr/", 16 | availability: {}, 17 | hasAvailability: false, 18 | timestamp: moment().local(), 19 | }, 20 | { 21 | id: 24181, 22 | name: "DoubleTree Hotel - Danvers", 23 | street: "50 Ferncroft Rd.", 24 | city: "Danvers", 25 | zip: "01923", 26 | signUpLink: "https://curative.com/sites/24181", 27 | hasAvailability: true, 28 | availability: { 29 | "02/27/2021": { 30 | numberAvailableAppointments: 1, 31 | hasAvailability: true, 32 | }, 33 | "02/28/2021": { 34 | numberAvailableAppointments: 0, 35 | hasAvailability: false, 36 | }, 37 | }, 38 | timestamp: moment().local(), 39 | }, 40 | ]; 41 | 42 | const realisticDefaultData = [ 43 | { 44 | id: 24181, 45 | name: "DoubleTree Hotel - Danvers", 46 | street: "50 Ferncroft Rd.", 47 | city: "Danvers", 48 | zip: "01923", 49 | signUpLink: "https://curative.com/sites/24181", 50 | hasAvailability: false, 51 | availability: { 52 | "02/18/2021": { 53 | numberAvailableAppointments: 0, 54 | hasAvailability: false, 55 | }, 56 | "02/19/2021": { 57 | numberAvailableAppointments: 0, 58 | hasAvailability: false, 59 | }, 60 | }, 61 | timestamp: moment().local(), 62 | }, 63 | { 64 | name: "Family Practice Group", 65 | street: "11 Water Street, Suite 1-A", 66 | city: "Arlington", 67 | zip: "02476", 68 | website: "https://bookfpg.timetap.com/#/", 69 | signUpLink: "https://bookfpg.timetap.com/#/", 70 | hasAvailability: false, 71 | timestamp: moment().local(), 72 | }, 73 | ]; 74 | 75 | describe("dataDefaulter Simple behavior", () => { 76 | it("should return scraped results when no cache values are present", () => { 77 | assert.deepStrictEqual( 78 | dataDefaulter.mergeResults(realisticTestData, []), 79 | realisticTestData 80 | ); 81 | }); 82 | it("should return cached results when no scraped results are present", () => { 83 | assert.deepStrictEqual( 84 | dataDefaulter.mergeResults([], realisticDefaultData), 85 | realisticDefaultData 86 | ); 87 | }); 88 | }); 89 | 90 | describe("dataDefaulter Conditionally inserting defaults", () => { 91 | const finalResults = dataDefaulter.mergeResults( 92 | realisticTestData, 93 | realisticDefaultData 94 | ); 95 | it("has the correct number of entries", () => { 96 | // Atrius, Danvers, and then default Family Practice Group 97 | assert.strictEqual(finalResults.length, 3); 98 | }); 99 | it("does not replace data with cache", () => { 100 | // Danvers stays the same 101 | assert.deepStrictEqual(finalResults[1], realisticTestData[1]); 102 | }); 103 | 104 | it("adds default data where an entry doesn't exist", () => { 105 | // Family Practice Group is added to the results array 106 | assert.deepStrictEqual(finalResults[2], realisticDefaultData[1]); 107 | }); 108 | }); 109 | 110 | describe("dataDefaulter Tolerance for stale data", () => { 111 | const secondsOfTolerance = 60; 112 | it("does not include stale data if specified", () => { 113 | const timestampedDefault = { 114 | ...realisticDefaultData[0], 115 | timestamp: new Date() - (secondsOfTolerance + 1) * 1000, 116 | }; 117 | assert.deepStrictEqual( 118 | dataDefaulter.mergeResults( 119 | [], 120 | [timestampedDefault], 121 | secondsOfTolerance 122 | ), 123 | [] 124 | ); 125 | }); 126 | 127 | it("includes recent data within the tolerance timeframe", () => { 128 | const timestampedDefault = { 129 | ...realisticDefaultData[0], 130 | timestamp: new Date() - (secondsOfTolerance - 1) * 1000, 131 | }; 132 | assert.deepStrictEqual( 133 | dataDefaulter.mergeResults( 134 | [], 135 | [timestampedDefault], 136 | secondsOfTolerance 137 | ), 138 | [timestampedDefault] 139 | ); 140 | }); 141 | }); 142 | 143 | describe("dataDefaulter Key generator", () => { 144 | it("generates the expected key", () => { 145 | assert.strictEqual( 146 | dataDefaulter.generateKey(realisticTestData[1]), 147 | "doubletreehoteldanvers|50ferncroftrd|danvers|01923|" 148 | ); 149 | }); 150 | }); 151 | -------------------------------------------------------------------------------- /no-browser-site-scrapers/ZocDoc/zocdocBase.js: -------------------------------------------------------------------------------- 1 | const fetch = require("node-fetch"); 2 | const moment = require("moment"); 3 | 4 | async function fetchAvailability() { 5 | const response = await fetch("https://api.zocdoc.com/directory/v2/gql", { 6 | headers: { 7 | accept: "*/*", 8 | "accept-language": "en-US,en;q=0.9", 9 | "content-type": "application/json", 10 | "sec-ch-ua": 11 | '"Google Chrome";v="89", "Chromium";v="89", ";Not A Brand";v="99"', 12 | "sec-ch-ua-mobile": "?0", 13 | "sec-fetch-dest": "empty", 14 | "sec-fetch-mode": "cors", 15 | "sec-fetch-site": "same-site", 16 | "x-zd-application": "patient-web-app", 17 | "x-zd-referer": "https://www.zocdoc.com/vaccine/screener?state=MA", 18 | "x-zd-url": 19 | "https://www.zocdoc.com/vaccine/search/MA?flavor=state-search", 20 | "x-zdata": "eyJob3N0Ijoid3d3LnpvY2RvYy5jb20ifQ==", 21 | "zd-application-name": "patient-web-app", 22 | "zd-referer": "https://www.zocdoc.com/vaccine/screener?state=MA", 23 | "zd-session-id": "7003f0975861439882d25da7d6d87954", 24 | "zd-tracking-id": "b28be354-5d7f-4ea2-bdad-1e05b17554d0", 25 | "zd-url": 26 | "https://www.zocdoc.com/vaccine/search/MA?flavor=state-search", 27 | "zd-user-agent": 28 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36", 29 | }, 30 | referrer: "https://www.zocdoc.com/", 31 | referrerPolicy: "strict-origin-when-cross-origin", 32 | body: 33 | '{"operationName":"providerLocationsAvailability","variables":{"directoryId":"-1","insurancePlanId":"-1","isNewPatient":true,"jumpAhead":true,"numDays":14,"procedureId":"5243","searchRequestId":"7345a02c-6dac-4108-8e76-1ac9f5c0052b","timeFilter":"AnyTime","firstAvailabilityMaxDays":30,"providerLocationIds":["pr_fSHH-Tyvm0SZvoK3pfH8tx|lo_EMLPse6C60qr6_M2rJmilx","pr_VUnpWUtg1k2WFBMK8IhZkx|lo_PhFQcSZjdUKZUHp63gDcmx","pr_pEgrY3r5qEuYKsKvc4Kavx|lo_f3k2t812AUa9NTIYJpbuKx","pr_TeD-JuoydUKqszEn2ATb8h|lo_jbrpfIgELEWWL2j5d3t6Sh","pr_iXjD9x2P-0OrLNoIknFr8R|lo_xmpTUhghfUC2n6cs3ZGHhh","pr_BDBebslqJU2vrCAvVMhYeh|lo_zDtK7NZWO0S_rgUqjxD1hB","pr_4Vg_3ZeLY0aHJJxsCU-WhB|lo_VA_6br22m02Iu57vrHWtaB","pr_CUmBnwtlz0C16bif5EU0IR|lo_X6zHHncQnkqyLp-rvpi1_R"]},"query":"query providerLocationsAvailability($directoryId: String, $insurancePlanId: String, $isNewPatient: Boolean, $isReschedule: Boolean, $jumpAhead: Boolean, $firstAvailabilityMaxDays: Int, $numDays: Int, $procedureId: String, $providerLocationIds: [String], $searchRequestId: String, $startDate: String, $timeFilter: TimeFilter, $widget: Boolean) {\\n providerLocations(ids: $providerLocationIds) {\\n id\\n ...availability\\n __typename\\n }\\n}\\n\\nfragment availability on ProviderLocation {\\n id\\n provider {\\n id\\n monolithId\\n __typename\\n }\\n location {\\n id\\n monolithId\\n state\\n phone\\n __typename\\n }\\n availability(directoryId: $directoryId, insurancePlanId: $insurancePlanId, isNewPatient: $isNewPatient, isReschedule: $isReschedule, jumpAhead: $jumpAhead, firstAvailabilityMaxDays: $firstAvailabilityMaxDays, numDays: $numDays, procedureId: $procedureId, searchRequestId: $searchRequestId, startDate: $startDate, timeFilter: $timeFilter, widget: $widget) {\\n times {\\n date\\n timeslots {\\n isResource\\n startTime\\n __typename\\n }\\n __typename\\n }\\n firstAvailability {\\n startTime\\n __typename\\n }\\n showGovernmentInsuranceNotice\\n timesgridId\\n today\\n __typename\\n }\\n __typename\\n}\\n"}', 34 | method: "POST", 35 | mode: "cors", 36 | }) 37 | .then((response) => response.json()) 38 | .then((json) => { 39 | return json; 40 | }) 41 | .catch((error) => console.log(`error fetching availability: ${error}`)); 42 | 43 | return response ? response : {}; 44 | } 45 | 46 | /** 47 | * 48 | * @param {*} availabilityResponses 49 | * @returns 50 | */ 51 | function parseAvailability(availabilityResponses) { 52 | function reformatDate(date) { 53 | var dateObj = moment(date + "T00:00:00"); 54 | return `${dateObj.month() + 1}/${dateObj.date()}/${dateObj.year()}`; 55 | } 56 | 57 | const data = availabilityResponses.data; 58 | const pLocations = data.providerLocations; 59 | 60 | const results = {}; 61 | 62 | Object.values(pLocations).map((pLoc) => { 63 | if (data.providerLocations.length > 0) { 64 | const times = pLoc.availability.times; 65 | 66 | const providerIdDateTimes = {}; 67 | times 68 | .filter((timeSlotDay) => timeSlotDay.timeslots.length > 0) 69 | .forEach((timeSlotDay) => { 70 | providerIdDateTimes[reformatDate(timeSlotDay.date)] = { 71 | numberAvailableAppointments: 72 | timeSlotDay.timeslots.length, 73 | hasAvailability: true, 74 | }; 75 | }); 76 | results[pLoc.id] = { availability: providerIdDateTimes }; 77 | } 78 | }); 79 | 80 | return results; 81 | } 82 | 83 | module.exports = { 84 | fetchAvailability, 85 | parseAvailability, 86 | }; 87 | --------------------------------------------------------------------------------