├── .dockerignore ├── .eslintrc.js ├── .github ├── pull_request_template.md └── workflows │ ├── docker.yaml │ ├── lint.yaml │ └── test.yaml ├── .gitignore ├── .husky └── pre-commit ├── .nvmrc ├── .prettierignore ├── .yarnrc.yml ├── Dockerfile ├── LICENSE ├── README.md ├── babel.config.json ├── data ├── README.md ├── cache_2022_summer │ ├── dev_data │ │ └── v2 │ │ │ ├── NeuEmployee.cache.json │ │ │ └── classes.cache.json │ └── requests │ │ ├── classParser.cache.msgpack │ │ ├── subjectAbberviationParser.cache.msgpack │ │ └── termParser.cache.msgpack └── major.json ├── docs ├── .nojekyll ├── CNAME ├── README.md ├── _sidebar.md ├── banner │ ├── apiSpec.html │ └── banner.md ├── deployment │ └── deploy.md ├── getting-started │ ├── prereqs.md │ ├── running.md │ └── stored-cache.md ├── helpful-commands.md ├── index.html ├── indexing │ └── elasticsearch.md ├── infrastructure │ ├── infrastructure.md │ ├── jumphost.md │ ├── migration │ │ ├── migration.md │ │ └── staging-migration.md │ └── scraping.md ├── notifications │ ├── notifications.md │ ├── twilioLandingPage.png │ ├── twilioSetup.png │ └── twilioVerifySID.png ├── scrapers │ └── scraping.md ├── search-logo-red.png └── updater │ └── updating.md ├── graphql ├── index.ts ├── resolvers │ ├── class.ts │ ├── major.ts │ ├── search.ts │ └── termInfo.ts ├── tests │ ├── __snapshots__ │ │ ├── class.test.seq.ts.snap │ │ ├── major.test.seq.ts.snap │ │ └── search.test.seq.ts.snap │ ├── class.test.seq.ts │ ├── major.test.seq.ts │ └── search.test.seq.ts └── typeDefs │ ├── class.js │ ├── classOccurrence.js │ ├── employee.js │ ├── major.js │ ├── majorOccurrence.js │ ├── search.js │ └── termInfo.js ├── infrastructure ├── aws │ ├── push-image │ ├── redeploy │ ├── run-task │ └── scrape ├── dev │ ├── compose.yaml │ ├── docker-compose-server.yml │ └── docker-postgresql-multiple-databases │ │ └── create-multiple-postgresql-databases.sh ├── prod │ └── entrypoint.sh └── terraform │ ├── .gitignore │ ├── acm.tf │ ├── alb.tf │ ├── ecr.tf │ ├── ecs_long_arn.tf │ ├── github_user.tf │ ├── jumphost.tf │ ├── modules │ └── course-catalog-api │ │ ├── README.md │ │ ├── domain.tf │ │ ├── ecs.tf │ │ ├── elasticsearch.tf │ │ ├── label.tf │ │ ├── outputs.tf │ │ ├── providers.tf │ │ ├── rds.tf │ │ └── variables.tf │ ├── network.tf │ ├── outputs.tf │ ├── prod.tf │ ├── providers.tf │ ├── remote.tf │ ├── staging.tf │ └── variables.tf ├── jest.config.js ├── package.json ├── prisma ├── .env ├── migrations │ ├── 20201230025329_initial_migration │ │ └── migration.sql │ ├── 20210117041214_update_major_table │ │ └── migration.sql │ ├── 20210125025244_drop_users │ │ └── migration.sql │ ├── 20210728002855_add_twilio_users │ │ └── migration.sql │ ├── 20210807051602_add_followed_section_relation │ │ └── migration.sql │ ├── 20210807062236_rename_phone_number_field │ │ └── migration.sql │ ├── 20210807172704_add_followed_courses │ │ └── migration.sql │ ├── 20210812025534_user_phone_number_unique │ │ └── migration.sql │ ├── 20211013174543_add_update_time_to_secs │ │ └── migration.sql │ ├── 20211017200010_add_term_i_ds │ │ └── migration.sql │ ├── 20211025231638_new_last_update_time │ │ └── migration.sql │ ├── 20211119161621_delete_cascade │ │ └── migration.sql │ ├── 20220930184332_streamline_employees │ │ └── migration.sql │ ├── 20231015192547_add_active_field_to_term_info │ │ └── migration.sql │ ├── 20240814155920_add_notif_counts │ │ └── migration.sql │ ├── 20250326202637_notif_logging │ │ └── migration.sql │ └── migration_lock.toml └── schema.prisma ├── scrapers ├── cache.ts ├── classes │ ├── classMapping.json │ ├── main.ts │ ├── parsersxe │ │ ├── __mocks__ │ │ │ ├── subjectAbbreviationParser.ts │ │ │ └── subjectAbbreviationTable.json │ │ ├── bannerv9Parser.ts │ │ ├── classParser.ts │ │ ├── meetingParser.ts │ │ ├── prereqParser.ts │ │ ├── sectionParser.ts │ │ ├── startup.ts │ │ ├── subjectAbbreviationParser.ts │ │ ├── termListParser.ts │ │ ├── termParser.ts │ │ ├── tests │ │ │ ├── __snapshots__ │ │ │ │ ├── classParser.test.js.snap │ │ │ │ ├── meetingParser.test.js.snap │ │ │ │ ├── prereqParser.test.js.snap │ │ │ │ ├── sectionParser.test.js.snap │ │ │ │ ├── termListParser.test.js.snap │ │ │ │ └── util.test.js.snap │ │ │ ├── bannerv9Parser.test.ts │ │ │ ├── classParser.test.js │ │ │ ├── data │ │ │ │ ├── classParser.data.js │ │ │ │ ├── meetingParser.data.js │ │ │ │ ├── prereqParser.data.js │ │ │ │ ├── sectionParser.data.js │ │ │ │ └── util │ │ │ │ │ ├── 1.html │ │ │ │ │ ├── 2.html │ │ │ │ │ └── 3.html │ │ │ ├── meetingParser.test.js │ │ │ ├── prereqParser.test.js │ │ │ ├── sectionParser.test.js │ │ │ ├── subjectAbbreviationParser.test.js │ │ │ ├── termListParser.test.js │ │ │ ├── termParser.test.js │ │ │ └── util.test.js │ │ └── util.ts │ ├── processors │ │ ├── addPrerequisiteFor.ts │ │ ├── markMissingRequisites.ts │ │ ├── simplifyPrereqs.ts │ │ └── tests │ │ │ ├── addPrerequisiteFor.test.js │ │ │ ├── data │ │ │ └── termDump.json │ │ │ ├── markMissingPrereqs.test.js │ │ │ ├── simplifyPrereqs.test.ts │ │ │ └── testData.js │ ├── termDump.ts │ └── tests │ │ └── classScraper.test.ts ├── employees │ ├── employeeMapping.json │ ├── employees.ts │ ├── matchEmployees.ts │ └── tests │ │ ├── __snapshots__ │ │ └── employees.test.ts.snap │ │ ├── data │ │ └── employees │ │ │ └── employee_results_BE.json │ │ └── employees.test.ts ├── filters.ts ├── main.ts └── request.ts ├── scripts ├── migrate_major_data.ts ├── populateES.ts ├── resetIndex.ts └── resetIndexWithoutLoss.ts ├── serializers ├── courseSerializer.ts ├── elasticCourseSerializer.ts ├── elasticProfSerializer.ts ├── hydrateCourseSerializer.ts ├── hydrateProfSerializer.ts ├── hydrateSerializer.ts └── profSerializer.ts ├── services ├── dumpProcessor.ts ├── notificationsManager.ts ├── notifyer.ts ├── prisma.ts ├── searcher.ts └── updater.ts ├── template.env ├── tests ├── README.md ├── database │ ├── __snapshots__ │ │ └── search.test.seq.ts.snap │ ├── dbTestEnv.ts │ ├── dumpProcessor.test.seq.ts │ ├── jest.config.js │ ├── notificationsManager.test.seq.ts │ ├── search.test.seq.ts │ └── updater.test.seq.ts ├── end_to_end │ ├── README.md │ ├── babel.config.json │ ├── elastic.test.git.ts │ ├── jest.config.js │ ├── scraper.test.git.ts │ ├── search.test.git.ts │ ├── setup.test.git.ts │ └── updater.test.git.ts ├── general │ ├── data │ │ ├── employeeMap.json │ │ ├── employeesSearchIndex.json │ │ ├── mockTermDump.json │ │ └── searchTestResultObjects.json │ ├── macros.test.ts │ ├── notifyer.test.ts │ ├── twilio.test.ts │ └── utils │ │ ├── elastic.test.ts │ │ ├── keys.test.ts │ │ └── macros.test.ts └── unit │ ├── jest.config.js │ ├── scrapers │ ├── cache.test.unit.ts │ └── request.test.unit.ts │ └── services │ └── searcher.test.unit.ts ├── tsconfig.eslint.json ├── tsconfig.json ├── twilio ├── client.ts ├── notifs.ts └── server.ts ├── types ├── notifTypes.ts ├── requestTypes.ts ├── scraperTypes.ts ├── searchTypes.ts ├── serializerTypes.ts └── types.ts ├── utils ├── elastic.ts ├── keys.ts └── macros.ts └── yarn.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | cache* 3 | dist 4 | public 5 | *.log 6 | .DS_Store 7 | data/* 8 | !data/* 9 | *.terraform.lock.hcl 10 | .env 11 | cache.zip 12 | .idea/* 13 | logs/ 14 | coverage/ -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | es2021: true, 4 | jest: true, 5 | node: true, 6 | }, 7 | // Unless you have a good reason, keep `extends` in the given order 8 | extends: [ 9 | "eslint:recommended", 10 | // Overrides the settings from above ^ which don't apply to Typescript 11 | // Keep these 3 in the same order 12 | // "plugin:@typescript-eslint/eslint-recommended", 13 | "plugin:@typescript-eslint/recommended", 14 | "plugin:promise/recommended", 15 | // "airbnb-base", 16 | // "airbnb-typescript/base", 17 | // Overrides AirBnB styles; keep in this order 18 | // "plugin:prettier/recommended", 19 | ], 20 | parser: "@typescript-eslint/parser", 21 | parserOptions: { 22 | project: "./tsconfig.eslint.json", 23 | ecmaVersion: 2021, 24 | }, 25 | plugins: ["@typescript-eslint/eslint-plugin", "deprecation" /*"prettier"*/], 26 | rules: { 27 | // // "@typescript-eslint/explicit-function-return-type": "off", 28 | // // "@typescript-eslint/explicit-module-boundary-types": "off", 29 | // // "@typescript-eslint/camelcase": "off", 30 | // "deprecation/deprecation": "warn", 31 | // // This is overridden from AirBnB's guide, since they (subjectively) ban for..of loops. 32 | // // https://github.com/airbnb/javascript/issues/1122#issuecomment-267580623 33 | // "no-restricted-syntax": [ 34 | // "error", 35 | // { 36 | // selector: "ForInStatement", 37 | // message: 38 | // "for..in loops iterate over the entire prototype chain, which is virtually never what you want. Use Object.{keys,values,entries}, and iterate over the resulting array.", 39 | // }, 40 | // ], 41 | // "no-underscore-dangle": "off", 42 | // "no-continue": "off", 43 | // "prefer-destructuring": "off", 44 | }, 45 | overrides: [ 46 | { 47 | files: ["*.js", "*.ts"], 48 | rules: { 49 | "@typescript-eslint/explicit-function-return-type": [ 50 | "warn", 51 | { 52 | allowExpressions: true, 53 | }, 54 | ], 55 | "@typescript-eslint/explicit-module-boundary-types": "off", 56 | "@typescript-eslint/ban-ts-comment": "warn", 57 | // We disable the base rule so it doesn't interfere w/ the TS one 58 | "no-unused-vars": "off", 59 | "@typescript-eslint/no-unused-vars": [ 60 | "warn", 61 | // Ignore this rule for variables starting with an underscore (eg. _ignoreme) 62 | { varsIgnorePattern: "^_" }, 63 | ], 64 | }, 65 | }, 66 | ], 67 | }; 68 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Purpose 2 | 3 | _In 2-3 sentences - what does this code actually do, and why?_ 4 | 5 | # Tickets 6 | 7 | _What Trello tickets (if any) are associated with this PR?_ 8 | 9 | - 10 | 11 | # Contributors 12 | 13 | _Who worked on this PR? Tag them with their Github `@username` for future reference_ 14 | 15 | Use the **"Assignees"** feature in Github 16 | 17 | # Feature List 18 | 19 | _Expand on the purpose - what does this code do, and how? This should be a list of the changes, more in-depth and technical_ 20 | 21 | - 22 | - 23 | - 24 | 25 | # Notes (Optional) 26 | 27 | _Is there anything reviewers should note? Do we need to add something later? Any followups? Was this ticket easier/harder than predicted?_ 28 | 29 | # Reviewers 30 | 31 | Primary reviewer: 32 | 33 | - Pick one primary reviewer 34 | - The team member with the most relevant knowledge about the code area this PR touches 35 | - **NOT** an author of the PR 36 | - If the primary reviewer is the project lead, _select two primary reviewers_ 37 | - Goal: facilitate knowledge transfer to other team members 38 | - Primary reviewers **are required to approve the PR** before it can be merged 39 | 40 | **Primary**: 41 | 42 | Use the **"Reviewers"** feature in Github 43 | 44 | Secondary reviewers: 45 | 46 | - Pick as many as appropriate — anyone who would benefit from reading the PR 47 | - Tag them using their Github usernames below 48 | 49 | **Secondary**: 50 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: Linting & Static code checking 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - master 8 | 9 | jobs: 10 | prettier: 11 | name: prettier 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | with: 16 | ref: ${{ github.head_ref }} 17 | 18 | - name: Prettify code 19 | uses: creyD/prettier_action@v4.3 20 | with: 21 | prettier_options: --write **/*.{js,ts} 22 | env: 23 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 24 | 25 | lint: 26 | name: Lint & Type checks 27 | runs-on: ubuntu-latest 28 | steps: 29 | - uses: actions/checkout@v4 30 | 31 | - name: Install Node 32 | uses: actions/setup-node@v4 33 | with: 34 | node-version: "22" 35 | 36 | - name: Enable Corepack 37 | run: corepack enable 38 | 39 | - name: Install deps 40 | run: yarn install 41 | 42 | - name: Check linter 43 | run: yarn lint 44 | 45 | - name: Check types 46 | run: yarn tsc 47 | 48 | dependency_checks: 49 | name: Dependency Checks 50 | runs-on: ubuntu-latest 51 | steps: 52 | - uses: actions/checkout@v4 53 | 54 | - name: Install Node 55 | uses: actions/setup-node@v4 56 | with: 57 | node-version: "22" 58 | 59 | - name: Enable Corepack 60 | run: corepack enable 61 | 62 | - name: Install deps 63 | run: yarn install 64 | 65 | - name: Checks for duplicate definitions in the yarn lockfile 66 | run: yarn dedupe --check 67 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | cache*/ 3 | dist 4 | public 5 | *.log 6 | .DS_Store 7 | data/* 8 | !data/* 9 | *.terraform.lock.hcl 10 | .env 11 | cache.zip 12 | .idea/* 13 | logs/ 14 | coverage/ 15 | .idea/ 16 | 17 | # Yarn 18 | .pnp.* 19 | .yarn/* 20 | !.yarn/patches 21 | !.yarn/plugins 22 | !.yarn/releases 23 | !.yarn/sdks 24 | !.yarn/versions 25 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | yarn pretty-quick --staged && yarn eslint --no-error-on-unmatched-pattern --fix $(git diff --name-only HEAD | grep -E '\\.(js|ts)$' | xargs) 2 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 22 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # Same as .gitignore 2 | node_modules/ 3 | cache 4 | dist 5 | public 6 | *.log 7 | .DS_Store 8 | data/* 9 | !data/major.json 10 | *.terraform.lock.hcl 11 | .env 12 | 13 | # Extras 14 | *.html 15 | # For elasticsearch query files used by VSCode extension 16 | *.es 17 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | nodeLinker: node-modules 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # build environment 2 | FROM node:22-alpine AS build 3 | WORKDIR /app 4 | 5 | # Install deps 6 | COPY package.json /app/package.json 7 | COPY yarn.lock /app/yarn.lock 8 | COPY .yarnrc.yml /app/.yarnrc.yml 9 | 10 | RUN corepack enable 11 | RUN yarn install --frozen-lockfile 12 | 13 | # Copy source 14 | COPY graphql /app/graphql 15 | COPY prisma /app/prisma 16 | COPY scrapers /app/scrapers 17 | COPY scripts /app/scripts 18 | COPY serializers /app/serializers 19 | COPY services /app/services 20 | COPY twilio /app/twilio 21 | COPY types /app/types 22 | COPY utils /app/utils 23 | COPY infrastructure/prod /app 24 | COPY babel.config.json /app 25 | 26 | RUN yarn build 27 | 28 | FROM node:22-alpine AS dist 29 | WORKDIR /dist 30 | RUN corepack enable 31 | 32 | COPY --from=build /app/dist . 33 | 34 | # TODO: This should be a `yarn workspaces focus --production` but 35 | # the dev and non-dev deps are a tangled mess rn 36 | RUN yarn workspaces focus 37 | 38 | # Manually install openssl for Prisma. The Alpine image should already 39 | # have it, but Prisma fails to detect it for some reason 40 | RUN set -ex; \ 41 | apk update; \ 42 | apk add --no-cache \ 43 | openssl 44 | 45 | # Get RDS Certificate 46 | RUN apk update && apk add wget && rm -rf /var/cache/apk/* \ 47 | && wget "https://s3.amazonaws.com/rds-downloads/rds-ca-2019-root.pem" 48 | ENV dbCertPath=/app/rds-ca-2019-root.pem 49 | 50 | ENV NODE_ENV=prod 51 | 52 | ENTRYPOINT ["/dist/entrypoint.sh"] 53 | 54 | EXPOSE 4000 8080 55 | CMD ["node", "graphql/index.js"] 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The Course Catalog API 2 | 3 | ## Overview 4 | 5 | **"SearchNEU"**, as a complete application, exists in two parts: 6 | 7 | - Backend: The backend is our API server, which does all of the heavy lifting. This stores all of the course data - names, IDs, sections, descriptions, etc. It also handles notifications. The user can interact with this data using the frontend. 8 | - The backend is also used by other applications (like GraduateNU). 9 | - Frontend: The frontend is what a user sees when they go to [searchneu.com](https://searchneu.com). It does not have any data on its own - whenever a user searches for a course, the frontend sends a request to the backend, which returns the data. The frontend handles display; the backend handles data processing. 10 | 11 | ## Docs 12 | 13 | Our documentation can be found at [https://apidocs.searchneu.com/](https://apidocs.searchneu.com/) 14 | -------------------------------------------------------------------------------- /babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": { 7 | "node": "16" 8 | } 9 | } 10 | ], 11 | "@babel/preset-typescript" 12 | ], 13 | "plugins": [], 14 | "ignore": ["node_modules"] 15 | } 16 | -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | The cache file in this directory is used for our Github workflow end-to-end tests. 2 | 3 | Please do not delete it :( -------------------------------------------------------------------------------- /data/cache_2022_summer/requests/classParser.cache.msgpack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandboxnu/course-catalog-api/8bc26893537521e25e1ca4d8deb1c9edd946e761/data/cache_2022_summer/requests/classParser.cache.msgpack -------------------------------------------------------------------------------- /data/cache_2022_summer/requests/subjectAbberviationParser.cache.msgpack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandboxnu/course-catalog-api/8bc26893537521e25e1ca4d8deb1c9edd946e761/data/cache_2022_summer/requests/subjectAbberviationParser.cache.msgpack -------------------------------------------------------------------------------- /data/cache_2022_summer/requests/termParser.cache.msgpack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandboxnu/course-catalog-api/8bc26893537521e25e1ca4d8deb1c9edd946e761/data/cache_2022_summer/requests/termParser.cache.msgpack -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandboxnu/course-catalog-api/8bc26893537521e25e1ca4d8deb1c9edd946e761/docs/.nojekyll -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | apidocs.searchneu.com -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | Welcome to the SearchNEU documentation! 2 | 3 | ## Overview 4 | 5 | **"SearchNEU"**, as a complete application, exists in two parts: 6 | 7 | - Backend: The backend is our API server, which does all of the heavy lifting. This stores all of the course data - names, IDs, sections, descriptions, etc. It also handles notifications. The user can interact with this data using the frontend. 8 | - The backend is also used by other applications (like GraduateNU). 9 | - Frontend: The frontend is what a user sees when they go to [searchneu.com](https://searchneu.com). It does not have any data on its own - whenever a user searches for a course, the frontend sends a request to the backend, which returns the data. The frontend handles display; the backend handles data processing. 10 | 11 | This is the documentation for the **backend**. 12 | -------------------------------------------------------------------------------- /docs/_sidebar.md: -------------------------------------------------------------------------------- 1 | - [🔗 Frontend documentation](https://docs.searchneu.com) 2 | 3 | --- 4 | 5 | - Getting started 6 | - [Prerequisites](getting-started/prereqs.md) 7 | - [Cache setup](getting-started/stored-cache.md) 8 | - [Running the backend](getting-started/running.md) 9 | - NEU's Course Catalog 10 | - [Banner](banner/banner.md) 11 | - Technical Details 12 | - [Scrapers](scrapers/scraping.md) 13 | - [Updater](updater/updating.md) 14 | - [Indexing/Searching](indexing/elasticsearch.md) 15 | - [Notifications](notifications/notifications.md) 16 | - Infrastructure 17 | - [Overview](infrastructure/infrastructure.md) 18 | - [Jumphost](infrastructure/jumphost.md) 19 | - [Scraping in Prod](infrastructure/scraping.md) 20 | - [Migration](infrastructure/migration/migration.md) 21 | - [Migrating the staging URL](infrastructure/migration/staging-migration.md) 22 | - [Deployment](deployment/deploy.md) 23 | - [Helpful Commands](helpful-commands.md) 24 | -------------------------------------------------------------------------------- /docs/banner/banner.md: -------------------------------------------------------------------------------- 1 | ## What is Banner? 2 | 3 | Banner is a product produced by Ellcian, which designs various solutions for higher education institutions. 4 | 5 | For Northeastern, our Banner instance can be found [here](https://nubanner.neu.edu/StudentRegistrationSsb/ssb/registration), and generally encompasses registration, planning, searching for courses, etc. 6 | 7 | ## How do we use it? 8 | 9 | Banner has no public API documentation - therefore, everything we use has been reverse-engineered. 10 | 11 | Check out the [API documentation](https://jennydaman.gitlab.io/nubanned/dark.html), and if that doesn't work, we have a local copy available here. 12 | 13 | ## What are "term IDs"? 14 | 15 | Banner uses "term IDs" to internally represent terms. For undergraduates, a "term" is a "semester", but a "term" can also be a "quarter" (for LAW and CPS) 16 | 17 | Currently, the term IDs are classified as follows - however, don't rely on this always being true! 18 | 19 | The structure of a term ID is `` 20 | 21 | - `` is the 4-digit academic year (ie. the year in which this academic year ends) 22 | - For example, for the 2022-2023 academic year, we would use `2023` 23 | - `` is a 1-digit code representing the season of the term. 24 | - `1` - Fall 25 | - `2` - Winter 26 | - `3` - Spring 27 | - `4` - Summer I 28 | - `5` - Full Summer 29 | - `6` - Summer II 30 | - `` is a 1-digit code representing what type this term is 31 | - `0` - Undergraduate Semester 32 | - `2` - Law Semester 33 | - `4` - CPS Semester 34 | - `5` - CPS Quarter 35 | - `8` - Law Quarter 36 | 37 | ### Examples 38 | 39 | - `202310` - Undergrad Fall Semester for the 2022-2023 academic year 40 | - `202125` - Winter CPS Quarter for 2020-2021 academic year 41 | - `202130` - Undergrad Spring Semester for 2020-2021 school year 42 | -------------------------------------------------------------------------------- /docs/getting-started/prereqs.md: -------------------------------------------------------------------------------- 1 | # Prerequisites/Dependencies 2 | 3 | This setup guide tries its best not to assume people have background knowledge, and provides basic setup instructions. 4 | 5 | ### Terminal 6 | 7 | To work on this project, you\'ll need a UNIX-based terminal. Mac/Linux users already have this. Windows users should install WSL, the Windows Subsystem for Linux. 8 | 9 | [WSL installation instructions](https://docs.microsoft.com/en-us/windows/wsl/install-win10). Also make sure to [upgrade to version 2](https://docs.microsoft.com/en-us/windows/wsl/install#upgrade-version-from-wsl-1-to-wsl-2). 10 | 11 | ?> **Tip:** We recommend installing [Windows Terminal](https://docs.microsoft.com/en-us/windows/terminal/install) for a better development experience than the Command Prompt. 12 | 13 | ### Docker Desktop 14 | 15 | - Install [Docker Desktop](https://docs.docker.com/desktop) 16 | - Ensure that `docker-compose` was installed - run `docker-compose --help` in your terminal to check 17 | - If using Windows, ensure that WSL2 integration is enabled ([WSL integration](https://docs.docker.com/desktop/windows/wsl/)) 18 | 19 | ### Fast Node Manager (fnm) 20 | 21 | - Install [FNM](https://github.com/Schniz/fnm) - this helps manage Node versions 22 | - Don't install anything else yet - we'll do that in a bit 23 | 24 | ### Source code 25 | 26 | - Clone the repo: `git clone https://github.com/sandboxnu/course-catalog-api` 27 | - Change into the repo directory: `cd ./course-catalog-api` 28 | - Switch Node versions: `fnm use` 29 | - There is a file called `.nvmrc` in the repository, which tells `fnm` which version to use 30 | 31 | ### Yarn 32 | 33 | - `yarn` is our package manager of choice - we use it to manage all of the dependencies we are using for this project. 34 | - Run `npm i -g yarn` 35 | - If you ever switch Node versions, you'll have to reinstall this (see this [issue](https://github.com/Schniz/fnm/issues/109)) 36 | 37 | ### Github notifications 38 | 39 | To make sure that our code gets reviewed promptly, we\'ll enable **Scheduled Reminders** for Github. 40 | 41 | You can follow the instructions [here on Github\'s documentation site.](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-your-membership-in-organizations/managing-your-scheduled-reminders) 42 | 43 | Whenever you get assigned as a reviewer for a teammate\'s code, you\'ll be notified on Slack. Don\'t worry if these terms aren\'t familiar yet! 44 | -------------------------------------------------------------------------------- /docs/getting-started/running.md: -------------------------------------------------------------------------------- 1 | Now for the fun part - actually running the `course-catalog-api`. 2 | 3 | Run these commands in order: 4 | 5 | 1. `cp template.env .env` 6 | - This will copy our templated `.env` file for your own use. Some of these environment variables are required for our codebase to work. **Make sure to fill them out!** 7 | 1. `yarn install` 8 | - This command installs all of our dependencies, and it does so locally. In other words, these dependencies are only visible to this project. 9 | 1. `yarn dev:docker` 10 | - This creates two Docker containers for us, which we'll use for running the backend. One is Postgres, a relational database, which we use for storing data. The other is Elasticsearch, which helps us return results for search query. 11 | 1. `yarn db:migrate` 12 | - This does all the setup for our newly-created database, creating all the tables that we need to store our data 13 | 1. `yarn db:refresh` 14 | - This generated a custom Prisma client for our project 15 | - **Prisma** is an [ORM](https://en.wikipedia.org/wiki/Object-relational_mapping) - it allows us to communicate with our database in a Javascript-based fashion, instead of executing raw SQL statements 16 | 17 | !> **Important:** If you are not on the Northeasern `NUWave` wifi (or don't want to wait for a scrape \[~30 minutes\]), please read the "Cache structure" page. 18 | 19 | 5. `yarn scrape` 20 | - Normally, this command would scrape Northeastern's course catalog for course data. 21 | - If you have installed the cache (see "Cache structure"), this command will just populate our database with the cached data. No network calls will be made. 22 | - If not, this will scrape Northeastern's live Banner API. 23 | 6. `yarn dev` 24 | - This command starts our GraphQL API. You should be able to access it at [`localhost:4000`](http://localhost:4000/) to see the GraphQL playground 25 | 26 | ?> Mac users may get a message along the lines of `Port 5000 already in use`. There's a process called `Control Center` running on that port; it's the AirPlay server. This can be disabled by turning off "AirPlay Receiver" in the "Sharing" System Preference. 27 | -------------------------------------------------------------------------------- /docs/getting-started/stored-cache.md: -------------------------------------------------------------------------------- 1 | ?> **Note:** If you're currently on the Northeastern `NUWave` wifi, you can skip this step during setup. However, we highly recommend reading this page! 2 | 3 | ## What's a cache? 4 | 5 | A cache is a locally-stored version of the data we would normally scrape from the Banner API. 6 | 7 | ## Why use a cache? 8 | 9 | This project uses a lot of data, and getting that data takes a long time. 10 | 11 | This is further complicated by restrictions that Northeastern imposes on its catalog: 12 | 13 | - Requests from outside the campus network (ie. from devices not connected to the `NUWave` network) are throttled 14 | - Responses are relatively slow (perhaps artificially slow) 15 | 16 | So, the cache saves us time in development, and also limits our interactions with Northeastern's servers (we don't want to overload them). 17 | 18 | ## Cache structure 19 | 20 | First, some background information on how our cache is structured. 21 | 22 | The basic cache structure looks like this: 23 | 24 | ``` 25 | cache 26 | | 27 | -- dev_data 28 | | 29 | -- v2 30 | -- requests 31 | ``` 32 | 33 | The `requests` directory contains cached request data. This is automatically handled by our `requests` library. 34 | 35 | The `dev_data` directory contains arbitrary data cached by developers. Data can be manually added to this directory throughout our code. 36 | 37 | Our caches use a versioning system. We are currently on `v2` of the cache structure, hence the `v2` sub-directory. 38 | 39 | ## Setup 40 | 41 | A cache can be obtained from [our archives](https://github.com/sandboxnu/course-catalog-api-cache). These contain (at the time of writing) all Northeastern semesters accessible on Banner, spanning from 2015 to 2023. 42 | 43 | You can `git clone` the repository, and then move one of the cache directories to the `course-catalog-api` directory (so, it'd look something like `course-catalog-api/20XX-XX-cache`. Rename that directory to `cache` (so, `course-catalog-api/cache`). 44 | 45 | Now, follow the steps on the **Running the backend** page to populate your database with this cached data. 46 | 47 | ### Using multiple caches 48 | 49 | If you'd like to use the cache of more than one academic year, you can "combine" them. Follow the instructions above for one cache, and ensure that the data is present in your database (ie. follow the steps on the **Running the backend**). 50 | 51 | Now, you can delete the `cache` directory, and repeat _both_ steps above. Each time you do so, the new cached data will be added to the existing data - you can "stack" different caches in this manner. 52 | 53 | ?> **Tip:** For development purposes, we don't recommend "stacking" many caches - it adds unecessary overhead for dev purposes. 54 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Document 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 | 15 | 16 | 17 | 18 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /docs/indexing/elasticsearch.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | 3 | In a nutshell - Elasticsearch (ES) is used to handle our searches. In their own words, Elasticsearch is "a distributed, free and open search and analytics engine for all types of data". 4 | 5 | Every search query goes through Elasticsearch, which does some weighting of results & returns course/employee information. 6 | 7 | ## Technical Information 8 | 9 | Elasticsearch (ES) has a concept of "indexes", which are similar to a "database" in a relational database. In short, it's a namespace for data. 10 | 11 | Our Elasticsearch setup has two indexes, `classes` and `employees`. 12 | 13 | ### Blue/Green Deployment 14 | 15 | Our ES indexes now use a blue/green deployment mechanism, which allows us to make changes without any downtime on the user side. 16 | 17 | We make use of **aliases** to make this work. An alias is essentially a mapping - `name -> index_name`, where `index_name` is any Elasticsearch index. The alias name is a constant - for example, the alias `classes` will always keep this name. However, the `index_name` referenced may change. 18 | 19 | Say you want to reset an ES index in order to catch new mapping changes for the documents, or in general to just refresh it. You may use the reset index action, but doing so drops all the data inside that ES index. Therefore, you might try reindexing instead. To reindex, we first "clone" the index by copying its data to another new index. This new index can have a different mapping but still catch the old data (there is a caveat here explained below). Thus, you now have two indexes, one with the updated mapping, and one with the old one. Both have data. Now, you simply switch the alias to point to this new one, then delete the old index. 20 | 21 | As an example: we have a `classes` alias, which at any given time may reference either the `classes_green` index or the `classes_blue` index. 22 | 23 | Any interactions with an alias that you might perform on an index will simply be forwarded to its reference. for example, if one were to execute a query on an alias `classes`, while it referenced `classes_green`, it would forward the query to `classes_green`, and return the result for the user. This extra layer of abstraction allows us to perform actions behind the scenes, while a service/user does not have to change behavior. The most important thing we do behind the scenes is reindexing. 24 | 25 | #### Caveats 26 | 27 | The caveat with blue green deployment and reindexing is that mapping differences can be drastic enough to cause issues with the reindexing. In general, small differences like a setting, data type change, etc can be interpeted. But big changes will require a script such that ES understands how to migrate the data from one index to another with a completely different mapping. We do not have functionality to support scripts yet. 28 | 29 | The second caveat is that ES is by nature non-transactional. This means we have no guarauntee that data transfers or insertions are successful. We currently have no mechanism for checking other than status codes on the requests. 30 | -------------------------------------------------------------------------------- /docs/infrastructure/infrastructure.md: -------------------------------------------------------------------------------- 1 | This page covers all of the infrastructure used to deploy the Course Catalog API. The Course catalog API uses a few 3rd party services. Most of these are critical to the operation of the API -- others are only for analytics. 2 | 3 | #### Notifications 4 | 5 | - Twilio 6 | 7 | #### Deployment 8 | 9 | - Terraform and AWS for deployment 10 | - Cloudflare for Domain Name System (DNS) management 11 | 12 | #### Analytics 13 | 14 | - Amplitude 15 | - FullStory - records video sessions 16 | - Google Analytics 17 | 18 | #### APM 19 | 20 | - Rollbar 21 | -------------------------------------------------------------------------------- /docs/infrastructure/jumphost.md: -------------------------------------------------------------------------------- 1 | ## What is a jumphost? 2 | 3 | A jump server, jump host or jump box is a system on a network used to access and manage devices in a separate security zone. For SearchNEU, the jumphost is an EC2 that can connect to the rest of the AWS infrastructure (database, elasticsearch, etc.) 4 | 5 | ## Accessing the jumphost 6 | 7 | 1. Generate an SSH key pair on your machine. 8 | 2. Give someone with prod access your public SSH key and have them follow [these instructions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) to add new SSH key to the Jumphost - see the section on `Add or replace a key pair for your instance`. Basically, add the new public SSH key to `.ssh/authorized_keys` on the jumphost. 9 | 3. On your machine, run `ssh -i @`. Someone on the team can probably tell you what `USER` and the Jumphost IP address are. 10 | 11 | ## Optional 12 | 13 | It can get annoying to have to keep typing out the user and remembering the Jumphost IP address. You can avoid this by setting up an SSH config file. The following instructions are for Mac and Linux users but there is probably an equivalent for Windows. 14 | 15 | 1. Create a file called `config` in `~/.ssh`. 16 | 2. Add these lines to `config` 17 | 18 | ``` 19 | Host 20 | HostName 21 | User 22 | IdentityFile 23 | ``` 24 | 25 | 3. Now you can ssh into the Jumphost by running `ssh `. 26 | -------------------------------------------------------------------------------- /docs/infrastructure/migration/staging-migration.md: -------------------------------------------------------------------------------- 1 | ## Changing the Staging URL 2 | 3 | This isn't something that needs to happen often, but we have had to change our staging URL before from `staging.api.searchneu.com` to `stagingapi.searchneu.com` because our SSL certificate only covered one wildcard subdomain. 4 | 5 | 1. In `staging.tf`, change `domains` to the desired name. 6 | 2. Apply and run Terraform with this change. At this point, the AWS load balancer should have a rule for the new domain, something like `IF Host is stagingapi.searchneu.com THEN forward to somewhere`. CloudFlare should also have a new CNAME record for this new domain. 7 | 3. However, if you visit the new domain, you'll probably get a 526 Invalid SSL Certificate error. This is because you need a new certificate in AWS ACM that covers this new domain. 8 | 4. To set up this new certificate, go to AWS ACM and request a new public certificate. Fill in the appropriate domain names and choose `DNS validation`. 9 | 5. The certificate status will say `Pending validation` until you add the given CNAME record(s) to CloudFlare with proxy status `DNS only`. 10 | 6. Finally, to put this certificate to use, go to the AWS load balancer -> Listeners -> select the listener with an SSL certificate -> Edit and change the default SSL certificate to the newly created one. 11 | -------------------------------------------------------------------------------- /docs/infrastructure/scraping.md: -------------------------------------------------------------------------------- 1 | Our scrapers in AWS are currently inactive, so we run our scrapes manually. This only needs to be done when classes for a new semester are released, or when a new environment is set up. 2 | 3 | 1. We prefer not running the actual scrape in production - instead, get a zip file of the cached information you want to insert 4 | 5 | - To get this information, you can either: 6 | - Run the scrapers in your local environment 7 | - Copy the information you need from the [Course Catalog cache](https://github.com/sandboxnu/course-catalog-api-cache) 8 | 9 | 2. Copy the zipped cache from your machine to the Jumphost. To do this, open up a terminal and run the following command: 10 | 11 | ```bash 12 | scp -i @: 13 | ``` 14 | 15 | - `JUMPHOST USER` is most likely `ubuntu` and you can determine `DIRECTORY` by running `pwd` inside the jumphost. 16 | 17 | 3. Unzip the cache inside `~/course-catalog-api` in the Jumphost, make sure the unzipped folder is named `cache`. 18 | 4. If the instance type of the jumphost is `t3 nano`, you'll need to stop the instance and change the instance type to something with more memory like `t3 large` in order for the scrapers to run successfully. If there's not enough memory, the scrapers will exit early with status code 137. 19 | 5. Run `fnm use` 20 | 6. Run `DATABASE_URL= elasticURL= yarn scrape` where the database URL and ElasticSearch URL are the secrets in the AWS Parameter Store. 21 | 22 | !> **Important:** Remember to change the instance type back to `t3.nano` when the scrape finishes to save credits! 23 | -------------------------------------------------------------------------------- /docs/notifications/twilioLandingPage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandboxnu/course-catalog-api/8bc26893537521e25e1ca4d8deb1c9edd946e761/docs/notifications/twilioLandingPage.png -------------------------------------------------------------------------------- /docs/notifications/twilioSetup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandboxnu/course-catalog-api/8bc26893537521e25e1ca4d8deb1c9edd946e761/docs/notifications/twilioSetup.png -------------------------------------------------------------------------------- /docs/notifications/twilioVerifySID.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandboxnu/course-catalog-api/8bc26893537521e25e1ca4d8deb1c9edd946e761/docs/notifications/twilioVerifySID.png -------------------------------------------------------------------------------- /docs/search-logo-red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sandboxnu/course-catalog-api/8bc26893537521e25e1ca4d8deb1c9edd946e761/docs/search-logo-red.png -------------------------------------------------------------------------------- /docs/updater/updating.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | 3 | The updater service can be thought of as a lightweight scrape. To compare the two: 4 | 5 | - **Scrapes** fetch all of the data we need for this API. It gets every aspect of the course and employee data, parses it, and saves it 6 | - **Updates** are used to update data, and requires some data to be present first. 7 | - Only frequently-changing data are updated by the updater. For example: 8 | - ✅ The number of seats available in a section (expected to change frequently) 9 | - ✅ The times/dates of a section's meetings 10 | - ❌ The name of the course (shouldn't change frequently) 11 | - ❌ The course's NUPath attributes 12 | 13 | The updater is also responsible for notification data. SearchNEU allows users to register for notifications if a section opens up a seat. Since the updater gathers seating information for sections, we use this to inform when to send out notifications. 14 | -------------------------------------------------------------------------------- /graphql/index.ts: -------------------------------------------------------------------------------- 1 | import { ApolloServer, gql } from "apollo-server"; 2 | import GraphQLJSON, { GraphQLJSONObject } from "graphql-type-json"; 3 | import macros from "../utils/macros"; 4 | 5 | import employeeTypeDef from "./typeDefs/employee"; 6 | 7 | import searchResolvers from "./resolvers/search"; 8 | import searchTypeDef from "./typeDefs/search"; 9 | 10 | import classResolvers from "./resolvers/class"; 11 | import classTypeDef from "./typeDefs/class"; 12 | import classOccurrenceTypeDef from "./typeDefs/classOccurrence"; 13 | 14 | import majorResolvers from "./resolvers/major"; 15 | import majorTypeDef from "./typeDefs/major"; 16 | import majorOccurrenceTypeDef from "./typeDefs/majorOccurrence"; 17 | 18 | import termInfoResolvers from "./resolvers/termInfo"; 19 | import termInfoTypeDef from "./typeDefs/termInfo"; 20 | 21 | if (macros.PROD || process.env.ENABLE_NOTIFS) { 22 | require("../twilio/server"); 23 | } 24 | 25 | // Enable JSON custom type 26 | const JSONResolvers = { 27 | JSON: GraphQLJSON, 28 | JSONObject: GraphQLJSONObject, 29 | }; 30 | 31 | // Base query so other typeDefs can do "extend type Query" 32 | const baseQuery = gql` 33 | scalar JSON 34 | scalar JSONObject 35 | 36 | type Query { 37 | _empty: String 38 | } 39 | 40 | type PageInfo { 41 | # When paginating forwards, are there more items 42 | hasNextPage: Boolean! 43 | } 44 | `; 45 | 46 | const server = new ApolloServer({ 47 | typeDefs: [ 48 | baseQuery, 49 | classTypeDef, 50 | classOccurrenceTypeDef, 51 | employeeTypeDef, 52 | majorTypeDef, 53 | majorOccurrenceTypeDef, 54 | searchTypeDef, 55 | termInfoTypeDef, 56 | ], 57 | resolvers: [ 58 | JSONResolvers, 59 | classResolvers, 60 | majorResolvers, 61 | searchResolvers, 62 | termInfoResolvers, 63 | ], 64 | debug: true, 65 | }); 66 | 67 | if (require.main === module) { 68 | server 69 | .listen() 70 | .then(({ url }) => { 71 | macros.log(`ready at ${url}`); 72 | return; 73 | }) 74 | .catch((err) => { 75 | macros.error(`error starting graphql server: ${JSON.stringify(err)}`); 76 | }); 77 | } 78 | 79 | export default server; 80 | -------------------------------------------------------------------------------- /graphql/resolvers/major.ts: -------------------------------------------------------------------------------- 1 | import { UserInputError } from "apollo-server"; 2 | import prisma from "../../services/prisma"; 3 | import { Major as PrismaMajor } from "@prisma/client"; 4 | 5 | const noResultsError = (recordType): never => { 6 | throw new UserInputError(`${recordType} not found!`); 7 | }; 8 | 9 | const getLatestMajorOccurrence = async ( 10 | majorId: string, 11 | ): Promise => { 12 | const majors: PrismaMajor[] = await prisma.major.findMany({ 13 | where: { majorId: majorId }, 14 | orderBy: { yearVersion: "desc" }, 15 | take: 1, 16 | }); 17 | 18 | return majors[0] || noResultsError("major"); 19 | }; 20 | 21 | const resolvers = { 22 | Query: { 23 | major: (parent, args) => { 24 | return getLatestMajorOccurrence(args.majorId); 25 | }, 26 | }, 27 | Major: { 28 | occurrence: async (major, args) => { 29 | const majors = await prisma.major.findMany({ 30 | where: { majorId: major.majorId, yearVersion: `${args.year}` }, 31 | take: 1, 32 | }); 33 | 34 | return majors[0] || noResultsError("occurrence"); 35 | }, 36 | latestOccurrence: (major) => { 37 | return getLatestMajorOccurrence(major.majorId); 38 | }, 39 | }, 40 | }; 41 | 42 | export default resolvers; 43 | -------------------------------------------------------------------------------- /graphql/resolvers/search.ts: -------------------------------------------------------------------------------- 1 | import { identity, pickBy } from "lodash"; 2 | import searcher from "../../services/searcher"; 3 | import { Course, Employee } from "../../types/types"; 4 | import { AggResults } from "../../types/searchTypes"; 5 | 6 | type SearchResultItem = Course | Employee; 7 | 8 | interface SearchResultItemConnection { 9 | totalCount: number; 10 | pageInfo: { 11 | hasNextPage: boolean; 12 | }; 13 | nodes: SearchResultItem[]; 14 | filterOptions: AggResults; 15 | } 16 | 17 | interface SearchArgs { 18 | termId: string; 19 | query?: string; 20 | subject?: string[]; 21 | nupath?: string[]; 22 | campus?: string[]; 23 | classType?: string[]; 24 | classIdRange?: { min: number; max: number }; 25 | honors?: boolean; 26 | // Pagination parameters 27 | offset?: number; 28 | first?: number; 29 | } 30 | const resolvers = { 31 | Query: { 32 | search: async ( 33 | parent, 34 | args: SearchArgs, 35 | ): Promise => { 36 | const { offset = 0, first = 10 } = args; 37 | const results = await searcher.search( 38 | args.query || "", 39 | args.termId, 40 | offset, 41 | offset + first, 42 | pickBy( 43 | { 44 | subject: args.subject, 45 | nupath: args.nupath, 46 | campus: args.campus, 47 | classType: args.classType, 48 | classIdRange: args.classIdRange, 49 | honors: args.honors, 50 | }, 51 | identity, 52 | ), 53 | ); 54 | 55 | const hasNextPage = offset + first < results.resultCount; 56 | 57 | return { 58 | totalCount: results.resultCount, 59 | nodes: results.searchContent.map((r) => 60 | r.type === "employee" 61 | ? r.employee 62 | : { ...r.class, sections: r.sections }, 63 | ), 64 | pageInfo: { 65 | hasNextPage, 66 | }, 67 | filterOptions: results.aggregations, 68 | }; 69 | }, 70 | }, 71 | 72 | SearchResultItem: { 73 | __resolveType(obj: SearchResultItem) { 74 | return "firstName" in obj ? "Employee" : "ClassOccurrence"; 75 | }, 76 | }, 77 | }; 78 | 79 | export default resolvers; 80 | -------------------------------------------------------------------------------- /graphql/resolvers/termInfo.ts: -------------------------------------------------------------------------------- 1 | import prisma from "../../services/prisma"; 2 | import { TermInfo } from "../../types/types"; 3 | 4 | type TermInfoCache = Record< 5 | string, 6 | { 7 | termInfos: TermInfo[]; 8 | lastUpdated: number; 9 | } 10 | >; 11 | 12 | /** 13 | * This mechanism caches the list of terms for each subcollege. 14 | * This list is only updated when new semesters are released (ie. 2-4 times a year) 15 | * 16 | * Every time a user first loads the SearchNEU page, they need to know which semesters are available. It makes no 17 | * sense to query Postgres every time - this isn't a scalable method, as Prisma (and psql) limit the number of connections. 18 | * Instead, we can cache the list of terms, only updating it every once in a while (and not on a user-by-user basis) 19 | */ 20 | const TERM_INFO_CACHE: TermInfoCache = {}; 21 | 22 | // How long should it take for the cache to be declared stale and re-fetched, in ms? (currently 2 hours) 23 | const CACHE_REFRESH_INTERVAL = 2 * 60 * 60 * 1000; 24 | 25 | // Checks if the cache is valid, or if it's time to revalidate and fetch from the database 26 | function isCacheValid(subCollege: string): boolean { 27 | if (subCollege in TERM_INFO_CACHE) { 28 | return ( 29 | TERM_INFO_CACHE[subCollege].lastUpdated + CACHE_REFRESH_INTERVAL > 30 | Date.now() 31 | ); 32 | } 33 | return false; 34 | } 35 | 36 | const getTermInfos = async (subCollege: string): Promise => { 37 | if (isCacheValid(subCollege)) { 38 | return TERM_INFO_CACHE[subCollege].termInfos; 39 | } else { 40 | // Cache is invalid (or doesn't exist yet), so we fetch from the database and cache it 41 | const termInfos = await prisma.termInfo.findMany({ 42 | where: { subCollege: subCollege }, 43 | orderBy: { termId: "desc" }, 44 | }); 45 | 46 | TERM_INFO_CACHE[subCollege] = { 47 | termInfos, 48 | lastUpdated: Date.now(), 49 | }; 50 | 51 | return termInfos; 52 | } 53 | }; 54 | 55 | const resolvers = { 56 | Query: { 57 | termInfos: async (parent, args) => { 58 | return getTermInfos(args.subCollege); 59 | }, 60 | }, 61 | }; 62 | 63 | export default resolvers; 64 | -------------------------------------------------------------------------------- /graphql/tests/__snapshots__/class.test.seq.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`class query can query in bulk 1`] = ` 4 | { 5 | "data": { 6 | "bulkClasses": [ 7 | { 8 | "classId": "3500", 9 | "latestOccurrence": { 10 | "termId": "201930", 11 | }, 12 | "name": "OOD", 13 | "subject": "CS", 14 | }, 15 | { 16 | "classId": "2500", 17 | "latestOccurrence": { 18 | "termId": "201930", 19 | }, 20 | "name": "Fundamentals of Computer Science 1", 21 | "subject": "CS", 22 | }, 23 | ], 24 | }, 25 | "errors": undefined, 26 | "extensions": undefined, 27 | "http": { 28 | "headers": Headers { 29 | Symbol(map): {}, 30 | }, 31 | }, 32 | } 33 | `; 34 | 35 | exports[`class query gets all occurrences 1`] = ` 36 | { 37 | "data": { 38 | "class": { 39 | "allOccurrences": [ 40 | { 41 | "termId": "201930", 42 | }, 43 | { 44 | "termId": "201830", 45 | }, 46 | ], 47 | "name": "Fundamentals of Computer Science 1", 48 | }, 49 | }, 50 | "errors": undefined, 51 | "extensions": undefined, 52 | "http": { 53 | "headers": Headers { 54 | Symbol(map): {}, 55 | }, 56 | }, 57 | } 58 | `; 59 | 60 | exports[`class query gets latest occurrence 1`] = ` 61 | { 62 | "data": { 63 | "class": { 64 | "latestOccurrence": { 65 | "termId": "201930", 66 | }, 67 | "name": "Fundamentals of Computer Science 1", 68 | }, 69 | }, 70 | "errors": undefined, 71 | "extensions": undefined, 72 | "http": { 73 | "headers": Headers { 74 | Symbol(map): {}, 75 | }, 76 | }, 77 | } 78 | `; 79 | 80 | exports[`class query gets specific occurrence 1`] = ` 81 | { 82 | "data": { 83 | "class": { 84 | "name": "Fundamentals of Computer Science 1", 85 | "occurrence": { 86 | "termId": "201930", 87 | }, 88 | }, 89 | }, 90 | "errors": undefined, 91 | "extensions": undefined, 92 | "http": { 93 | "headers": Headers { 94 | Symbol(map): {}, 95 | }, 96 | }, 97 | } 98 | `; 99 | 100 | exports[`class query gets the name of class from subject and classId 1`] = ` 101 | { 102 | "data": { 103 | "class": { 104 | "name": "Fundamentals of Computer Science 1", 105 | }, 106 | }, 107 | "errors": undefined, 108 | "extensions": undefined, 109 | "http": { 110 | "headers": Headers { 111 | Symbol(map): {}, 112 | }, 113 | }, 114 | } 115 | `; 116 | 117 | exports[`classByHash query gets class from class hash 1`] = ` 118 | { 119 | "data": { 120 | "classByHash": { 121 | "classId": "2500", 122 | "name": "Fundamentals of Computer Science 1", 123 | "subject": "CS", 124 | "termId": "201830", 125 | }, 126 | }, 127 | "errors": undefined, 128 | "extensions": undefined, 129 | "http": { 130 | "headers": Headers { 131 | Symbol(map): {}, 132 | }, 133 | }, 134 | } 135 | `; 136 | 137 | exports[`sectionByHash query gets section from section id 1`] = ` 138 | { 139 | "data": { 140 | "sectionByHash": { 141 | "campus": "Boston", 142 | "classId": "2500", 143 | "classType": "Lecture", 144 | "crn": "12345", 145 | "honors": false, 146 | "meetings": {}, 147 | "seatsCapacity": 5, 148 | "seatsRemaining": 2, 149 | "subject": "CS", 150 | "termId": "201830", 151 | }, 152 | }, 153 | "errors": undefined, 154 | "extensions": undefined, 155 | "http": { 156 | "headers": Headers { 157 | Symbol(map): {}, 158 | }, 159 | }, 160 | } 161 | `; 162 | -------------------------------------------------------------------------------- /graphql/tests/__snapshots__/major.test.seq.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`cannot find major from non-present majorId 1`] = ` 4 | { 5 | "data": { 6 | "major": null, 7 | }, 8 | "errors": [ 9 | [GraphQLError: major not found!], 10 | ], 11 | "extensions": undefined, 12 | "http": { 13 | "headers": Headers { 14 | Symbol(map): {}, 15 | }, 16 | }, 17 | } 18 | `; 19 | 20 | exports[`cannot find majorOccurrence from non-present year 1`] = ` 21 | { 22 | "data": { 23 | "major": { 24 | "majorId": "computer-information-science/computer-science/bscs", 25 | "occurrence": null, 26 | }, 27 | }, 28 | "errors": [ 29 | [GraphQLError: occurrence not found!], 30 | ], 31 | "extensions": undefined, 32 | "http": { 33 | "headers": Headers { 34 | Symbol(map): {}, 35 | }, 36 | }, 37 | } 38 | `; 39 | 40 | exports[`gets latest occurrence 1`] = ` 41 | { 42 | "data": { 43 | "major": { 44 | "latestOccurrence": { 45 | "spec": { 46 | "name": "Computer Science, BSCS", 47 | "yearVersion": 2018, 48 | }, 49 | "yearVersion": "2018", 50 | }, 51 | "majorId": "computer-information-science/computer-science/bscs", 52 | }, 53 | }, 54 | "errors": undefined, 55 | "extensions": undefined, 56 | "http": { 57 | "headers": Headers { 58 | Symbol(map): {}, 59 | }, 60 | }, 61 | } 62 | `; 63 | 64 | exports[`gets major from majorId 1`] = ` 65 | { 66 | "data": { 67 | "major": { 68 | "majorId": "computer-information-science/computer-science/bscs", 69 | }, 70 | }, 71 | "errors": undefined, 72 | "extensions": undefined, 73 | "http": { 74 | "headers": Headers { 75 | Symbol(map): {}, 76 | }, 77 | }, 78 | } 79 | `; 80 | 81 | exports[`gets specific occurrence 1`] = ` 82 | { 83 | "data": { 84 | "major": { 85 | "majorId": "computer-information-science/computer-science/bscs", 86 | "occurrence": { 87 | "plansOfStudy": [ 88 | { 89 | "id": "0", 90 | "years": [ 91 | 1000, 92 | ], 93 | }, 94 | ], 95 | "spec": { 96 | "name": "Computer Science, BSCS", 97 | "yearVersion": 2017, 98 | }, 99 | "yearVersion": "2017", 100 | }, 101 | }, 102 | }, 103 | "errors": undefined, 104 | "extensions": undefined, 105 | "http": { 106 | "headers": Headers { 107 | Symbol(map): {}, 108 | }, 109 | }, 110 | } 111 | `; 112 | -------------------------------------------------------------------------------- /graphql/tests/__snapshots__/search.test.seq.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`search resolver searches for blank in term gets results 1`] = ` 4 | { 5 | "data": { 6 | "search": { 7 | "filterOptions": { 8 | "campus": null, 9 | "classType": [ 10 | { 11 | "count": 10, 12 | "value": "Lecture", 13 | }, 14 | ], 15 | "nupath": [ 16 | { 17 | "count": 10, 18 | "value": "Writing Intensive", 19 | }, 20 | ], 21 | "subject": null, 22 | }, 23 | "nodes": [ 24 | { 25 | "__typename": "ClassOccurrence", 26 | "classAttributes": [], 27 | "desc": "a class", 28 | "lastUpdateTime": 20, 29 | "name": "Fundamentals of Computer Science 1", 30 | "optPrereqsFor": null, 31 | "prereqsFor": null, 32 | "sections": [ 33 | { 34 | "crn": "1234", 35 | "profs": [], 36 | }, 37 | ], 38 | "url": "url", 39 | }, 40 | ], 41 | "pageInfo": { 42 | "hasNextPage": false, 43 | }, 44 | "totalCount": 10, 45 | }, 46 | }, 47 | "errors": undefined, 48 | "extensions": undefined, 49 | "http": { 50 | "headers": Headers { 51 | Symbol(map): {}, 52 | }, 53 | }, 54 | } 55 | `; 56 | -------------------------------------------------------------------------------- /graphql/tests/major.test.seq.ts: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | import prisma from "../../services/prisma"; 3 | import server from "../index"; 4 | import { DocumentNode } from "graphql"; 5 | import { GraphQLResponse } from "apollo-server-core"; 6 | 7 | const query = async (queryBody: { 8 | query: string | DocumentNode; 9 | }): Promise => { 10 | return server.executeOperation(queryBody); 11 | }; 12 | 13 | beforeAll(async () => { 14 | await prisma.major.deleteMany({}); 15 | 16 | await prisma.major.create({ 17 | data: { 18 | majorId: "computer-information-science/computer-science/bscs", 19 | yearVersion: "2018", 20 | spec: { name: "Computer Science, BSCS", yearVersion: 2018 }, 21 | plansOfStudy: [{ years: [1000], id: "0" }], 22 | }, 23 | }); 24 | 25 | await prisma.major.create({ 26 | data: { 27 | majorId: "computer-information-science/computer-science/bscs", 28 | yearVersion: "2017", 29 | spec: { name: "Computer Science, BSCS", yearVersion: 2017 }, 30 | plansOfStudy: [{ years: [1000], id: "0" }], 31 | }, 32 | }); 33 | 34 | await prisma.major.create({ 35 | data: { 36 | majorId: "science/biochemistry/biochemistry-bs", 37 | yearVersion: "2018", 38 | spec: { name: "Biochemistry, BS", yearVersion: 2018 }, 39 | plansOfStudy: [{ years: [1000], id: "0" }], 40 | }, 41 | }); 42 | }); 43 | 44 | it("gets major from majorId", async () => { 45 | const res = await query({ 46 | query: gql` 47 | query major { 48 | major(majorId: "computer-information-science/computer-science/bscs") { 49 | majorId 50 | } 51 | } 52 | `, 53 | }); 54 | expect(res).toMatchSnapshot(); 55 | }); 56 | 57 | it("gets specific occurrence", async () => { 58 | const res = await query({ 59 | query: gql` 60 | query major { 61 | major(majorId: "computer-information-science/computer-science/bscs") { 62 | majorId 63 | occurrence(year: 2017) { 64 | yearVersion 65 | spec 66 | plansOfStudy 67 | } 68 | } 69 | } 70 | `, 71 | }); 72 | 73 | expect(res).toMatchSnapshot(); 74 | }); 75 | 76 | it("gets latest occurrence", async () => { 77 | const res = await query({ 78 | query: gql` 79 | query major { 80 | major(majorId: "computer-information-science/computer-science/bscs") { 81 | majorId 82 | latestOccurrence { 83 | yearVersion 84 | spec 85 | } 86 | } 87 | } 88 | `, 89 | }); 90 | expect(res).toMatchSnapshot(); 91 | }); 92 | 93 | it("cannot find major from non-present majorId", async () => { 94 | const res = await query({ 95 | query: gql` 96 | query major { 97 | major(majorId: "humanities/lovecraftian-studies/lovecraft-studies-ba") { 98 | majorId 99 | latestOccurrence { 100 | yearVersion 101 | } 102 | } 103 | } 104 | `, 105 | }); 106 | expect(res).toMatchSnapshot(); 107 | }); 108 | 109 | it("cannot find majorOccurrence from non-present year", async () => { 110 | const res = await query({ 111 | query: gql` 112 | query major { 113 | major(majorId: "computer-information-science/computer-science/bscs") { 114 | majorId 115 | occurrence(year: 1984) { 116 | spec 117 | } 118 | } 119 | } 120 | `, 121 | }); 122 | expect(res).toMatchSnapshot(); 123 | }); 124 | -------------------------------------------------------------------------------- /graphql/typeDefs/class.js: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | 3 | const typeDef = gql` 4 | extend type Query { 5 | class(subject: String!, classId: String!): Class 6 | bulkClasses(input: [BulkClassInput!]!): [Class!] 7 | classByHash(hash: String!): ClassOccurrence 8 | sectionByHash(hash: String!): Section 9 | } 10 | 11 | input BulkClassInput { 12 | subject: String! 13 | classId: String! 14 | } 15 | 16 | type Class { 17 | name: String! 18 | subject: String! 19 | classId: String! 20 | 21 | occurrence(termId: String!): ClassOccurrence 22 | latestOccurrence: ClassOccurrence 23 | allOccurrences: [ClassOccurrence]! 24 | } 25 | `; 26 | 27 | export default typeDef; 28 | -------------------------------------------------------------------------------- /graphql/typeDefs/classOccurrence.js: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | 3 | const typeDef = gql` 4 | type ClassOccurrence { 5 | name: String! 6 | subject: String! 7 | classId: String! 8 | termId: String! 9 | 10 | desc: String! 11 | prereqs: JSON 12 | coreqs: JSON 13 | prereqsFor: JSON 14 | optPrereqsFor: JSON 15 | maxCredits: Int 16 | minCredits: Int 17 | classAttributes: [String!]! 18 | url: String! 19 | prettyUrl: String 20 | lastUpdateTime: Float 21 | nupath: [String!]! 22 | sections: [Section!]! 23 | host: String! 24 | feeAmount: Int 25 | feeDescription: String 26 | } 27 | 28 | type Section { 29 | termId: String! 30 | subject: String! 31 | classId: String! 32 | classType: String! 33 | crn: String! 34 | seatsCapacity: Int! 35 | seatsRemaining: Int! 36 | waitCapacity: Int! 37 | waitRemaining: Int! 38 | campus: String! 39 | honors: Boolean! 40 | url: String! 41 | profs: [String!]! 42 | meetings: JSON 43 | host: String! 44 | lastUpdateTime: Float 45 | } 46 | `; 47 | 48 | export default typeDef; 49 | -------------------------------------------------------------------------------- /graphql/typeDefs/employee.js: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | 3 | const typeDef = gql` 4 | type Employee { 5 | name: String! 6 | firstName: String! 7 | lastName: String! 8 | email: String 9 | primaryDepartment: String 10 | primaryRole: String 11 | phone: String 12 | officeRoom: String 13 | } 14 | `; 15 | 16 | export default typeDef; 17 | -------------------------------------------------------------------------------- /graphql/typeDefs/major.js: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | 3 | const typeDef = gql` 4 | extend type Query { 5 | major(majorId: String!): Major 6 | } 7 | 8 | type Major { 9 | majorId: String! 10 | yearVersion: String! 11 | 12 | occurrence(year: Int!): MajorOccurrence 13 | latestOccurrence: MajorOccurrence 14 | } 15 | `; 16 | 17 | export default typeDef; 18 | -------------------------------------------------------------------------------- /graphql/typeDefs/majorOccurrence.js: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | 3 | const typeDef = gql` 4 | type MajorOccurrence { 5 | majorId: String! 6 | yearVersion: String! 7 | 8 | spec: JSON! 9 | plansOfStudy: JSON! 10 | } 11 | `; 12 | 13 | export default typeDef; 14 | -------------------------------------------------------------------------------- /graphql/typeDefs/search.js: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | 3 | const typeDef = gql` 4 | extend type Query { 5 | search( 6 | termId: String! 7 | query: String 8 | subject: [String!] 9 | nupath: [String!] 10 | campus: [String!] 11 | classType: [String!] 12 | classIdRange: IntRange 13 | honors: Boolean 14 | 15 | """ 16 | " 17 | Get elements after the given offset 18 | """ 19 | offset: Int 20 | """ 21 | Get n elements 22 | """ 23 | first: Int 24 | ): SearchResultItemConnection 25 | } 26 | 27 | input IntRange { 28 | min: Int! 29 | max: Int! 30 | } 31 | 32 | type SearchResultItemConnection { 33 | totalCount: Int! 34 | pageInfo: PageInfo! 35 | nodes: [SearchResultItem] 36 | filterOptions: FilterOptions! 37 | } 38 | 39 | type FilterOptions { 40 | nupath: [FilterAgg!] 41 | subject: [FilterAgg!] 42 | classType: [FilterAgg!] 43 | campus: [FilterAgg!] 44 | honors: [FilterAgg!] 45 | } 46 | 47 | type FilterAgg { 48 | value: String! 49 | count: Int! 50 | description: String 51 | } 52 | 53 | union SearchResultItem = ClassOccurrence | Employee 54 | `; 55 | 56 | export default typeDef; 57 | -------------------------------------------------------------------------------- /graphql/typeDefs/termInfo.js: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | 3 | const typeDef = gql` 4 | extend type Query { 5 | termInfos(subCollege: String!): [TermInfo!]! 6 | } 7 | 8 | type TermInfo { 9 | termId: String! 10 | subCollege: String! 11 | text: String! 12 | } 13 | `; 14 | 15 | export default typeDef; 16 | -------------------------------------------------------------------------------- /infrastructure/aws/push-image: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ACCOUNTID=$(aws sts get-caller-identity --query Account --output text) 3 | TAG=$(git log -1 --pretty=%h) 4 | REPO=$ACCOUNTID.dkr.ecr.us-east-1.amazonaws.com/course-catalog-api 5 | IMG=$REPO:$TAG 6 | 7 | aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ACCOUNTID.dkr.ecr.us-east-1.amazonaws.com 8 | docker build -t $IMG . 9 | docker tag $IMG $REPO:staging 10 | docker push $IMG 11 | docker push $REPO:staging 12 | -------------------------------------------------------------------------------- /infrastructure/aws/redeploy: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ ! " prod staging " =~ " $1 " ]]; then 4 | echo "Please provide environment to use: prod or staging" 5 | exit 1 6 | fi 7 | CLUSTER="$1-course-catalog-api" 8 | SERVICES=( "$CLUSTER-webserver" "$CLUSTER-update" ) 9 | 10 | # Disable aws from sending stdout to less 11 | export AWS_PAGER="" 12 | 13 | echo "Redeploying cluster $CLUSTER with last pushed image" 14 | 15 | if [[ "prod" = "$1" ]]; then 16 | # If not, add prod tag to staging image 17 | echo "Adding prod tag to staging image" 18 | MANIFEST=$(aws ecr batch-get-image --repository-name course-catalog-api --image-ids imageTag=staging --query 'images[].imageManifest' --output text) 19 | aws ecr put-image --repository-name course-catalog-api --image-tag prod --image-manifest "$MANIFEST" 20 | fi 21 | 22 | echo "Forcing new deployment" 23 | for s in "${SERVICES[@]}" 24 | do 25 | aws ecs update-service --cluster $CLUSTER --service $s --force-new-deployment > /dev/null 26 | done 27 | echo "Check AWS Console for logs" 28 | -------------------------------------------------------------------------------- /infrastructure/aws/run-task: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; } 4 | 5 | if [[ ! " prod staging " =~ " $1 " ]]; then 6 | echo "Please provide environment to use: prod or staging" 7 | exit 1 8 | fi 9 | CLUSTER="$1-course-catalog-api" 10 | SERVICE_WEB=$CLUSTER-webserver 11 | TASK_WEB=$CLUSTER-webserver 12 | 13 | # Disable aws from sending stdout to less 14 | export AWS_PAGER="" 15 | 16 | echo "Running one-off task on $CLUSTER cluster with command: $2" 17 | 18 | # Get the network config from the web app service 19 | NETCONFIG=$(aws ecs describe-services --cluster $CLUSTER --services $SERVICE_WEB --output json | jq '.services[0].networkConfiguration' | jq '.awsvpcConfiguration.assignPublicIp = "DISABLED"') 20 | OVERRIDES=$(printf '{"containerOverrides":[{"name":"%s","command":["%s"]}]}' "$TASK" $(join_by '","' $2)) 21 | 22 | aws ecs run-task --overrides "$OVERRIDES" --started-by "one-off task CLI" --group "one-off" --launch-type "FARGATE" --network-configuration "$NETCONFIG" --task-definition $TASK_WEB --cluster $CLUSTER --output json 23 | -------------------------------------------------------------------------------- /infrastructure/aws/scrape: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | if [[ ! " prod staging " =~ " $1 " ]]; then 5 | echo "Please provide environment to use: prod or staging" 6 | exit 1 7 | fi 8 | CLUSTER="$1-course-catalog-api" 9 | SERVICE_WEB=$CLUSTER-webserver 10 | TASK=$CLUSTER-scrape 11 | 12 | # Disable aws from sending stdout to less 13 | export AWS_PAGER="" 14 | 15 | echo "Scraping on $CLUSTER cluster" 16 | 17 | # Get the network config from the web app service 18 | NETCONFIG=$(aws ecs describe-services --cluster $CLUSTER --services $SERVICE_WEB --output json | jq '.services[0].networkConfiguration' | jq '.awsvpcConfiguration.assignPublicIp = "ENABLED"') 19 | 20 | aws ecs run-task --started-by "CLI scrape" --launch-type "FARGATE" --network-configuration "$NETCONFIG" --task-definition $TASK --cluster $CLUSTER --output json 21 | -------------------------------------------------------------------------------- /infrastructure/dev/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | postgresql: 3 | image: postgres:16.3 4 | ports: 5 | - 5432:5432 6 | volumes: 7 | - ./docker-postgresql-multiple-databases:/docker-entrypoint-initdb.d 8 | - pg:/var/lib/postgresql/data 9 | environment: 10 | - POSTGRES_MULTIPLE_DATABASES=searchneu_dev,searchneu_test 11 | - POSTGRES_USER=postgres 12 | env_file: 13 | - ../../.env 14 | 15 | opensearch: 16 | image: opensearchproject/opensearch:2.19.0 17 | ports: 18 | - 9200:9200 19 | environment: 20 | - discovery.type=single-node 21 | - plugins.security.disabled=true 22 | - OPENSEARCH_INITIAL_ADMIN_PASSWORD=sUpp3rS3curePa55W0RD! 23 | volumes: 24 | pg: 25 | -------------------------------------------------------------------------------- /infrastructure/dev/docker-compose-server.yml: -------------------------------------------------------------------------------- 1 | services: 2 | postgresql: 3 | image: postgres:11.19-bullseye 4 | ports: 5 | - 5432:5432 6 | volumes: 7 | - ./docker-postgresql-multiple-databases:/docker-entrypoint-initdb.d 8 | - pg:/var/lib/postgresql/data 9 | environment: 10 | POSTGRES_MULTIPLE_DATABASES: searchneu_dev,searchneu_test 11 | POSTGRES_USER: postgres 12 | POSTGRES_PASSWORD: default_password 13 | es: 14 | image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2 15 | ports: 16 | - 9200:9200 17 | environment: 18 | - discovery.type=single-node 19 | web: 20 | build: ../../ 21 | depends_on: 22 | - es 23 | - postgresql 24 | ports: 25 | - 4000:4000 26 | - 8080:8080 27 | environment: 28 | DATABASE_URL: postgresql://postgres:default_password@postgresql:5432/searchneu_dev 29 | POSTGRES_PASSWORD: default_password 30 | elasticURL: http://es:9200 31 | TWILIO_PHONE_NUMBER: 32 | TWILIO_ACCOUNT_SID: 33 | TWILIO_AUTH_TOKEN: 34 | TWILIO_VERIFY_SERVICE_ID: 35 | CLIENT_ORIGIN: http://localhost:5000 36 | JWT_SECRET: 37 | SLACK_WEBHOOK_URL: 38 | 39 | volumes: 40 | pg: 41 | -------------------------------------------------------------------------------- /infrastructure/dev/docker-postgresql-multiple-databases/create-multiple-postgresql-databases.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # From https://github.com/mrts/docker-postgresql-multiple-databases 4 | 5 | set -e 6 | set -u 7 | 8 | function create_user_and_database() { 9 | local database=$1 10 | echo " Creating user and database '$database'" 11 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL 12 | CREATE USER $database; 13 | CREATE DATABASE $database; 14 | GRANT ALL PRIVILEGES ON DATABASE $database TO $database; 15 | EOSQL 16 | } 17 | 18 | if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then 19 | echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" 20 | for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do 21 | create_user_and_database $db 22 | done 23 | echo "Multiple databases created" 24 | fi -------------------------------------------------------------------------------- /infrastructure/prod/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Run a production prisma migration 4 | yarn prisma migrate deploy --preview-feature 5 | yarn db:refresh 6 | 7 | exec "$@" 8 | -------------------------------------------------------------------------------- /infrastructure/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform 2 | *.tfstate* 3 | *.tfvars -------------------------------------------------------------------------------- /infrastructure/terraform/acm.tf: -------------------------------------------------------------------------------- 1 | # Get HTTPS cert 2 | resource "aws_acm_certificate" "cert" { 3 | domain_name = "api.searchneu.com" 4 | subject_alternative_names = ["*.searchneu.com"] 5 | validation_method = "DNS" 6 | 7 | lifecycle { 8 | create_before_destroy = true 9 | } 10 | } 11 | 12 | resource "cloudflare_record" "cert" { 13 | for_each = { 14 | for dvo in aws_acm_certificate.cert.domain_validation_options : dvo.domain_name => { 15 | name = dvo.resource_record_name 16 | record = dvo.resource_record_value 17 | type = dvo.resource_record_type 18 | } 19 | } 20 | 21 | allow_overwrite = true 22 | name = each.value.name 23 | value = trimsuffix(each.value.record, ".") 24 | ttl = 1 25 | type = each.value.type 26 | zone_id = var.cloudflare_zone_id 27 | } 28 | -------------------------------------------------------------------------------- /infrastructure/terraform/alb.tf: -------------------------------------------------------------------------------- 1 | # ============= Load Balancer ================ 2 | module "alb" { 3 | source = "terraform-aws-modules/alb/aws" 4 | version = "~> 5.0" 5 | 6 | name = "course-catalog-api-alb" 7 | 8 | load_balancer_type = "application" 9 | 10 | vpc_id = aws_vpc.main.id 11 | subnets = aws_subnet.public.*.id 12 | security_groups = [aws_security_group.lb.id] 13 | 14 | http_tcp_listeners = [ 15 | { 16 | port = 80 17 | protocol = "HTTP" 18 | action_type = "redirect" 19 | redirect = { 20 | port = "443" 21 | protocol = "HTTPS" 22 | status_code = "HTTP_301" 23 | } 24 | } 25 | ] 26 | 27 | # Set default action to 404 28 | https_listeners = [ 29 | { 30 | port = 443 31 | protocol = "HTTPS" 32 | certificate_arn = aws_acm_certificate.cert.arn 33 | action_type = "fixed-response" 34 | fixed_response = { 35 | content_type = "text/html" 36 | status_code = "404" 37 | } 38 | } 39 | ] 40 | 41 | tags = { 42 | Description = "Load balance traffic to all envs" 43 | } 44 | } 45 | 46 | # ALB Security Group: Edit to restrict access to the application 47 | resource "aws_security_group" "lb" { 48 | name = "load-balancer-security-group" 49 | description = "controls access to the ALB" 50 | vpc_id = aws_vpc.main.id 51 | 52 | ingress { 53 | protocol = "tcp" 54 | from_port = 80 55 | to_port = 80 56 | cidr_blocks = ["0.0.0.0/0"] 57 | } 58 | 59 | ingress { 60 | protocol = "tcp" 61 | from_port = 443 62 | to_port = 443 63 | cidr_blocks = ["0.0.0.0/0"] 64 | } 65 | 66 | egress { 67 | protocol = "-1" 68 | from_port = 0 69 | to_port = 0 70 | cidr_blocks = ["0.0.0.0/0"] 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /infrastructure/terraform/ecr.tf: -------------------------------------------------------------------------------- 1 | resource "aws_ecr_repository" "app" { 2 | name = var.name 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure/terraform/ecs_long_arn.tf: -------------------------------------------------------------------------------- 1 | // Ensure we enroll ecs with long resource IDs 2 | // Delete after after date when all new accounts will be enrolled by default (unclear when) 3 | resource "null_resource" "enable_long_ecs_resource_ids" { 4 | provisioner "local-exec" { 5 | command = < Fundamentals of Computer Science", 82 | "OOD => Object Oriented Design", 83 | "PL => Programming Languages", 84 | "ML => Machine Learning", 85 | "AI => Artificial Intelligence", 86 | "NLP => Natural Language Processing", 87 | "orgo => Organic Chemistry", 88 | "swd => Software Development", 89 | "choir => Chorus", 90 | "syssec => Systems Security", 91 | "netsec => Network Security", 92 | "id => Interaction Design" 93 | ] 94 | } 95 | } 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /scrapers/classes/main.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | import "colors"; 6 | 7 | import cache from "../cache"; 8 | import macros from "../../utils/macros"; 9 | 10 | // Processors 11 | import markMissingRequisites from "./processors/markMissingRequisites"; 12 | import addPrerequisiteFor from "./processors/addPrerequisiteFor"; 13 | 14 | // Parsers 15 | import { instance as bannerv9Parser } from "./parsersxe/bannerv9Parser"; 16 | import { ParsedCourseSR, ParsedTermSR } from "../../types/scraperTypes"; 17 | 18 | // This is the main entry point for scraping classes 19 | // This file calls into the first Banner v8 parser, the processors, and hopefully soon, the v9 parsers too. 20 | // Call the main(['neu']) function below to scrape a college 21 | // This file also generates the search index and data dumps. 22 | 23 | class Main { 24 | static COLLEGE_ABRV = "neu"; 25 | 26 | /** 27 | * Standardize the data we get back from Banner, to ensure it matches our expectations. 28 | */ 29 | runProcessors(courses: ParsedCourseSR[]): void { 30 | courses.map((c) => (c.modifiedInProcessor = false)); 31 | // Run the processors, sequentially 32 | markMissingRequisites.go(courses); 33 | addPrerequisiteFor.go(courses); 34 | } 35 | 36 | /** 37 | * The main entrypoint for scraping courses. 38 | */ 39 | async main(termIds: string[]): Promise { 40 | if (macros.DEV && !process.env.CUSTOM_SCRAPE) { 41 | const cached = await cache.get( 42 | macros.DEV_DATA_DIR, 43 | "classes", 44 | Main.COLLEGE_ABRV, 45 | ); 46 | 47 | if (cached) { 48 | macros.log("Using cached class data - not rescraping"); 49 | return cached as ParsedTermSR; 50 | } 51 | } 52 | 53 | macros.log("Scraping classes...".blue.underline); 54 | const dump = await bannerv9Parser.main(termIds); 55 | macros.log("Done scraping classes\n\n".green.underline); 56 | 57 | this.runProcessors(dump.classes); 58 | 59 | // We don't overwrite cache on custom scrape - cache should always represent a full scrape 60 | if (macros.DEV && !process.env.CUSTOM_SCRAPE) { 61 | await cache.set(macros.DEV_DATA_DIR, "classes", Main.COLLEGE_ABRV, dump); 62 | macros.log("classes file saved!"); 63 | } 64 | 65 | return dump; 66 | } 67 | } 68 | 69 | const instance = new Main(); 70 | 71 | export default instance; 72 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/__mocks__/subjectAbbreviationParser.ts: -------------------------------------------------------------------------------- 1 | import data from "./subjectAbbreviationTable.json"; 2 | 3 | export function getSubjectAbbreviations(): Record { 4 | return data; 5 | } 6 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/sectionParser.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import Request from "../../request"; 7 | import util from "./util"; 8 | import MeetingParser from "./meetingParser"; 9 | import { SectionSR } from "../../../types/scraperTypes"; 10 | import { Section } from "../../../types/types"; 11 | 12 | const requestObj = new Request("sectionParser"); 13 | 14 | class SectionParser { 15 | async parseSectionsOfClass( 16 | termId: string, 17 | subject: string, 18 | classId: string, 19 | ): Promise { 20 | const cookieJar = await util.getCookiesForSearch(termId); 21 | const req = await requestObj.get( 22 | "https://nubanner.neu.edu/StudentRegistrationSsb/ssb/searchResults/searchResults", 23 | { 24 | searchParams: { 25 | txt_term: termId, 26 | txt_subject: subject, 27 | txt_courseNumber: classId, 28 | pageOffset: 0, 29 | pageMaxSize: 500, 30 | }, 31 | cookieJar: cookieJar, 32 | }, 33 | ); 34 | 35 | const bodyObj = JSON.parse(req.body); 36 | if (bodyObj.success) { 37 | return bodyObj.data.map((sr) => { 38 | return this.parseSectionFromSearchResult(sr); 39 | }); 40 | } 41 | return false; 42 | } 43 | 44 | /** 45 | * Search results already has all relevant section data 46 | * @param SR Section item from /searchResults 47 | */ 48 | parseSectionFromSearchResult(SR: SectionSR): Section { 49 | return { 50 | host: "neu.edu", 51 | termId: SR.term, 52 | subject: SR.subject, 53 | classId: SR.courseNumber, 54 | crn: SR.courseReferenceNumber, 55 | seatsCapacity: SR.maximumEnrollment, 56 | seatsRemaining: SR.seatsAvailable, 57 | waitCapacity: SR.waitCapacity, 58 | waitRemaining: SR.waitAvailable, 59 | lastUpdateTime: Date.now(), 60 | classType: SR.scheduleTypeDescription, 61 | campus: SR.campusDescription, 62 | honors: SR.sectionAttributes.some((a) => { 63 | return a.description === "Honors"; 64 | }), 65 | url: 66 | "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?" + 67 | `cat_term_in=${SR.term}&subj_code_in=${SR.subject}&crse_numb_in=${SR.courseNumber}`, 68 | profs: SR.faculty.map(MeetingParser.profName), 69 | meetings: MeetingParser.parseMeetings(SR.meetingsFaculty), 70 | }; 71 | } 72 | } 73 | 74 | export default new SectionParser(); 75 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/startup.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | /* eslint-disable */ 6 | // entrypoint for using the node binary itself to run bannerv9Parser.ts 7 | // so we can use --heap-prof 8 | // use NODE_ENV=prod because caching messes up everything 9 | 10 | const maxTerms = 200; 11 | 12 | require("@babel/register"); 13 | require("regenerator-runtime/runtime"); 14 | 15 | console.log(`[${new Date()}]\tstarting parsersxe/startup.js`); 16 | 17 | const Parser = require("./bannerv9Parser.ts").default; 18 | 19 | Parser.getAllTermInfos( 20 | `https://nubanner.neu.edu/StudentRegistrationSsb/ssb/classSearch/getTerms?offset=1&max=${maxTerms}&searchTerm=`, 21 | ) 22 | .then((terms) => 23 | Parser.main(terms) 24 | .then(saveFile) 25 | .catch((exception) => { 26 | return saveFile(exception.toString()); 27 | }), 28 | ) 29 | .finally(() => { 30 | return console.log(`[${new Date()}]\t~finally done~`); 31 | }); 32 | 33 | function saveFile(obj) { 34 | const fname = "cache/result.json"; 35 | require("fs").writeFileSync(fname, JSON.stringify(obj, null, 4)); 36 | console.log(`saved to file: ${fname}`); 37 | } 38 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/subjectAbbreviationParser.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-use-before-define */ 2 | /* 3 | * This file is part of Search NEU and licensed under AGPL3. 4 | * See the license file in the root folder for details. 5 | */ 6 | 7 | import _ from "lodash"; 8 | import he from "he"; 9 | import macros from "../../../utils/macros"; 10 | import Request from "../../request"; 11 | import { SubjectDescription } from "../../../types/scraperTypes"; 12 | 13 | const request = new Request("subjectAbberviationParser"); 14 | 15 | /** 16 | * Get the subject abberviations for use in parsing prereqs 17 | */ 18 | export const getSubjectAbbreviations = _.memoize(async (termId: string) => { 19 | macros.log( 20 | `SubjectAbbreviationParser: Not memoized. Scraping term ${termId}`, 21 | ); 22 | const subjectResponse = await requestSubjects(termId); 23 | return createDescriptionTable(subjectResponse); 24 | }); 25 | 26 | export const getSubjectDescriptions = _.memoize(async (termId: string) => { 27 | const subjectResponse = await requestSubjects(termId); 28 | return createAbbrTable(subjectResponse); 29 | }); 30 | 31 | async function requestSubjects(termId: string): Promise { 32 | const MAX = 500; // If there are more than 500 THIS WILL BREAK. Would make it smarter but not worth it rn. 33 | const URL = 34 | "https://nubanner.neu.edu/StudentRegistrationSsb/ssb/courseSearch/get_subject"; 35 | const subjectUrl = `${URL}?searchTerm=&term=${termId}&offset=1&max=${MAX}`; 36 | const response = await request.get(subjectUrl); 37 | 38 | if (response.statusCode !== 200) { 39 | macros.error(`Problem with request for subjects ${subjectUrl}`); 40 | } 41 | return JSON.parse(response.body); 42 | } 43 | 44 | function createDescriptionTable( 45 | subjects: SubjectDescription[], 46 | ): Record { 47 | const mappedSubjects = subjects.map((subject) => { 48 | return { 49 | subjectCode: subject.code, 50 | description: he.decode(subject.description), 51 | }; 52 | }); 53 | 54 | const mappedByDesc = _.keyBy(mappedSubjects, "description"); 55 | return _.mapValues(mappedByDesc, "subjectCode"); 56 | } 57 | 58 | function createAbbrTable( 59 | subjects: SubjectDescription[], 60 | ): Record { 61 | const mappedSubjects = subjects.map((subject) => { 62 | return { 63 | description: he.decode(subject.description) as string, 64 | subjectCode: subject.code, 65 | }; 66 | }); 67 | 68 | const mappedByCode = _.keyBy(mappedSubjects, "subjectCode"); 69 | return _.mapValues(mappedByCode, "description"); 70 | } 71 | 72 | // Export for testing https://philipwalton.com/articles/how-to-unit-test-private-functions-in-javascript/ 73 | export { 74 | createDescriptionTable as _createDescriptionTable, 75 | createAbbrTable as _createAbbrTable, 76 | requestSubjects as _requestSubjects, 77 | }; 78 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/termListParser.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import { TermInfo } from "../../../types/types"; 7 | 8 | class TermListParser { 9 | serializeTermsList( 10 | termsFromBanner: { code: string; description: string }[], 11 | ): TermInfo[] { 12 | const activeTermInfos = termsFromBanner.filter( 13 | (term) => !term.description.includes("View Only"), 14 | ); 15 | const activeTermIds = activeTermInfos.map((termInfo) => 16 | Number(termInfo.code), 17 | ); 18 | /* The smallest active termInfo code. 19 | All termInfo's with codes greater than or equal to this are considered active.*/ 20 | const minActiveTermInfoCode = Math.min(...activeTermIds); 21 | return termsFromBanner.map((term) => { 22 | const subCollege = this.determineSubCollegeName(term.description); 23 | 24 | /* This removes any instance of 'Law ', 'CPS ', and ' (View Only)' 25 | These strings are unnecessary (for LAW and CPS, the subCollege tells us all we need) */ 26 | const text = term.description.replace( 27 | /(Law\s|CPS\s)|\s\(View Only\)/gi, 28 | "", 29 | ); 30 | 31 | return { 32 | host: "neu.edu", 33 | termId: term.code, 34 | text: text, 35 | subCollege: subCollege, 36 | active: Number(term.code) >= minActiveTermInfoCode, 37 | }; 38 | }); 39 | } 40 | 41 | /** 42 | * "Spring 2019 Semester" -> "NEU" 43 | * "Spring 2019 Law Quarter" -> "LAW" 44 | * "Spring 2019 CPS Quarter" -> "CPS" 45 | * 46 | * @param termDesc 47 | * @returns {string} 48 | */ 49 | determineSubCollegeName(termDesc: string): string { 50 | if (termDesc.includes("CPS")) { 51 | return "CPS"; 52 | } else if (termDesc.includes("Law")) { 53 | return "LAW"; 54 | } else { 55 | return "NEU"; 56 | } 57 | } 58 | } 59 | 60 | export default new TermListParser(); 61 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/__snapshots__/classParser.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`classParser parseClassFromSearchResult parses and sends extra requests 1`] = ` 4 | { 5 | "classAttributes": [ 6 | "innovation", 7 | "bizz", 8 | ], 9 | "classId": "2311", 10 | "college": "NEU", 11 | "coreqs": { 12 | "type": "and", 13 | "values": [ 14 | { 15 | "classId": "1800", 16 | "subject": "CS", 17 | }, 18 | ], 19 | }, 20 | "desc": "class description 123", 21 | "feeAmount": null, 22 | "feeDescription": "", 23 | "host": "neu.edu", 24 | "lastUpdateTime": 1578252414987, 25 | "maxCredits": 4, 26 | "minCredits": 4, 27 | "name": "Organic Chemistry 1", 28 | "nupath": [], 29 | "prereqs": { 30 | "type": "and", 31 | "values": [ 32 | { 33 | "classId": "1800", 34 | "subject": "CS", 35 | }, 36 | ], 37 | }, 38 | "prettyUrl": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311", 39 | "subject": "CHEM", 40 | "termId": "202010", 41 | "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311", 42 | } 43 | `; 44 | 45 | exports[`classParser parseClassFromSearchResult parses and sends extra requests 2`] = ` 46 | { 47 | "classAttributes": [ 48 | "innovation", 49 | "bizz", 50 | ], 51 | "classId": "2500", 52 | "college": "NEU", 53 | "coreqs": { 54 | "type": "and", 55 | "values": [ 56 | { 57 | "classId": "1800", 58 | "subject": "CS", 59 | }, 60 | ], 61 | }, 62 | "desc": "class description 123", 63 | "feeAmount": null, 64 | "feeDescription": "", 65 | "host": "neu.edu", 66 | "lastUpdateTime": 1578252414987, 67 | "maxCredits": 4, 68 | "minCredits": 4, 69 | "name": "Fundamentals of Computer Science 1", 70 | "nupath": [], 71 | "prereqs": { 72 | "type": "and", 73 | "values": [ 74 | { 75 | "classId": "1800", 76 | "subject": "CS", 77 | }, 78 | ], 79 | }, 80 | "prettyUrl": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CS&crse_numb_in=2500", 81 | "subject": "CS", 82 | "termId": "202010", 83 | "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CS&crse_numb_in=2500", 84 | } 85 | `; 86 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/__snapshots__/meetingParser.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`meetingParser.ts 1`] = ` 4 | [ 5 | { 6 | "endDate": 18067, 7 | "startDate": 18022, 8 | "times": { 9 | "2": [ 10 | { 11 | "end": 48000, 12 | "start": 35400, 13 | }, 14 | ], 15 | "4": [ 16 | { 17 | "end": 48000, 18 | "start": 35400, 19 | }, 20 | ], 21 | "5": [ 22 | { 23 | "end": 48000, 24 | "start": 35400, 25 | }, 26 | ], 27 | }, 28 | "type": "Class", 29 | "where": "Forsyth Building 242", 30 | }, 31 | { 32 | "endDate": 18072, 33 | "startDate": 18022, 34 | "times": { 35 | "1": [ 36 | { 37 | "end": 52200, 38 | "start": 36000, 39 | }, 40 | ], 41 | "3": [ 42 | { 43 | "end": 52200, 44 | "start": 36000, 45 | }, 46 | ], 47 | }, 48 | "type": "Lab", 49 | "where": "Behrakis Health Sciences Cntr 420", 50 | }, 51 | { 52 | "endDate": 18072, 53 | "startDate": 18072, 54 | "times": { 55 | "2": [ 56 | { 57 | "end": 63000, 58 | "start": 55800, 59 | }, 60 | ], 61 | }, 62 | "type": "Final Exam", 63 | "where": "Cargill Hall 094", 64 | }, 65 | ] 66 | `; 67 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/__snapshots__/prereqParser.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`prereqParser should handle nested parenthesized prerequisites or "Graduate Admissions REQ" 1`] = ` 4 | { 5 | "type": "or", 6 | "values": [ 7 | { 8 | "type": "and", 9 | "values": [ 10 | { 11 | "classId": "1342", 12 | "subject": "MATH", 13 | }, 14 | { 15 | "type": "or", 16 | "values": [ 17 | { 18 | "classId": "1151", 19 | "subject": "PHYS", 20 | }, 21 | { 22 | "classId": "1161", 23 | "subject": "PHYS", 24 | }, 25 | ], 26 | }, 27 | ], 28 | }, 29 | { 30 | "type": "and", 31 | "values": [ 32 | { 33 | "classId": "1141", 34 | "subject": "PHYS", 35 | }, 36 | { 37 | "classId": "1241", 38 | "subject": "MATH", 39 | }, 40 | ], 41 | }, 42 | ], 43 | } 44 | `; 45 | 46 | exports[`prereqParser should handle nested parenthesized prerequisites or "Graduate Admissions REQ" 2`] = ` 47 | { 48 | "type": "or", 49 | "values": [ 50 | { 51 | "type": "and", 52 | "values": [ 53 | { 54 | "classId": "1119", 55 | "subject": "BIOL", 56 | }, 57 | { 58 | "classId": "1121", 59 | "subject": "BIOL", 60 | }, 61 | { 62 | "type": "or", 63 | "values": [ 64 | { 65 | "classId": "1101", 66 | "subject": "CHEM", 67 | }, 68 | { 69 | "classId": "1211", 70 | "subject": "CHEM", 71 | }, 72 | ], 73 | }, 74 | ], 75 | }, 76 | "Graduate Admission", 77 | ], 78 | } 79 | `; 80 | 81 | exports[`prereqParser should handle parenthesized prerequisites 2 1`] = ` 82 | { 83 | "type": "and", 84 | "values": [ 85 | { 86 | "type": "or", 87 | "values": [ 88 | { 89 | "classId": "3600", 90 | "subject": "CS", 91 | }, 92 | { 93 | "classId": "3650", 94 | "subject": "CS", 95 | }, 96 | { 97 | "classId": "5600", 98 | "subject": "CS", 99 | }, 100 | { 101 | "classId": "5600", 102 | "subject": "CS", 103 | }, 104 | ], 105 | }, 106 | { 107 | "type": "or", 108 | "values": [ 109 | { 110 | "classId": "4800", 111 | "subject": "CS", 112 | }, 113 | { 114 | "classId": "5800", 115 | "subject": "CS", 116 | }, 117 | { 118 | "classId": "5800", 119 | "subject": "CS", 120 | }, 121 | ], 122 | }, 123 | ], 124 | } 125 | `; 126 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/__snapshots__/sectionParser.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`sectionParser should match snapshot 1`] = ` 4 | { 5 | "campus": "Online", 6 | "classId": "2311", 7 | "classType": "Lecture", 8 | "crn": "10259", 9 | "honors": true, 10 | "host": "neu.edu", 11 | "lastUpdateTime": 1578252414987, 12 | "meetings": [ 13 | { 14 | "endDate": 18234, 15 | "startDate": 18143, 16 | "times": { 17 | "1": [ 18 | { 19 | "end": 52800, 20 | "start": 48900, 21 | }, 22 | ], 23 | "3": [ 24 | { 25 | "end": 52800, 26 | "start": 48900, 27 | }, 28 | ], 29 | "4": [ 30 | { 31 | "end": 52800, 32 | "start": 48900, 33 | }, 34 | ], 35 | }, 36 | "type": "Class", 37 | "where": "Shillman Hall 220", 38 | }, 39 | { 40 | "endDate": 18243, 41 | "startDate": 18236, 42 | "times": {}, 43 | "type": "Final Exam", 44 | "where": "TBA", 45 | }, 46 | ], 47 | "profs": [ 48 | "Oyindasola Oyelaran", 49 | ], 50 | "seatsCapacity": 76, 51 | "seatsRemaining": 36, 52 | "subject": "CHEM", 53 | "termId": "202010", 54 | "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311", 55 | "waitCapacity": 0, 56 | "waitRemaining": 0, 57 | } 58 | `; 59 | 60 | exports[`sectionParser should match snapshot 2`] = ` 61 | { 62 | "campus": "Boston", 63 | "classId": "2500", 64 | "classType": "Lecture", 65 | "crn": "30340", 66 | "honors": false, 67 | "host": "neu.edu", 68 | "lastUpdateTime": 1578252414987, 69 | "meetings": [ 70 | { 71 | "endDate": 18003, 72 | "startDate": 17903, 73 | "times": { 74 | "1": [ 75 | { 76 | "end": 37200, 77 | "start": 33300, 78 | }, 79 | ], 80 | "3": [ 81 | { 82 | "end": 37200, 83 | "start": 33300, 84 | }, 85 | ], 86 | "4": [ 87 | { 88 | "end": 37200, 89 | "start": 33300, 90 | }, 91 | ], 92 | }, 93 | "type": "Class", 94 | "where": "East Village 024", 95 | }, 96 | ], 97 | "profs": [ 98 | "Alan Mislove", 99 | ], 100 | "seatsCapacity": 75, 101 | "seatsRemaining": 4, 102 | "subject": "CS", 103 | "termId": "201930", 104 | "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=201930&subj_code_in=CS&crse_numb_in=2500", 105 | "waitCapacity": 0, 106 | "waitRemaining": 0, 107 | } 108 | `; 109 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/__snapshots__/termListParser.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`termListParser pulls out relevant data 1`] = ` 4 | [ 5 | { 6 | "active": true, 7 | "host": "neu.edu", 8 | "subCollege": "CPS", 9 | "termId": "202034", 10 | "text": "Spring 2020 Semester", 11 | }, 12 | { 13 | "active": true, 14 | "host": "neu.edu", 15 | "subCollege": "LAW", 16 | "termId": "202032", 17 | "text": "Spring 2020 Semester", 18 | }, 19 | { 20 | "active": true, 21 | "host": "neu.edu", 22 | "subCollege": "NEU", 23 | "termId": "202030", 24 | "text": "Spring 2020 Semester", 25 | }, 26 | ] 27 | `; 28 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/__snapshots__/util.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`parseTable ignores columns too wide and blank cells 1`] = ` 4 | [ 5 | { 6 | "days": undefined, 7 | "time": "11:00 am - 11:50 am", 8 | "type": "Class", 9 | }, 10 | { 11 | "days": "MW", 12 | "time": "", 13 | "type": "Class", 14 | }, 15 | ] 16 | `; 17 | 18 | exports[`parseTable pulls out right data 1`] = ` 19 | [ 20 | { 21 | "days": "MWF", 22 | "time": "11:00 am - 11:50 am", 23 | "type": "Class", 24 | }, 25 | { 26 | "days": "MW", 27 | "time": "12:00 pm - 1:50 pm", 28 | "type": "Class", 29 | }, 30 | ] 31 | `; 32 | 33 | exports[`parseTable uniquifies the head 1`] = ` 34 | [ 35 | { 36 | "days": undefined, 37 | "time": "Class", 38 | "time1": "11:00 am - 11:50 am", 39 | }, 40 | { 41 | "days": "MW", 42 | "time": "Class", 43 | "time1": "", 44 | }, 45 | ] 46 | `; 47 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/data/classParser.data.js: -------------------------------------------------------------------------------- 1 | export default { 2 | getCourseAttributes1: " Business Admin UBBA ", 3 | getCourseAttributes2: 4 | "\n \n NUpath Natural/Designed World NCND
NU Core Science/Tech Lvl 1 NCT1
Computer&Info Sci UBCS ", 5 | chem2311: { 6 | id: 17189, 7 | termEffective: "201110", 8 | courseNumber: "2311", 9 | subject: "CHEM", 10 | subjectCode: "CHEM", 11 | college: "College of Science", 12 | collegeCode: "SC", 13 | department: "Chemistry & Chemical Biology", 14 | departmentCode: "CHEM", 15 | courseTitle: "Organic Chemistry 1", 16 | durationUnit: null, 17 | numberOfUnits: null, 18 | attributes: null, 19 | gradeModes: null, 20 | ceu: null, 21 | courseScheduleTypes: null, 22 | courseLevels: null, 23 | creditHourHigh: null, 24 | creditHourLow: 4, 25 | creditHourIndicator: null, 26 | lectureHourLow: 4, 27 | lectureHourHigh: null, 28 | lectureHourIndicator: null, 29 | billHourLow: 4, 30 | billHourHigh: null, 31 | billHourIndicator: null, 32 | labHourLow: null, 33 | labHourHigh: null, 34 | labHourIndicator: null, 35 | otherHourLow: null, 36 | otherHourHigh: null, 37 | otherHourIndicator: null, 38 | description: null, 39 | subjectDescription: "Chemistry & Chemical Biology", 40 | courseDescription: 41 | "Introduces nomenclature, preparation, properties, stereochemistry, and reactions of common organic c", 42 | division: null, 43 | termStart: "200410", 44 | termEnd: "999999", 45 | preRequisiteCheckMethodCde: "B", 46 | anySections: null, 47 | }, 48 | cs2500: { 49 | id: 10962, 50 | termEffective: "200410", 51 | courseNumber: "2500", 52 | subject: "CS", 53 | subjectCode: "CS", 54 | college: "Khoury Coll of Comp Sciences", 55 | collegeCode: "CS", 56 | department: "Computer Science", 57 | departmentCode: "CS", 58 | courseTitle: "Fundamentals of Computer Science 1", 59 | durationUnit: null, 60 | numberOfUnits: null, 61 | attributes: null, 62 | gradeModes: null, 63 | ceu: null, 64 | courseScheduleTypes: null, 65 | courseLevels: null, 66 | creditHourHigh: null, 67 | creditHourLow: 4, 68 | creditHourIndicator: null, 69 | lectureHourLow: 4, 70 | lectureHourHigh: null, 71 | lectureHourIndicator: null, 72 | billHourLow: 4, 73 | billHourHigh: null, 74 | billHourIndicator: null, 75 | labHourLow: null, 76 | labHourHigh: null, 77 | labHourIndicator: null, 78 | otherHourLow: null, 79 | otherHourHigh: null, 80 | otherHourIndicator: null, 81 | description: null, 82 | subjectDescription: "Computer Science", 83 | courseDescription: 84 | "Introduces the fundamental ideas of computing and the principles of programming. Discusses a systema", 85 | division: null, 86 | termStart: "200410", 87 | termEnd: "999999", 88 | preRequisiteCheckMethodCde: "B", 89 | anySections: null, 90 | }, 91 | }; 92 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/data/util/1.html: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
3 | Scheduled Meeting Times 4 |
TypeTimeDays
Class11:00 am - 11:50 amMWF
Class12:00 pm - 1:50 pmMW
23 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/data/util/2.html: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
3 | Scheduled Meeting Times 4 |
TypeTimeDays
Class11:00 am - 11:50 am
ClassMWMWF
24 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/data/util/3.html: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
3 | Scheduled Meeting Times 4 |
TimeTimeDays
Class11:00 am - 11:50 am
ClassMWMWF
24 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/meetingParser.test.js: -------------------------------------------------------------------------------- 1 | import MeetingParser, { forTesting } from "../meetingParser"; 2 | import data from "./data/meetingParser.data"; 3 | 4 | const { hhmmToSeconds, mmddyyyyToDaysSinceEpoch } = forTesting; 5 | 6 | it("meetingParser.ts", () => { 7 | expect(MeetingParser.parseMeetings(data.htlh2200)).toMatchSnapshot(); 8 | }); 9 | 10 | it("profname", () => { 11 | expect(MeetingParser.profName({ displayName: "Chu, Daj" })).toEqual( 12 | "Daj Chu", 13 | ); 14 | expect(MeetingParser.profName("Lastname, Firstname")).toEqual( 15 | "Firstname Lastname", 16 | ); 17 | }); 18 | 19 | it("hhmmToSeconds", () => { 20 | expect(hhmmToSeconds("0000")).toEqual(0); 21 | expect(hhmmToSeconds("0010")).toEqual(600); 22 | expect(hhmmToSeconds("0100")).toEqual(3600); 23 | expect(hhmmToSeconds("0101")).toEqual(3660); 24 | }); 25 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/sectionParser.test.js: -------------------------------------------------------------------------------- 1 | import SectionParser from "../sectionParser"; 2 | import data from "./data/sectionParser.data"; 3 | 4 | beforeAll(() => { 5 | Date.now = jest.fn(() => 1578252414987); 6 | // jest.spyOn(Date, "now").mockReturnValue(1578252414987); 7 | expect(Date.now()).toEqual(1578252414987); 8 | }); 9 | 10 | describe("sectionParser", () => { 11 | const chem2311Section = SectionParser.parseSectionFromSearchResult( 12 | data.chem2311, 13 | ); 14 | const cs2500Section = SectionParser.parseSectionFromSearchResult(data.cs2500); 15 | it("should match snapshot", () => { 16 | // Snapshot test gives full coverage, but other tests also exist to clearly spec things out 17 | // DO NOT use the consts above - we need to create them inside the test case for the mock of Date.now() to work 18 | expect( 19 | SectionParser.parseSectionFromSearchResult(data.chem2311), 20 | ).toMatchSnapshot(); 21 | expect( 22 | SectionParser.parseSectionFromSearchResult(data.cs2500), 23 | ).toMatchSnapshot(); 24 | }); 25 | 26 | it("should detect Honors", () => { 27 | expect(chem2311Section.honors).toBeTruthy(); 28 | expect(cs2500Section.honors).toBeFalsy(); 29 | }); 30 | 31 | it("should detect campus", () => { 32 | expect(chem2311Section.campus).toBe("Online"); 33 | expect(cs2500Section.campus).toBe("Boston"); 34 | }); 35 | }); 36 | 37 | afterAll(() => { 38 | jest.restoreAllMocks(); 39 | }); 40 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/subjectAbbreviationParser.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-underscore-dangle */ 2 | import * as subject from "../subjectAbbreviationParser"; 3 | import nock from "nock"; 4 | 5 | const scope = nock("https://nubanner.neu.edu") 6 | .get(/term=termsTest/) 7 | .reply(200, [ 8 | { 9 | code: "ACCT", 10 | description: "Accounting", 11 | }, 12 | { 13 | code: "AVM", 14 | description: "Adv Manufacturing System - CPS", 15 | }, 16 | ]) 17 | .persist(); 18 | 19 | afterAll(() => { 20 | scope.persist(false); 21 | }); 22 | 23 | describe("subjectAbbreviationParser", () => { 24 | it("_createDescriptionTable builds mapping", () => { 25 | const banner = [ 26 | { 27 | code: "ACCT", 28 | description: "Accounting", 29 | }, 30 | { 31 | code: "AVM", 32 | description: "Adv Manufacturing System - CPS", 33 | }, 34 | ]; 35 | const map = { 36 | Accounting: "ACCT", 37 | "Adv Manufacturing System - CPS": "AVM", 38 | }; 39 | expect(subject._createDescriptionTable(banner)).toEqual(map); 40 | }); 41 | 42 | it("_createAbbrTable builds mapping", () => { 43 | const banner = [ 44 | { 45 | code: "ACCT", 46 | description: "Accounting", 47 | }, 48 | { 49 | code: "AVM", 50 | description: "Adv Manufacturing System - CPS", 51 | }, 52 | ]; 53 | const map = { 54 | ACCT: "Accounting", 55 | AVM: "Adv Manufacturing System - CPS", 56 | }; 57 | expect(subject._createAbbrTable(banner)).toEqual(map); 58 | }); 59 | 60 | it("requesting subjects", async () => { 61 | expect(await subject.getSubjectDescriptions("termsTest")).toEqual({ 62 | ACCT: "Accounting", 63 | AVM: "Adv Manufacturing System - CPS", 64 | }); 65 | 66 | expect(await subject.getSubjectAbbreviations("termsTest")).toEqual({ 67 | Accounting: "ACCT", 68 | "Adv Manufacturing System - CPS": "AVM", 69 | }); 70 | }); 71 | }); 72 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/termListParser.test.js: -------------------------------------------------------------------------------- 1 | import TermListParser from "../termListParser"; 2 | 3 | describe("termListParser", () => { 4 | it("pulls out relevant data", () => { 5 | const list = [ 6 | { 7 | code: "202034", 8 | description: "Spring 2020 CPS Semester", 9 | }, 10 | { 11 | code: "202032", 12 | description: "Spring 2020 Law Semester", 13 | }, 14 | { 15 | code: "202030", 16 | description: "Spring 2020 Semester", 17 | }, 18 | ]; 19 | expect(TermListParser.serializeTermsList(list)).toMatchSnapshot(); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/termParser.test.js: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | import TermParser from "../termParser"; 3 | 4 | describe("termParser", () => { 5 | // not worth testing parseTerm as it just maps other parsers over the results of requests 6 | // not worth testing requestsClassesForTerm and requestsSectionsForTerm as they just call concatpagination 7 | 8 | describe("concatPagination", () => { 9 | const mockReq = jest.fn(); 10 | // Mock implementation just returns slices of the range [0,4] 11 | mockReq.mockImplementation((offset, pageSize) => { 12 | // This func can potentially return false - helps w tests 13 | if (pageSize === 1 && offset === 1) { 14 | return false; 15 | } 16 | return { 17 | items: _.range(offset, Math.min(5, offset + pageSize)), 18 | totalCount: 5, 19 | }; 20 | }); 21 | afterEach(() => { 22 | mockReq.mockClear(); 23 | }); 24 | 25 | it("should throw an Error if data is missing", async () => { 26 | await expect( 27 | TermParser.concatPagination(async (_x, _y) => false, 200), 28 | ).rejects.toThrowError("Missing data"); 29 | }); 30 | 31 | it("should call the callback with appropriate args", async () => { 32 | await TermParser.concatPagination(mockReq, 10); 33 | expect(mockReq.mock.calls).toEqual([ 34 | [0, 1], 35 | [0, 10], 36 | ]); 37 | }); 38 | 39 | it("should default to asking for 500 items", async () => { 40 | const data = await TermParser.concatPagination(mockReq); 41 | expect(data).toEqual([0, 1, 2, 3, 4]); 42 | expect(mockReq.mock.calls).toEqual([ 43 | [0, 1], 44 | [0, 500], 45 | ]); 46 | }); 47 | 48 | it("should make multiple requests and stitch together", async () => { 49 | const data = await TermParser.concatPagination(mockReq, 2); 50 | expect(data).toEqual([0, 1, 2, 3, 4]); 51 | expect(mockReq.mock.calls).toEqual([ 52 | [0, 1], 53 | [0, 2], 54 | [2, 2], 55 | [4, 2], 56 | ]); 57 | }); 58 | 59 | it("should throw an error if some data chunks return false", async () => { 60 | // Any calls to the mock req of (1, 1) will return false 61 | // This should trigger an error down the line, since data is missing 62 | await expect( 63 | TermParser.concatPagination(mockReq, 1), 64 | ).rejects.toThrowError("Missing data"); 65 | }); 66 | }); 67 | }); 68 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/tests/util.test.js: -------------------------------------------------------------------------------- 1 | import fs from "fs-extra"; 2 | import path from "path"; 3 | import cheerio from "cheerio"; 4 | import util from "../util"; 5 | 6 | describe("parseTable", () => { 7 | it("pulls out right data", async () => { 8 | const body = await fs.readFile( 9 | path.join(__dirname, "data", "util", "1.html"), 10 | "utf8", 11 | ); 12 | 13 | const rawTable = cheerio.load(body)("table"); 14 | const parsedTable = util.parseTable(rawTable); 15 | expect(parsedTable).toMatchSnapshot(); 16 | }); 17 | 18 | it("ignores columns too wide and blank cells", async () => { 19 | const body = await fs.readFile( 20 | path.join(__dirname, "data", "util", "2.html"), 21 | "utf8", 22 | ); 23 | 24 | const rawTable = cheerio.load(body)("table"); 25 | const parsedTable = util.parseTable(rawTable); 26 | expect(parsedTable).toMatchSnapshot(); 27 | }); 28 | 29 | it("uniquifies the head", async () => { 30 | const body = await fs.readFile( 31 | path.join(__dirname, "data", "util", "3.html"), 32 | "utf8", 33 | ); 34 | 35 | const rawTable = cheerio.load(body)("table"); 36 | const parsedTable = util.parseTable(rawTable); 37 | expect(parsedTable).toMatchSnapshot(); 38 | }); 39 | }); 40 | 41 | it("uniquifies", () => { 42 | expect(util.uniquify(["1", "11"], "1")).toBe("12"); 43 | }); 44 | 45 | describe("parseTable", () => { 46 | it("table without name/table", () => { 47 | expect(util.parseTable([])).toEqual([]); 48 | expect( 49 | util.parseTable([ 50 | { 51 | name: "not table", 52 | }, 53 | ]), 54 | ).toEqual([]); 55 | }); 56 | 57 | it("table with no rows", () => { 58 | expect( 59 | util.parseTable([ 60 | { 61 | name: "table", 62 | rows: [], 63 | }, 64 | ]), 65 | ).toEqual([]); 66 | }); 67 | }); 68 | -------------------------------------------------------------------------------- /scrapers/classes/parsersxe/util.ts: -------------------------------------------------------------------------------- 1 | import $ from "cheerio"; 2 | import _ from "lodash"; 3 | import Request from "../../request"; 4 | import macros from "../../../utils/macros"; 5 | import { CookieJar } from "tough-cookie"; 6 | 7 | const requestObj = new Request("util"); 8 | 9 | function validCell(el: cheerio.Element): boolean { 10 | return el.type === "tag" && ["th", "td"].includes(el.name); 11 | } 12 | 13 | /** 14 | * Modify a string to avoid collisions with set 15 | * @param {[String]} set array to avoid collisions with 16 | * @param {String} value String to uniquify 17 | * appends a number to end of the string such that it doesn't collide 18 | */ 19 | function uniquify(set: string[], value: string): string { 20 | if (set.includes(value)) { 21 | let append = 1; 22 | while (set.includes(value + append)) { 23 | append++; 24 | } 25 | return value + append; 26 | } 27 | return value; 28 | } 29 | 30 | /** 31 | * Parse a table using it's head (or first row) as keys 32 | * @param {Cheerio} table Cheerio object of table 33 | * @returns A list of {key: value} where key comes from header 34 | */ 35 | function parseTable(table: cheerio.Cheerio): Record[] { 36 | // Empty table 37 | if (table.length !== 1) { 38 | return []; 39 | } 40 | // Non-table 41 | if (!("name" in table[0]) || table[0].name !== "table") { 42 | return []; 43 | } 44 | 45 | //includes both header rows and body rows 46 | const rows: cheerio.TagElement[] = $("tr", table).get(); 47 | if (rows.length === 0) { 48 | macros.error("zero rows???"); 49 | return []; 50 | } 51 | 52 | //the headers 53 | const heads: string[] = rows[0].children 54 | .filter(validCell) 55 | .reduce((acc: string[], element) => { 56 | const head: string = $(element) 57 | .text() 58 | .trim() 59 | .toLowerCase() 60 | .replace(/\s/gi, ""); 61 | const uniqueHead = uniquify(acc, head); 62 | acc.push(uniqueHead); 63 | return acc; 64 | }, []); 65 | 66 | // add the other rows 67 | const ret: Record[] = []; 68 | 69 | rows.slice(1).forEach((row: cheerio.TagElement) => { 70 | const values: string[] = row.children 71 | .filter(validCell) 72 | .map((el) => $(el).text()); 73 | if (values.length > heads.length) { 74 | // TODO look into which classes trigger this 75 | macros.warn( 76 | "Table row is longer than head, ignoring some content", 77 | heads, 78 | values, 79 | ); 80 | } 81 | 82 | ret.push(_.zipObject(heads, values) as Record); 83 | }); 84 | 85 | return ret; 86 | } 87 | 88 | async function getCookiesForSearch(termId: string): Promise { 89 | const cookieJar = new CookieJar(); 90 | // first, get the cookies 91 | // https://jennydaman.gitlab.io/nubanned/dark.html#studentregistrationssb-clickcontinue-post 92 | const clickContinue = await requestObj.post( 93 | "https://nubanner.neu.edu/StudentRegistrationSsb/ssb/term/search?mode=search", 94 | { 95 | form: { 96 | term: termId, 97 | }, 98 | cacheRequests: false, 99 | cookieJar: cookieJar, 100 | }, 101 | ); 102 | 103 | const bodyObj = JSON.parse(clickContinue.body); 104 | 105 | if (bodyObj.regAllowed === false) { 106 | macros.error( 107 | `failed to get cookies (from clickContinue) for the term ${termId}`, 108 | clickContinue.body, 109 | ); 110 | } 111 | 112 | for (const cookie of clickContinue.headers["set-cookie"]) { 113 | cookieJar.setCookie( 114 | cookie, 115 | "https://nubanner.neu.edu/StudentRegistrationSsb/", 116 | ); 117 | } 118 | return cookieJar; 119 | } 120 | 121 | export default { 122 | parseTable: parseTable, 123 | getCookiesForSearch: getCookiesForSearch, 124 | uniquify: uniquify, 125 | }; 126 | -------------------------------------------------------------------------------- /scrapers/classes/processors/markMissingRequisites.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import macros from "../../../utils/macros"; 7 | import keys from "../../../utils/keys"; 8 | import simplifyRequirements from "./simplifyPrereqs"; 9 | import { ParsedCourseSR } from "../../../types/scraperTypes"; 10 | import { isBooleanReq, isCourseReq, Requisite } from "../../../types/types"; 11 | 12 | // This file process the prereqs on each class and ensures that they point to other, valid classes. 13 | // If they point to a class that does not exist, they are marked as missing. 14 | 15 | export class MarkMissingRequisites { 16 | private classMap: Record = {}; 17 | 18 | updatePrereqs(prereqs: Requisite, host: string, termId: string): Requisite { 19 | if (!isBooleanReq(prereqs)) { 20 | return prereqs; 21 | } 22 | 23 | for (const prereqEntry of prereqs.values) { 24 | if (isCourseReq(prereqEntry)) { 25 | const hash = keys.getClassHash({ 26 | host: host, 27 | termId: termId, 28 | subject: prereqEntry.subject, 29 | classId: prereqEntry.classId, 30 | }); 31 | 32 | if (!this.classMap[hash]) { 33 | prereqEntry.missing = true; 34 | } 35 | } else if (isBooleanReq(prereqEntry)) { 36 | this.updatePrereqs(prereqEntry, host, termId); 37 | } else if (typeof prereqEntry !== "string") { 38 | macros.error("wtf is ", prereqEntry, prereqs); 39 | } 40 | } 41 | return prereqs; 42 | } 43 | 44 | /** 45 | * Marks missing requisites - are all of our prereq/coreqs real classes that we know about? 46 | * eg. Algo (CS3000) used to be CS1500, and NEU _still_ uses that code in a bunch of spots. Should be marked as missing 47 | */ 48 | go(classes: ParsedCourseSR[]): void { 49 | // Create a course mapping 50 | for (const aClass of classes) { 51 | const key = keys.getClassHash(aClass); 52 | this.classMap[key] = aClass; 53 | } 54 | 55 | // loop through classes to update, and get the new data from all the classes 56 | for (const aClass of classes) { 57 | if (aClass.prereqs) { 58 | const prereqs = this.updatePrereqs( 59 | aClass.prereqs, 60 | aClass.host, 61 | aClass.termId, 62 | ); 63 | 64 | // And simplify tree again 65 | aClass.prereqs = simplifyRequirements(prereqs); 66 | } 67 | 68 | if (aClass.coreqs) { 69 | const coreqs = this.updatePrereqs( 70 | aClass.coreqs, 71 | aClass.host, 72 | aClass.termId, 73 | ); 74 | 75 | aClass.coreqs = simplifyRequirements(coreqs); 76 | } 77 | 78 | if (aClass.coreqs || aClass.prereqs) { 79 | aClass.modifiedInProcessor = true; 80 | } 81 | } 82 | } 83 | } 84 | 85 | export default new MarkMissingRequisites(); 86 | -------------------------------------------------------------------------------- /scrapers/classes/processors/simplifyPrereqs.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import { 7 | Requisite, 8 | isCourseReq, 9 | BooleanReq, 10 | isBooleanReq, 11 | } from "../../../types/types"; 12 | 13 | //this is given the output of formatRequirements, where data.type and data.values exist 14 | // if there is an or embedded in another or, merge them (and and's too) 15 | // and if there is a subvalue of only 1 len, merge that too 16 | function simplifyRequirementsBase(data: Requisite): Requisite { 17 | if (typeof data === "string") { 18 | return data; 19 | } 20 | 21 | if (isCourseReq(data)) { 22 | return data; 23 | } 24 | 25 | // Must have .values and .type from here on 26 | const retVal = { 27 | type: data.type, 28 | values: [], 29 | }; 30 | 31 | // Simplify all children 32 | data.values.forEach((subData) => { 33 | subData = simplifyRequirementsBase(subData); 34 | 35 | if (isBooleanReq(subData)) { 36 | //if same type, merge 37 | if (subData.type === data.type) { 38 | retVal.values = retVal.values.concat(subData.values); 39 | return; 40 | 41 | // If only contains 1 value, merge 42 | } 43 | 44 | if (subData.values.length === 1) { 45 | retVal.values.push(subData.values[0]); 46 | return; 47 | } 48 | } 49 | 50 | //just add the subdata 51 | retVal.values.push(subData); 52 | }); 53 | 54 | // Simplify this node 55 | if (retVal.values.length === 1) { 56 | return retVal.values[0]; 57 | } 58 | 59 | return retVal; 60 | } 61 | 62 | export default function simplifyRequirements(data: Requisite): BooleanReq { 63 | data = simplifyRequirementsBase(data); 64 | if (!isBooleanReq(data)) { 65 | return { 66 | type: "and", 67 | values: [data], 68 | }; 69 | } 70 | 71 | return data; 72 | } 73 | -------------------------------------------------------------------------------- /scrapers/classes/processors/tests/simplifyPrereqs.test.ts: -------------------------------------------------------------------------------- 1 | import simplifyRequirements from "../simplifyPrereqs"; 2 | import { CourseReq } from "../../../../types/types"; 3 | 4 | function C(s: string): CourseReq { 5 | return { 6 | subject: "PHYS", 7 | classId: s, 8 | }; 9 | } 10 | 11 | it("simplifyRequirements shoudl work", () => { 12 | expect( 13 | simplifyRequirements({ 14 | type: "or", 15 | values: [ 16 | { 17 | type: "or", 18 | values: [ 19 | C("1"), 20 | { 21 | type: "or", 22 | values: [C("6")], 23 | }, 24 | ], 25 | }, 26 | { 27 | type: "or", 28 | values: [ 29 | C("1"), 30 | { 31 | type: "or", 32 | values: [ 33 | { 34 | type: "or", 35 | values: [ 36 | C("1"), 37 | { 38 | type: "or", 39 | values: [C("6")], 40 | }, 41 | ], 42 | }, 43 | { 44 | type: "or", 45 | values: [ 46 | C("1"), 47 | { 48 | type: "or", 49 | values: [C("6")], 50 | }, 51 | ], 52 | }, 53 | ], 54 | }, 55 | ], 56 | }, 57 | ], 58 | }), 59 | ).toEqual({ 60 | type: "or", 61 | values: ["1", "6", "1", "1", "6", "1", "6"].map(C), 62 | }); 63 | }); 64 | 65 | it("simplifyRequirements shoudl work2", () => { 66 | expect( 67 | simplifyRequirements({ 68 | type: "and", 69 | values: [ 70 | { 71 | type: "or", 72 | values: [ 73 | { 74 | subject: "PHYS", 75 | classId: "1148", 76 | }, 77 | { 78 | subject: "PHYS", 79 | classId: "1148", 80 | }, 81 | ], 82 | }, 83 | ], 84 | }), 85 | ).toEqual({ 86 | type: "or", 87 | values: [ 88 | { 89 | subject: "PHYS", 90 | classId: "1148", 91 | }, 92 | { 93 | subject: "PHYS", 94 | classId: "1148", 95 | }, 96 | ], 97 | }); 98 | }); 99 | 100 | it("redundant single-value boolean reqs are simplified", () => { 101 | expect( 102 | simplifyRequirements({ 103 | type: "and", 104 | values: [ 105 | { 106 | type: "or", 107 | values: [ 108 | { 109 | type: "and", 110 | values: ["Graduation Clearance"], 111 | }, 112 | { 113 | subject: "PHYS", 114 | classId: "1148", 115 | }, 116 | ], 117 | }, 118 | ], 119 | }), 120 | ).toEqual({ 121 | type: "or", 122 | values: [ 123 | "Graduation Clearance", 124 | { 125 | subject: "PHYS", 126 | classId: "1148", 127 | }, 128 | ], 129 | }); 130 | }); 131 | 132 | it("simplifyRequirements should put course req in a boolreq", () => { 133 | expect( 134 | simplifyRequirements({ 135 | subject: "PHYS", 136 | classId: "1148", 137 | }), 138 | ).toEqual({ 139 | type: "and", 140 | values: [ 141 | { 142 | subject: "PHYS", 143 | classId: "1148", 144 | }, 145 | ], 146 | }); 147 | }); 148 | -------------------------------------------------------------------------------- /scrapers/classes/processors/tests/testData.js: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import fs from "fs-extra"; 7 | import path from "path"; 8 | 9 | // This file is responsible for loading the termDump.json and giving different instances of the data to each test. 10 | // It needs to give different instances of the data to each test because the tests modify the data. 11 | // Can use as much RAM as needed. 12 | // It takes about 43 ms to JSON.parse the string. It takes lodash about 80 ms to _.cloneDeep the JS object. 13 | // A way to make this seem faster would be to pre-create a bunch of clones of the data and then store them in RAM, 14 | // and then return them when they are needed, and clone it more times for later. 15 | 16 | const filePromise = fs.readFile(path.join(__dirname, "data", "termDump.json")); 17 | 18 | exports.loadTermDump = async function loadTermDump() { 19 | const string = await filePromise; 20 | return JSON.parse(string); 21 | }; 22 | -------------------------------------------------------------------------------- /scrapers/classes/termDump.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import path from "path"; 7 | import fs from "fs-extra"; 8 | 9 | import macros from "../../utils/macros"; 10 | import keys from "../../utils/keys"; 11 | import { ParsedTermSR } from "../../types/scraperTypes"; 12 | 13 | // Creates the term dump of classes. 14 | 15 | class TermDump { 16 | async main(termDump: ParsedTermSR): Promise { 17 | const termMapDump: Record> = {}; 18 | macros.log("TERM DUMPING"); 19 | 20 | for (const aClass of termDump.classes) { 21 | const hash = keys.getClassHash(aClass); 22 | 23 | const termHash = keys.getTermHash({ 24 | host: aClass.host, 25 | termId: aClass.termId, 26 | }); 27 | 28 | if (!termMapDump[termHash]) { 29 | termMapDump[termHash] = { 30 | classMap: {}, 31 | sectionMap: {}, 32 | subjectMap: {}, 33 | termId: aClass.termId, 34 | host: aClass.host, 35 | }; 36 | } 37 | 38 | termMapDump[termHash]["classMap"][hash] = aClass; 39 | } 40 | 41 | for (const section of termDump.sections) { 42 | const hash = keys.getSectionHash(section); 43 | 44 | const termHash = keys.getTermHash({ 45 | host: section.host, 46 | termId: section.termId, 47 | }); 48 | 49 | if (!termMapDump[termHash]) { 50 | macros.warn("Found section with no class?", termHash, hash); 51 | termMapDump[termHash] = { 52 | classMap: {}, 53 | sectionMap: {}, 54 | subjectMap: {}, 55 | termId: section.termId, 56 | host: section.host, 57 | }; 58 | } 59 | 60 | termMapDump[termHash]["sectionMap"][hash] = section; 61 | } 62 | 63 | const promises = []; 64 | 65 | const values = Object.values(termMapDump); 66 | 67 | for (const value of values) { 68 | // Put them in a different file. 69 | if (!("host" in value && "termId" in value)) { 70 | macros.error("No host or Id?", value); 71 | continue; 72 | } 73 | 74 | const folderPath = path.join( 75 | macros.PUBLIC_DIR, 76 | "getTermDump", 77 | value["host"] as string, 78 | ); 79 | promises.push( 80 | fs.ensureDir(folderPath).then(() => { 81 | return fs.writeFile( 82 | path.join(folderPath, `${value["termId"]}.json`), 83 | JSON.stringify(value), 84 | ); 85 | }), 86 | ); 87 | } 88 | const outerFolderPath = path.join(macros.PUBLIC_DIR, "getTermDump"); 89 | promises.push( 90 | fs.ensureDir(outerFolderPath).then(() => { 91 | return fs.writeFile( 92 | path.join(outerFolderPath, "allTerms.json"), 93 | JSON.stringify(termDump), 94 | ); 95 | }), 96 | ); 97 | return Promise.all(promises); 98 | } 99 | } 100 | 101 | export default new TermDump(); 102 | -------------------------------------------------------------------------------- /scrapers/classes/tests/classScraper.test.ts: -------------------------------------------------------------------------------- 1 | import scraper from "../../main"; 2 | import prisma from "../../../services/prisma"; 3 | 4 | describe("getTermsIds", () => { 5 | beforeEach(() => { 6 | delete process.env.TERMS_TO_SCRAPE; 7 | delete process.env.NUMBER_OF_TERMS; 8 | }); 9 | 10 | it("returns the termsStr if and only if they're in the terms list", async () => { 11 | process.env.TERMS_TO_SCRAPE = "202210,202230,202250"; 12 | expect(await scraper.getTermIdsToScrape([])).toEqual([]); 13 | expect(await scraper.getTermIdsToScrape(["202210"])).toEqual(["202210"]); 14 | expect( 15 | await scraper.getTermIdsToScrape(["202210", "202230", "202250", "1234"]), 16 | ).toEqual(["202210", "202230", "202250"]); 17 | }); 18 | 19 | it("without a termStr, it takes NUMBER_OF_TERMS_TO_PARSE terms", async () => { 20 | process.env.NUMBER_OF_TERMS = "0"; 21 | const termIds = new Array(10).fill("a"); 22 | expect((await scraper.getTermIdsToScrape(termIds)).length).toBe(0); 23 | 24 | process.env.NUMBER_OF_TERMS = "5"; 25 | expect((await scraper.getTermIdsToScrape(termIds)).length).toBe(5); 26 | 27 | process.env.NUMBER_OF_TERMS = "20"; 28 | expect((await scraper.getTermIdsToScrape(termIds)).length).toBe(10); 29 | }); 30 | 31 | describe("defaults to only terms which don't already exist in the DB", () => { 32 | let termIds: string[]; 33 | beforeEach(() => { 34 | termIds = ["123", "456", "789", "000"]; 35 | }); 36 | 37 | it("returns all if there are no existing term IDs", async () => { 38 | // @ts-expect-error - the type isn't a PrismaPromise so TS will complain 39 | jest.spyOn(prisma.termInfo, "findMany").mockReturnValue([]); 40 | 41 | expect((await scraper.getTermIdsToScrape(termIds)).length).toBe(4); 42 | }); 43 | 44 | it("returns those newer than those that already exist in the DB", async () => { 45 | const termsToReturn = [{ termId: "123" }, { termId: "456" }]; 46 | // @ts-expect-error - the type isn't a PrismaPromise so TS will complain 47 | jest.spyOn(prisma.termInfo, "findMany").mockReturnValue(termsToReturn); 48 | 49 | const returnedTerms = await scraper.getTermIdsToScrape(termIds); 50 | expect(returnedTerms).toEqual(["789"]); 51 | }); 52 | 53 | it("returns an empty list if all terms already exist", async () => { 54 | const termsToReturn = termIds.map((t) => ({ termId: t })); 55 | // @ts-expect-error - the type isn't a PrismaPromise so TS will complain 56 | jest.spyOn(prisma.termInfo, "findMany").mockReturnValue(termsToReturn); 57 | 58 | const returnedTerms = await scraper.getTermIdsToScrape(termIds); 59 | expect(returnedTerms).toEqual([]); 60 | }); 61 | }); 62 | }); 63 | -------------------------------------------------------------------------------- /scrapers/employees/employeeMapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": { 3 | "properties": { 4 | "employee": { 5 | "properties": { 6 | "name": { 7 | "type": "text", 8 | "analyzer": "autocomplete", 9 | "search_analyzer": "standard" 10 | } 11 | } 12 | } 13 | } 14 | }, 15 | "settings": { 16 | "analysis": { 17 | "analyzer": { 18 | "autocomplete": { 19 | "tokenizer": "autocomplete", 20 | "filter": ["lowercase"] 21 | } 22 | }, 23 | "tokenizer": { 24 | "autocomplete": { 25 | "type": "edge_ngram", 26 | "min_gram": 2, 27 | "max_gram": 20, 28 | "token_chars": ["letter", "digit"] 29 | } 30 | } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /scrapers/employees/tests/__snapshots__/employees.test.ts.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`static data should be able to parse API responses 1`] = ` 4 | [ 5 | { 6 | "email": "m.abedi@northeastern.edu", 7 | "firstName": "Mehdi", 8 | "id": "m.abedi@northeastern.edu", 9 | "lastName": "Abedi", 10 | "name": "Mehdi Abedi", 11 | "officeRoom": "334 SN", 12 | "phone": "6173734427", 13 | "primaryDepartment": "Mechanical + Industrial Engineering", 14 | "primaryRole": "Associate Teaching Professor", 15 | }, 16 | { 17 | "email": "h.abeels@northeastern.edu", 18 | "firstName": "Hannah", 19 | "id": "h.abeels@northeastern.edu", 20 | "lastName": "Abeels", 21 | "name": "Hannah Abeels", 22 | "officeRoom": "KN 120 Knowles", 23 | "phone": "6173735149", 24 | "primaryDepartment": "School Of Law-Academic", 25 | "primaryRole": "Asst Dir - Online & Hybrid Pro", 26 | }, 27 | { 28 | "email": "s.abel@northeastern.edu", 29 | "firstName": "Sherry", 30 | "id": "s.abel@northeastern.edu", 31 | "lastName": "Abel", 32 | "name": "Sherry Abel", 33 | "officeRoom": "101 N. Tyron St", 34 | "phone": "9802248466", 35 | "primaryDepartment": "School Of Nursing", 36 | "primaryRole": "Part-time Lecturer", 37 | }, 38 | { 39 | "email": "p.abelli@northeastern.edu", 40 | "firstName": "Peter", 41 | "id": "p.abelli@northeastern.edu", 42 | "lastName": "Abelli", 43 | "name": "Peter Abelli", 44 | "officeRoom": "140 CN", 45 | "phone": "6173735599", 46 | "primaryDepartment": "Maintenance-HVAC-Dorms", 47 | "primaryRole": "HVAC Systems Technician", 48 | }, 49 | { 50 | "email": "m.abels@northeastern.edu", 51 | "firstName": "Margot", 52 | "id": "m.abels@northeastern.edu", 53 | "lastName": "Abels", 54 | "name": "Margot Abels", 55 | "officeRoom": "310 RP", 56 | "phone": null, 57 | "primaryDepartment": "Human Services", 58 | "primaryRole": "Assistant Teaching Professor", 59 | }, 60 | { 61 | "email": "e.ackerman@northeastern.edu", 62 | "firstName": "Elizabeth", 63 | "id": "e.ackerman@northeastern.edu", 64 | "lastName": "Ackerman", 65 | "name": "Elizabeth Ackerman", 66 | "officeRoom": "102 RB", 67 | "phone": null, 68 | "primaryDepartment": "BCHS Office of the Dean", 69 | "primaryRole": "Part-Time Lecturer", 70 | }, 71 | { 72 | "email": "m.acosta@northeastern.edu", 73 | "firstName": "Beth", 74 | "id": "m.acosta@northeastern.edu", 75 | "lastName": "Acosta", 76 | "name": "Beth Acosta", 77 | "officeRoom": "RB 103 RB", 78 | "phone": "6173735177", 79 | "primaryDepartment": "School Of Nursing", 80 | "primaryRole": "Part-time Lecturer", 81 | }, 82 | { 83 | "email": "m.adams@northeastern.edu", 84 | "firstName": "Marybeth", 85 | "id": "m.adams@northeastern.edu", 86 | "lastName": "Adams-Khoury", 87 | "name": "Marybeth Adams-Khoury", 88 | "officeRoom": "102 RB", 89 | "phone": null, 90 | "primaryDepartment": "BCHS Office of the Dean", 91 | "primaryRole": "Part-Time Lecturer", 92 | }, 93 | { 94 | "email": "j.adelberg@northeastern.edu", 95 | "firstName": "Jeff", 96 | "id": "j.adelberg@northeastern.edu", 97 | "lastName": "Adelberg", 98 | "name": "Jeff Adelberg", 99 | "officeRoom": "360 Huntington Ave", 100 | "phone": null, 101 | "primaryDepartment": "Theater", 102 | "primaryRole": "Part-Time Lecturer", 103 | }, 104 | ] 105 | `; 106 | -------------------------------------------------------------------------------- /scrapers/employees/tests/employees.test.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import employeeData from "./data/employees/employee_results_BE.json"; 7 | 8 | import employees from "../employees"; 9 | import { validate } from "uuid"; 10 | 11 | // This should not be top-level, but we needed a workaround. 12 | // https://github.com/facebook/jest/issues/11543 13 | jest.unmock("request-promise-native"); 14 | jest.setTimeout(20_000); 15 | 16 | describe("static data", () => { 17 | // Test to make sure parsing of an employees result page stays the same 18 | it("should be able to parse API responses", async () => { 19 | const employeeList = employees.parseApiResponse(employeeData); 20 | 21 | for (const employee of employeeList) { 22 | // https://stackoverflow.com/questions/46155/whats-the-best-way-to-validate-an-email-address-in-javascript 23 | const validId = 24 | employee.id.match(/\S+@\S+\.\S+/) || validate(employee.id); 25 | expect(validId).toBeTruthy(); 26 | expect(employee.name.toLowerCase().includes("do not use")).toBeFalsy(); 27 | 28 | // The employees API returns 'Not Avaiable' for some fields instead of making them null 29 | // This makes sure we filter them all out 30 | const matchesNotAvailable = (obj: any): boolean => 31 | /Not Available/.test(JSON.stringify(obj)); 32 | 33 | expect(matchesNotAvailable({ key: "Not Available" })).toBeTruthy(); // Sanity-check the regex 34 | expect(matchesNotAvailable(employee)).toBeFalsy(); 35 | } 36 | 37 | expect(employeeList).toMatchSnapshot(); 38 | }); 39 | }); 40 | 41 | describe("scraping employees", () => { 42 | it("should be able to query the API", async () => { 43 | await employees.queryEmployeesApi(); 44 | 45 | // Don't use a precise number - this hits the live API, so we don't know how many there are 46 | expect(employees.people.length).toBeGreaterThan(1_000); 47 | 48 | for (const employee of employees.people) { 49 | // Ensure that the fields we expect to be required actually are 50 | expect(employee.name).toBeTruthy(); 51 | expect(employee.firstName).toBeTruthy(); 52 | expect(employee.lastName).toBeTruthy(); 53 | } 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /scrapers/filters.ts: -------------------------------------------------------------------------------- 1 | const filters = { 2 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 3 | campus: (campus) => true, 4 | subject: (subject) => ["CS"].includes(subject), 5 | courseNumber: (courseNumber) => courseNumber >= 2500, 6 | truncate: true, 7 | }; 8 | 9 | export default filters; 10 | -------------------------------------------------------------------------------- /scrapers/main.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import matchEmployees from "./employees/matchEmployees"; 7 | import macros from "../utils/macros"; 8 | import classes from "./classes/main"; 9 | import dumpProcessor from "../services/dumpProcessor"; 10 | import prisma from "../services/prisma"; 11 | import { instance as bannerv9parser } from "./classes/parsersxe/bannerv9Parser"; 12 | import "colors"; 13 | 14 | // Main file for scraping 15 | // Run this to run all the scrapers 16 | 17 | class Main { 18 | /** 19 | * Given a list of term IDs, return a subset which will be used to run the scrapers. 20 | * The returned term IDs can be determined by environment variables or by their existance in our database 21 | */ 22 | async getTermIdsToScrape(termIds: string[]): Promise { 23 | const termsToScrapeStr = process.env.TERMS_TO_SCRAPE; 24 | const numOfTermsStr = Number.parseInt(process.env.NUMBER_OF_TERMS); 25 | 26 | if (termsToScrapeStr) { 27 | const unfilteredTermIds = termsToScrapeStr.split(","); 28 | 29 | const terms = unfilteredTermIds.filter((termId) => { 30 | const keep = termIds.includes(termId); 31 | if (!keep && termId !== null) { 32 | macros.warn(`"${termId}" not in given list - skipping`); 33 | } 34 | return keep; 35 | }); 36 | 37 | macros.log("Scraping using user-provided TERMS_TO_SCRAPE"); 38 | return terms; 39 | } else if (!isNaN(numOfTermsStr)) { 40 | return termIds.slice(0, numOfTermsStr); 41 | } else { 42 | const termInfosWithData = await prisma.termInfo.findMany({ 43 | select: { termId: true }, 44 | }); 45 | const termIdsWithData = termInfosWithData.map((t) => t.termId).sort(); 46 | const newestTermIdWithData = termIdsWithData[termIdsWithData.length - 1]; 47 | 48 | return termIds.filter( 49 | (t) => newestTermIdWithData === undefined || t > newestTermIdWithData, 50 | ); 51 | } 52 | } 53 | 54 | async main(): Promise { 55 | const start = Date.now(); 56 | // Get the TermInfo information from Banner 57 | const allTermInfos = await bannerv9parser.getAllTermInfos(); 58 | const termsToScrape = await this.getTermIdsToScrape( 59 | allTermInfos.map((t) => t.termId), 60 | ); 61 | 62 | // Scraping should NOT be resolved simultaneously (eg. via p-map): 63 | // *Employee scraping takes SO MUCH less time (which is why we run it first) 64 | // * So, not running scraping in parallel doesn't hurt us 65 | // * It would make logging a mess (are the logs from employee scraping, or from class scraping?) 66 | const mergedEmployees = await matchEmployees.main(); 67 | const termDump = await classes.main(termsToScrape); 68 | 69 | await dumpProcessor.main({ 70 | termDump: termDump, 71 | profDump: mergedEmployees, 72 | deleteOutdatedData: true, 73 | allTermInfos: allTermInfos, 74 | }); 75 | 76 | const totalTime = Date.now() - start; 77 | 78 | macros.log( 79 | `Done scraping: took ${totalTime} ms (${(totalTime / 60000).toFixed( 80 | 2, 81 | )} minutes)\n\n`.green.underline, 82 | ); 83 | } 84 | } 85 | 86 | const instance = new Main(); 87 | export default instance; 88 | 89 | if (require.main === module) { 90 | instance 91 | .main() 92 | .then(() => prisma.$disconnect()) 93 | .catch((err) => macros.error(err)); 94 | } 95 | -------------------------------------------------------------------------------- /scripts/migrate_major_data.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | import fs from "fs-extra"; 6 | import path from "path"; 7 | import pMap from "p-map"; 8 | import { Major, Prisma } from "@prisma/client"; 9 | import prisma from "../services/prisma"; 10 | 11 | // In order to execute this module, you need a directory `data` 12 | // that contains the file `majors.json`. The JSON object in 13 | // that file must conform to the `MajorJSON` interface. 14 | // This file will then insert all majors provided in the file 15 | // into the database. 16 | 17 | const FILE_NAME = "majors.json"; 18 | const CONCURRENCY_COUNT = 10; 19 | 20 | interface MajorInput { 21 | id: string; 22 | yearVersion: string; 23 | major: Prisma.JsonObject; 24 | plansOfStudy: Prisma.JsonObject; 25 | } 26 | 27 | interface MajorJSON { 28 | all_objects: MajorInput[]; 29 | } 30 | 31 | function fetchData(): MajorJSON { 32 | return JSON.parse( 33 | fs.readFileSync(path.join(__dirname, "..", "data", FILE_NAME), "utf-8"), 34 | ); 35 | } 36 | 37 | function migrateData(majorDirectory: MajorInput[]): Promise { 38 | return pMap( 39 | majorDirectory, 40 | (m: MajorInput) => { 41 | const majorId = m.id; 42 | const yearVersion = String(m.yearVersion); 43 | 44 | const newMajor: Major = { 45 | majorId, 46 | yearVersion, 47 | spec: m.major, 48 | plansOfStudy: m.plansOfStudy, 49 | }; 50 | 51 | return prisma.major.upsert({ 52 | where: { yearVersion_majorId: { majorId, yearVersion } }, 53 | create: newMajor, 54 | update: newMajor, 55 | }); 56 | }, 57 | { concurrency: CONCURRENCY_COUNT }, 58 | ); 59 | } 60 | 61 | (async () => { 62 | const startTime = Date.now(); 63 | const ms = await migrateData(fetchData().all_objects); 64 | const duration = (Date.now() - startTime) / 1000; // how long inserting took in seconds 65 | console.log( 66 | `Success! ${ms.length} majors were inserted or updated in ${duration} seconds! You may exit.`, 67 | ); 68 | })(); 69 | -------------------------------------------------------------------------------- /scripts/populateES.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | * 5 | * script to fill elasticsearch by quering postgres 6 | */ 7 | import { Course, Professor } from "@prisma/client"; 8 | 9 | import elastic from "../utils/elastic"; 10 | import prisma from "../services/prisma"; 11 | import ElasticCourseSerializer from "../serializers/elasticCourseSerializer"; 12 | import ElasticProfSerializer from "../serializers/elasticProfSerializer"; 13 | import macros from "../utils/macros"; 14 | 15 | export async function bulkUpsertCourses( 16 | courses: Course[], 17 | ): Promise> { 18 | // FIXME this pattern is bad 19 | const serializedCourses = await new ElasticCourseSerializer().bulkSerialize( 20 | courses, 21 | true, 22 | ); 23 | return elastic.bulkIndexFromMap(elastic.CLASS_ALIAS, serializedCourses); 24 | } 25 | 26 | export async function bulkUpsertProfs( 27 | profs: Professor[], 28 | ): Promise> { 29 | const serializedProfs = await new ElasticProfSerializer().bulkSerialize( 30 | profs, 31 | ); 32 | return elastic.bulkIndexFromMap(elastic.EMPLOYEE_ALIAS, serializedProfs); 33 | } 34 | 35 | export async function populateES(): Promise { 36 | // FIXME - These Prisma calls used to be in parallel, but Prisma seems to be having issues with parallel calls - 37 | // our connection limit is 30, and RDS (our AWS db) reports a max of ~15-20 connections at any time, yet parallel calls 38 | // (specifically this one) cause our updater to periodically die due to a lack of free connections (allegedly). 39 | // This can be switched back to parallel later, but this is low priority - all it does is slow our updater a tiny bit. 40 | await bulkUpsertCourses(await prisma.course.findMany()); 41 | await bulkUpsertProfs(await prisma.professor.findMany()); 42 | } 43 | 44 | if (require.main === module) { 45 | macros.log( 46 | `Populating ES at ${macros.getEnvVariable( 47 | "elasticURL", 48 | )} from Postgres at ${macros.getEnvVariable("dbHost")}`, 49 | ); 50 | (async () => { 51 | await populateES(); 52 | macros.log("Success! Closing elastic client and exiting."); 53 | elastic.closeClient(); 54 | process.exit(); 55 | })().catch((e) => macros.error(e)); 56 | } 57 | -------------------------------------------------------------------------------- /scripts/resetIndex.ts: -------------------------------------------------------------------------------- 1 | import elastic from "../utils/elastic"; 2 | import macros from "../utils/macros"; 3 | 4 | if (require.main === module) { 5 | macros.log( 6 | "Resetting indices for URL", 7 | macros.getEnvVariable("elasticURL") || "localhost:9200", 8 | ); 9 | (async () => { 10 | await elastic.resetIndex(); 11 | macros.log("Success! Closing elastic client and exiting."); 12 | elastic.closeClient(); 13 | process.exit(); 14 | })().catch((e) => macros.error(e)); 15 | } 16 | -------------------------------------------------------------------------------- /scripts/resetIndexWithoutLoss.ts: -------------------------------------------------------------------------------- 1 | import elastic from "../utils/elastic"; 2 | import macros from "../utils/macros"; 3 | 4 | if (require.main === module) { 5 | macros.log( 6 | "Resetting indices without data loss for URL", 7 | macros.getEnvVariable("elasticURL") || "localhost:9200", 8 | ); 9 | (async () => { 10 | await elastic.resetIndexWithoutLoss(); 11 | macros.log("Success! Closing elastic client and exiting."); 12 | elastic.closeClient(); 13 | process.exit(); 14 | })().catch((e) => macros.error(e)); 15 | } 16 | -------------------------------------------------------------------------------- /serializers/elasticCourseSerializer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | import _ from "lodash"; 6 | 7 | import CourseSerializer from "./courseSerializer"; 8 | import { ESCourse, ESSection } from "../types/serializerTypes"; 9 | 10 | class ElasticCourseSerializer extends CourseSerializer { 11 | courseProps(): string[] { 12 | return []; 13 | } 14 | 15 | finishCourseObj(course): ESCourse { 16 | return _.pick(course, [ 17 | "host", 18 | "name", 19 | "subject", 20 | "classId", 21 | "termId", 22 | "nupath", 23 | ]); 24 | } 25 | 26 | finishSectionObj(section): ESSection { 27 | return _.pick(section, ["profs", "classType", "crn", "campus", "honors"]); 28 | } 29 | } 30 | 31 | export default ElasticCourseSerializer; 32 | -------------------------------------------------------------------------------- /serializers/elasticProfSerializer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | import _ from "lodash"; 6 | import ProfSerializer from "./profSerializer"; 7 | import { Professor as PrismaProfessor } from "@prisma/client"; 8 | import { ESProfessor } from "../types/serializerTypes"; 9 | 10 | class ElasticProfSerializer extends ProfSerializer { 11 | _serializeProf(prof: PrismaProfessor): ESProfessor { 12 | return _.pick(prof, ["id", "name", "email", "phone"]); 13 | } 14 | } 15 | 16 | export default ElasticProfSerializer; 17 | -------------------------------------------------------------------------------- /serializers/hydrateCourseSerializer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | import _ from "lodash"; 6 | import CourseSerializer from "./courseSerializer"; 7 | import { Course, Section } from "../types/types"; 8 | import { SerializedSection } from "../types/serializerTypes"; 9 | 10 | class HydrateCourseSerializer extends CourseSerializer { 11 | courseProps(): string[] { 12 | return ["lastUpdateTime", "termId", "host", "subject", "classId"]; 13 | } 14 | 15 | finishCourseObj(course: Course): Course { 16 | return course; 17 | } 18 | 19 | finishSectionObj(section: SerializedSection): Section { 20 | // We know this will work, but Typescript doesn't 21 | // In the main class, we add the fields from this.courseProps() to the section 22 | // This creates a proper Section, but TS doesn't know we do that. 23 | return _.omit(section, ["id", "classHash"]) as unknown as Section; 24 | } 25 | } 26 | 27 | export default HydrateCourseSerializer; 28 | -------------------------------------------------------------------------------- /serializers/hydrateProfSerializer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | import ProfSerializer from "./profSerializer"; 6 | import { Professor as PrismaProfessor } from "@prisma/client"; 7 | 8 | class HydrateProfSerializer extends ProfSerializer { 9 | _serializeProf(prof: PrismaProfessor): PrismaProfessor { 10 | return prof; 11 | } 12 | } 13 | 14 | export default HydrateProfSerializer; 15 | -------------------------------------------------------------------------------- /serializers/hydrateSerializer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | import prisma from "../services/prisma"; 6 | import HydrateCourseSerializer from "./hydrateCourseSerializer"; 7 | import HydrateProfSerializer from "./hydrateProfSerializer"; 8 | import { 9 | Course as PrismaCourse, 10 | Professor as PrismaProfessor, 11 | } from "@prisma/client"; 12 | import { 13 | CourseSearchResult, 14 | ProfessorSearchResult, 15 | SearchResult, 16 | } from "../types/searchTypes"; 17 | 18 | class HydrateSerializer { 19 | courseSerializer: HydrateCourseSerializer; 20 | profSerializer: HydrateProfSerializer; 21 | 22 | constructor() { 23 | this.courseSerializer = new HydrateCourseSerializer(); 24 | this.profSerializer = new HydrateProfSerializer(); 25 | } 26 | 27 | async bulkSerialize(instances: any[]): Promise { 28 | const profs = instances.filter((instance) => { 29 | return instance._source.type === "employee"; 30 | }); 31 | 32 | const courses = instances.filter((instance) => { 33 | return instance._source.type === "class"; 34 | }); 35 | 36 | const profData: PrismaProfessor[] = await prisma.professor.findMany({ 37 | where: { 38 | id: { 39 | in: profs.map((prof) => prof._id), 40 | }, 41 | }, 42 | }); 43 | 44 | const courseData: PrismaCourse[] = await prisma.course.findMany({ 45 | where: { 46 | id: { 47 | in: courses.map((course) => course._id), 48 | }, 49 | }, 50 | }); 51 | 52 | const serializedProfs = (await this.profSerializer.bulkSerialize( 53 | profData, 54 | )) as Record; 55 | 56 | const serializedCourses = (await this.courseSerializer.bulkSerialize( 57 | courseData, 58 | )) as Record; 59 | 60 | const serializedResults = { ...serializedProfs, ...serializedCourses }; 61 | return instances 62 | .map((instance) => serializedResults[instance._id]) 63 | .filter((elem) => elem); 64 | } 65 | } 66 | 67 | export default HydrateSerializer; 68 | -------------------------------------------------------------------------------- /serializers/profSerializer.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | import _ from "lodash"; 6 | import { Professor as PrismaProfessor } from "@prisma/client"; 7 | import { SerializedProfessor } from "../types/serializerTypes"; 8 | 9 | class ProfSerializer> { 10 | async bulkSerialize( 11 | instances: PrismaProfessor[], 12 | ): Promise>> { 13 | return _.keyBy( 14 | instances.map((instance) => { 15 | return this._bulkSerializeProf(this._serializeProf(instance)); 16 | }), 17 | (res) => res.employee.id, 18 | ); 19 | } 20 | 21 | _bulkSerializeProf(prof: T): SerializedProfessor { 22 | return { 23 | employee: prof, 24 | type: "employee", 25 | }; 26 | } 27 | 28 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 29 | _serializeProf(prof: PrismaProfessor): T { 30 | throw new Error("serializeProf not implemented"); 31 | } 32 | } 33 | 34 | export default ProfSerializer; 35 | -------------------------------------------------------------------------------- /services/prisma.ts: -------------------------------------------------------------------------------- 1 | import { PrismaClient } from "@prisma/client"; 2 | import macros from "../utils/macros"; 3 | 4 | let prisma: PrismaClient; 5 | try { 6 | prisma = new PrismaClient({ 7 | log: ["info", "warn", "error"], 8 | }); 9 | macros.log("** Created new Prisma client"); 10 | } catch (e) { 11 | macros.error(e); 12 | } 13 | 14 | export default prisma; 15 | -------------------------------------------------------------------------------- /template.env: -------------------------------------------------------------------------------- 1 | # These dummy variables are needed for our Twilio code (due to how their library is written) 2 | # Removing them will cause a crash when running 3 | TWILIO_PHONE_NUMBER=this_is_a_fake_variable 4 | TWILIO_ACCOUNT_SID=AC_this_is_also_a_fake_variable_but_must_start_with_AC 5 | TWILIO_AUTH_TOKEN=this_is_a_fake_variable 6 | TWILIO_VERIFY_SERVICE_ID=this_is_a_fake_variable 7 | 8 | CLIENT_ORIGIN=http://localhost:5000 9 | JWT_SECRET= 10 | 11 | # This is only used for local development 12 | POSTGRES_PASSWORD=default_password -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | We have three types of tests: 2 | 3 | - General: 4 | 5 | - Description: 6 | - These are basic integration/acceptance/unit tests. No specific setup is necessary for them. 7 | - Requirements to run: 8 | - Download the repo 9 | - Locations: 10 | - These tests can be found in the `general` folder, or anywhere else in the codebase. For some tests, they are kept in the same directory as their code for ease of access. 11 | - We don't really prefer one way over the other, but do ask that the `general` directory remain (somewhat) organized 12 | 13 | - Database: 14 | - Description: 15 | - These tests are database specific, and as such, need a working database instance to test against 16 | - Requirements to run: 17 | - Download the repo 18 | - Have a working PSQL instance running on your local device 19 | - Locations: 20 | - Same as **general** tests. The database tests can be differentiated from the general tests by the `.seq` extension, such as `search.test.seq.ts` 21 | 22 | Neither **General** nor **Database** tests should require an internet connection. All requests should be cached. In other words -- if Banner dies, these tests should pass nonetheless. 23 | 24 | End to end tests **can** send real requests (and should), as the Banner-CCA connection is critical for our use cases. 25 | 26 | - End to end 27 | - Description: 28 | - These are end-to-end tests. They do everything, from environment setup onwards. As such -- these tests shouldn't be run locally, they're moreso meant for the CI 29 | - Requirements to run: 30 | - Download the repo 31 | - NOTE: Highly discourage running this locally, but: 32 | - _Have a local environment (DB, Elasticsearch) that you're willing to completely trash_ 33 | - **Note**: When running in CI, these tests set up their own environment. They can't do that locally, so don't run it locally unless you're willing to fully reset your local environment. 34 | - Locations: 35 | - **Only** in the `tests/end_to_end` directory 36 | -------------------------------------------------------------------------------- /tests/database/dbTestEnv.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-var-requires */ 2 | // tests/nexus-test-environment.js 3 | import dotenv from "dotenv"; 4 | const { Client } = require("pg"); 5 | const NodeEnvironment = require("jest-environment-node").TestEnvironment; 6 | const { v4: uuid } = require("uuid"); 7 | const util = require("util"); 8 | const exec = util.promisify(require("child_process").exec); 9 | 10 | const prismaBinary = "./node_modules/.bin/prisma"; 11 | 12 | /** 13 | * Custom test environment for Nexus, Prisma and Postgres 14 | */ 15 | class PrismaTestEnvironment extends NodeEnvironment { 16 | constructor(config) { 17 | super(config); 18 | // Generate a unique schema identifier for this test context 19 | this.schema = `test_${uuid()}`; 20 | // Generate the pg connection string for the test schema 21 | dotenv.config(); 22 | this.databaseUrl = `postgresql://postgres:${process.env.POSTGRES_PASSWORD}@localhost:5432/searchneu_test?schema=${this.schema}`; 23 | } 24 | 25 | async setup() { 26 | // Set the equired environment variable to contain the connection string 27 | // to our database test schema 28 | process.env.DATABASE_URL = this.databaseUrl; 29 | this.global.process.env.DATABASE_URL = this.databaseUrl; 30 | process.env.DATABASE_URL_WITH_CONNECTIONS = this.databaseUrl; 31 | this.global.process.env.DATABASE_URL_WITH_CONNECTIONS = this.databaseUrl; 32 | // Run the migrations to ensure our schema has the required structure 33 | await exec(`${prismaBinary} migrate dev --preview-feature`); 34 | await exec(`${prismaBinary} generate`); 35 | return super.setup(); 36 | } 37 | 38 | async teardown() { 39 | // Drop the schema after the tests have completed 40 | const client = new Client({ 41 | connectionString: this.databaseUrl, 42 | }); 43 | await client 44 | .connect() 45 | .then(() => console.log(`Connected to ${this.databaseUrl}`)) 46 | .catch((err) => console.log(err)); 47 | await client 48 | .query(`DROP SCHEMA IF EXISTS "${this.schema}" CASCADE`) 49 | .then(() => console.log(`Dropped schema '${this.schema}' (if exists)`)) 50 | .catch((err) => console.log(err)) 51 | .finally(() => client.end); 52 | 53 | await client.end(); 54 | } 55 | } 56 | module.exports = PrismaTestEnvironment; 57 | -------------------------------------------------------------------------------- /tests/database/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | displayName: "Database Tests", 3 | rootDir: "../../", 4 | moduleFileExtensions: ["js", "json", "node", "ts"], 5 | testMatch: ["**/*.(spec|test).seq.[jt]s?(x)"], 6 | testEnvironment: "/tests/database/dbTestEnv.ts", 7 | }; 8 | -------------------------------------------------------------------------------- /tests/database/notificationsManager.test.seq.ts: -------------------------------------------------------------------------------- 1 | import prisma from "../../services/prisma"; 2 | import notifs from "../../services/notificationsManager"; 3 | 4 | async function insertCourses(courseIds: string[]): Promise { 5 | for (const course of courseIds) { 6 | await prisma.course.create({ 7 | data: { 8 | id: course, 9 | }, 10 | }); 11 | } 12 | } 13 | 14 | async function insertSections(sectionIds: string[]): Promise { 15 | for (const section of sectionIds) { 16 | await prisma.section.create({ 17 | data: { 18 | id: section, 19 | }, 20 | }); 21 | } 22 | } 23 | 24 | it("upserting a user doesn't fail on duplicate", async () => { 25 | const phoneNumber = "this is a phone number string"; 26 | await notifs.upsertUser(phoneNumber); 27 | await notifs.upsertUser(phoneNumber); 28 | await notifs.upsertUser(phoneNumber); 29 | }); 30 | 31 | describe("user subscriptions", () => { 32 | const phoneNumber = "911"; 33 | const sectionIds = ["a", "b", "c"]; 34 | const courseIds = ["1"]; 35 | 36 | beforeEach(async () => { 37 | await prisma.section.deleteMany({}); 38 | await prisma.course.deleteMany({}); 39 | }); 40 | 41 | it("cannot insert subscriptions for nonexistent courses/sections", async () => { 42 | await notifs.upsertUser(phoneNumber); 43 | await expect( 44 | notifs.putUserSubscriptions(phoneNumber, sectionIds, courseIds), 45 | ).rejects.toThrow("Foreign key constraint violated"); 46 | }); 47 | 48 | it("can insert new subscriptions", async () => { 49 | await insertCourses(courseIds); 50 | await insertSections(sectionIds); 51 | 52 | await notifs.upsertUser(phoneNumber); 53 | await notifs.putUserSubscriptions(phoneNumber, sectionIds, courseIds); 54 | expect(await notifs.getUserSubscriptions(phoneNumber)).toEqual({ 55 | phoneNumber, 56 | sectionIds: [], 57 | courseIds: [], 58 | }); 59 | }); 60 | 61 | it("gets no subscriptions for a user that doesn't exist", async () => { 62 | expect(await notifs.getUserSubscriptions(phoneNumber)).toEqual({ 63 | phoneNumber, 64 | courseIds: [], 65 | sectionIds: [], 66 | }); 67 | }); 68 | 69 | it("duplicate users and subscriptions aren't counted twice", async () => { 70 | await insertCourses(courseIds); 71 | await insertSections(sectionIds); 72 | 73 | await notifs.upsertUser(phoneNumber); 74 | await notifs.putUserSubscriptions(phoneNumber, sectionIds, courseIds); 75 | await notifs.upsertUser(phoneNumber); 76 | await notifs.upsertUser(phoneNumber); 77 | await notifs.putUserSubscriptions( 78 | phoneNumber, 79 | sectionIds.slice(0, 1), 80 | courseIds, 81 | ); 82 | 83 | expect(await notifs.getUserSubscriptions(phoneNumber)).toEqual({ 84 | phoneNumber, 85 | sectionIds: [], 86 | courseIds: [], 87 | }); 88 | }); 89 | 90 | it("subscriptions can be deleted", async () => { 91 | await insertCourses(courseIds); 92 | await insertSections(sectionIds); 93 | 94 | await notifs.upsertUser(phoneNumber); 95 | await notifs.putUserSubscriptions(phoneNumber, sectionIds, courseIds); 96 | await notifs.deleteUserSubscriptions(phoneNumber, sectionIds, []); 97 | expect(await notifs.getUserSubscriptions(phoneNumber)).toEqual({ 98 | phoneNumber, 99 | sectionIds: [], 100 | courseIds: [], 101 | }); 102 | 103 | await notifs.deleteAllUserSubscriptions(phoneNumber); 104 | expect(await notifs.getUserSubscriptions(phoneNumber)).toEqual({ 105 | phoneNumber, 106 | sectionIds: [], 107 | courseIds: [], 108 | }); 109 | 110 | await notifs.deleteAllUserSubscriptions(phoneNumber); 111 | expect(await notifs.getUserSubscriptions(phoneNumber)).toEqual({ 112 | phoneNumber, 113 | sectionIds: [], 114 | courseIds: [], 115 | }); 116 | }); 117 | }); 118 | -------------------------------------------------------------------------------- /tests/end_to_end/README.md: -------------------------------------------------------------------------------- 1 | ### Not for local testing 2 | 3 | Don't run these locally. All of the setup work is done via Github workflows (see the `.github` folder). 4 | 5 | These tests are all based on an environment which has already undergone some _specific_ setup, and it assumes the tests are running on a disposable environment; data isn't preserved, existing setups aren't respected, etc. 6 | 7 | At a bare minimum, the Docker containers are running, the schemas have been populated, and there's some data - check the workflows for more information. 8 | -------------------------------------------------------------------------------- /tests/end_to_end/babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": { 7 | "node": "22" 8 | } 9 | } 10 | ], 11 | "@babel/preset-typescript" 12 | ], 13 | "plugins": [], 14 | "ignore": ["node_modules"] 15 | } 16 | -------------------------------------------------------------------------------- /tests/end_to_end/elastic.test.git.ts: -------------------------------------------------------------------------------- 1 | import classMap from "../../scrapers/classes/classMapping.json"; 2 | import client from "../../utils/elastic"; 3 | import employeeMap from "../../scrapers/employees/employeeMapping.json"; 4 | 5 | it("Connections", async () => { 6 | expect(await client.isConnected()).toBeTruthy(); 7 | }); 8 | 9 | it("fetchIndexName", async () => { 10 | expect(client["indexes"]["classes"]).toEqual({ 11 | name: "", 12 | mapping: classMap, 13 | }); 14 | await client.fetchIndexName("classes"); 15 | 16 | expect(client["indexes"]["classes"]["name"]).toBe("classes_blue"); 17 | }); 18 | 19 | it("Creating indexes", async () => { 20 | const indexName = "indexname"; 21 | const aliasName = "aliasname"; 22 | 23 | await client.createIndex(indexName, classMap); 24 | await expect(client.createIndex(indexName, classMap)).rejects.toThrowError(); 25 | 26 | await client.createAlias(indexName, aliasName); 27 | 28 | // @ts-expect-error - we know the type is missing, that's the point 29 | client["indexes"][aliasName] = { mapping: classMap }; 30 | expect(client["indexes"][aliasName]["name"]).toBeUndefined(); 31 | await client.fetchIndexName(aliasName); 32 | expect(client["indexes"][aliasName]["name"]).toMatch(`${aliasName}_`); 33 | 34 | await client.deleteIndex(indexName); 35 | await expect(client.deleteIndex(indexName)).rejects.toThrowError(); 36 | 37 | await client.createIndex(indexName, classMap); 38 | await client.deleteIndex(indexName); 39 | }); 40 | 41 | it("queries", async () => { 42 | const aliasName = "e2e-employees-jason"; 43 | 44 | // @ts-expect-error - type doesn't match, that's OK 45 | client["indexes"][aliasName] = { 46 | mapping: employeeMap, 47 | }; 48 | 49 | const id = "Jason Jason"; 50 | await client.bulkIndexFromMap(aliasName, { 51 | "Jason Jason": { 52 | type: "employee", 53 | employee: { 54 | id: id, 55 | name: "Jason Jason", 56 | emails: ["jason@jason.jason"], 57 | phone: "911", 58 | }, 59 | }, 60 | }); 61 | 62 | await new Promise((resolve) => setTimeout(resolve, 1_000)); // We need a little pause for the indexing 63 | 64 | const getId = async (): Promise => { 65 | // @ts-expect-error - body type is inaccurate 66 | const resp = await client.query(aliasName, 0, 10, { 67 | query: { match_all: {} }, 68 | }); 69 | return resp?.body?.hits?.hits?.[0]?.["_id"] ?? ""; 70 | }; 71 | 72 | expect(await getId()).toBe(id); 73 | 74 | await client.resetIndexWithoutLoss(); 75 | await new Promise((resolve) => setTimeout(resolve, 1_000)); // We need a little pause for the indexing 76 | 77 | expect(await getId()).toBe(id); 78 | }, 40_000); 79 | -------------------------------------------------------------------------------- /tests/end_to_end/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | displayName: "Tests for GitHub Workflows", 3 | rootDir: "./", 4 | testEnvironment: "node", 5 | moduleFileExtensions: ["js", "json", "node", "ts"], 6 | testMatch: ["**/*.(spec|test).git.[jt]s?(x)"], 7 | }; 8 | -------------------------------------------------------------------------------- /tests/end_to_end/scraper.test.git.ts: -------------------------------------------------------------------------------- 1 | import employees from "../../scrapers/employees/employees"; 2 | import fs from "fs-extra"; 3 | import macros from "../../utils/macros"; 4 | import cache from "../../scrapers/cache"; 5 | 6 | describe("scraping employees", () => { 7 | it("should be able to query the API and cache it", async () => { 8 | jest.spyOn(employees, "queryEmployeesApi"); 9 | const result = await employees.main(); 10 | const result2 = await employees.main(); 11 | 12 | // The IDs are random for each run, so we remove them. 13 | result.forEach((res) => delete res.id); 14 | result2.forEach((res) => delete res.id); 15 | 16 | expect(result2).toEqual(result); 17 | // Of the two calls, only one should have queried the live API 18 | // The other would use the cache 19 | expect(employees.queryEmployeesApi).toHaveBeenCalledTimes(0); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /tests/end_to_end/setup.test.git.ts: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | import prisma from "../../services/prisma"; 3 | import server from "../../graphql/index"; 4 | import { DocumentNode } from "graphql"; 5 | import { GraphQLResponse } from "apollo-server-core"; 6 | 7 | const NUM_TERMIDS = 3; 8 | const NUMS_COURSES = { 9 | "202240": 635, 10 | "202250": 502, 11 | "202260": 545, 12 | }; 13 | const NUMS_SECTIONS = { 14 | "202240": 813 + 1, // There are 813 sections, but one additional section that we added during setup 15 | "202250": 1380, 16 | "202260": 650, 17 | }; 18 | const NUM_SECTIONS = Object.values(NUMS_SECTIONS).reduce((a, b) => a + b, 0); 19 | const NUM_COURSES = Object.values(NUMS_COURSES).reduce((a, b) => a + b, 0); 20 | 21 | async function query(q: DocumentNode): Promise { 22 | return await server.executeOperation({ query: q }); 23 | } 24 | 25 | describe("TermID setup", () => { 26 | test("term IDs are in the database", async () => { 27 | expect(await prisma.termInfo.count()).toBe(NUM_TERMIDS); 28 | }); 29 | 30 | test("termIDs are accessible through GraphQL", async () => { 31 | const res = await query(gql` 32 | query { 33 | termInfos(subCollege: "NEU") { 34 | text 35 | termId 36 | subCollege 37 | } 38 | } 39 | `); 40 | expect(res.data?.termInfos.length).toBe(NUM_TERMIDS); 41 | }); 42 | 43 | test("The termIDs in the database match those in GraphQL", async () => { 44 | const res = await query(gql` 45 | query { 46 | termInfos(subCollege: "NEU") { 47 | text 48 | termId 49 | subCollege 50 | } 51 | } 52 | `); 53 | const gqlTermInfos = res.data?.termInfos.map((t) => t.termId).sort(); 54 | 55 | const dbTermInfos = ( 56 | await prisma.termInfo.findMany({ 57 | select: { 58 | termId: true, 59 | subCollege: true, 60 | text: true, 61 | }, 62 | }) 63 | ) 64 | .map((t) => t.termId) 65 | .sort(); 66 | 67 | expect(dbTermInfos).toEqual(gqlTermInfos); 68 | }); 69 | }); 70 | 71 | describe("Course and section setup", () => { 72 | test("courses/sections are in the database", async () => { 73 | for (const [termId, expected] of Object.entries(NUMS_COURSES)) { 74 | const actual = await prisma.course.count({ 75 | where: { 76 | termId: termId, 77 | }, 78 | }); 79 | expect(actual).toBe(expected); 80 | } 81 | 82 | expect(await prisma.course.count()).toBe(NUM_COURSES); 83 | 84 | for (const [termId, expected] of Object.entries(NUMS_SECTIONS)) { 85 | const actual = await prisma.section.count({ 86 | where: { 87 | course: { 88 | termId: termId, 89 | }, 90 | }, 91 | }); 92 | 93 | expect(actual).toBe(expected); 94 | } 95 | 96 | expect(await prisma.section.count()).toBe(NUM_SECTIONS); 97 | }); 98 | 99 | test("Courses/sections are in GraphQL", async () => { 100 | for (const termId of Object.keys(NUMS_COURSES)) { 101 | const res = await query(gql` 102 | query { 103 | search(termId: "${termId}", query: "") { 104 | totalCount 105 | } 106 | } 107 | `); 108 | // It won't exactly match the number we have, but at least make sure we have SOMETHING 109 | expect(res.data?.search.totalCount).toBeGreaterThan(0); 110 | } 111 | }); 112 | }); 113 | -------------------------------------------------------------------------------- /tests/end_to_end/updater.test.git.ts: -------------------------------------------------------------------------------- 1 | import { gql } from "apollo-server"; 2 | import server from "../../graphql/index"; 3 | import { DocumentNode } from "graphql"; 4 | import { GraphQLResponse } from "apollo-server-core"; 5 | 6 | async function query(q: DocumentNode): Promise { 7 | return await server.executeOperation({ query: q }); 8 | } 9 | 10 | describe("Searching for courses", () => { 11 | //https://trello.com/c/fs503gwU/241-process-for-deleting-non-existent-sections-courses 12 | // In our setup, one instance of "202240/CS/2501" had its `last_update_time` set to the 20th century 13 | // As such - it's outdated, and the updater should have removed it 14 | test("outdated courses are removed by the updater", async () => { 15 | const res = await query(gql` 16 | query search { 17 | search(termId: "202240", query: "CS2501") { 18 | nodes { 19 | ... on ClassOccurrence { 20 | sections { 21 | termId 22 | crn 23 | } 24 | } 25 | } 26 | } 27 | } 28 | `); 29 | 30 | const crns = res.data?.search.nodes[0].sections.map((s) => s.crn); 31 | expect(crns.includes("123456789")).toBeTruthy(); 32 | expect(crns.includes("987654321")).toBeFalsy(); 33 | }); 34 | }); 35 | -------------------------------------------------------------------------------- /tests/general/macros.test.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import macros from "../../utils/macros"; 7 | 8 | it("logging things work", () => { 9 | macros.warn(); 10 | macros.verbose(); 11 | macros.error("fjdaj"); 12 | }); 13 | 14 | it("some other stuff doesnt crash", () => { 15 | macros.logAmplitudeEvent("test event", { hi: 4 } as any); 16 | }); 17 | 18 | it("logAmplitudeEvent should not crash", () => { 19 | macros.logAmplitudeEvent("event_from_testing", { a: 3 } as any); 20 | }); 21 | -------------------------------------------------------------------------------- /tests/general/utils/elastic.test.ts: -------------------------------------------------------------------------------- 1 | import elastic from "../../../utils/elastic"; 2 | import classMap from "../../../scrapers/classes/classMapping.json"; 3 | 4 | describe("elastic tests", () => { 5 | beforeEach(() => { 6 | jest.resetAllMocks(); 7 | jest.restoreAllMocks(); 8 | }); 9 | 10 | it("fetchIndexNames", async () => { 11 | jest.spyOn(elastic, "fetchIndexName"); 12 | 13 | await elastic.fetchIndexNames(); 14 | 15 | for (const name of Object.keys(elastic["indexes"])) { 16 | expect(elastic.fetchIndexName).toHaveBeenCalledWith(name); 17 | } 18 | }); 19 | 20 | describe("fetchIndexName", () => { 21 | it("no indexes exist", async () => { 22 | elastic["doesIndexExist"] = jest.fn().mockImplementationOnce(() => false); 23 | elastic.createAlias = jest.fn().mockImplementationOnce(() => { 24 | // void 25 | }); 26 | elastic.createIndex = jest.fn().mockImplementationOnce(() => { 27 | // void 28 | }); 29 | 30 | await elastic.fetchIndexName(elastic.CLASS_ALIAS); 31 | expect(elastic.createIndex).toHaveBeenCalledWith( 32 | `${elastic.CLASS_ALIAS}_blue`, 33 | classMap, 34 | ); 35 | }); 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /tests/general/utils/keys.test.ts: -------------------------------------------------------------------------------- 1 | import Keys from "../../../utils/keys"; 2 | 3 | describe("getHashWithKeysSlice", () => { 4 | it("no obj", () => { 5 | expect(Keys.getHashWithKeysSlice(null, 0)).toBeNull(); 6 | }); 7 | 8 | it("checks if all hash keys are present", () => { 9 | expect(Keys.getHashWithKeysSlice({ host: "", termId: "" }, 3)).toBeNull(); 10 | expect(Keys.getHashWithKeysSlice({}, 1)).toBeNull(); 11 | }); 12 | 13 | it("no output", () => { 14 | expect(Keys.getHashWithKeysSlice({}, 0)).toBe(""); 15 | }); 16 | 17 | it("output keys", () => { 18 | expect( 19 | Keys.getHashWithKeysSlice({ host: "host name", termId: "1234" }, 2), 20 | ).toBe("host_name/1234"); 21 | expect( 22 | Keys.getHashWithKeysSlice( 23 | { host: "northeastern", termId: "1234", subject: "computer science" }, 24 | 2, 25 | ), 26 | ).toBe("northeastern/1234"); 27 | expect( 28 | Keys.getHashWithKeysSlice( 29 | { host: "northeastern", termId: "1234", subject: "computer science" }, 30 | 3, 31 | ), 32 | ).toBe("northeastern/1234/computer_science"); 33 | }); 34 | }); 35 | 36 | it("getHostHash", () => { 37 | expect( 38 | Keys.getHostHash({ 39 | host: "northeastern", 40 | termId: "1234", 41 | subject: "computer science", 42 | }), 43 | ).toBe("northeastern"); 44 | expect(Keys.getHostHash({ host: null })).toBeNull(); 45 | expect(Keys.getHostHash({})).toBeNull(); 46 | }); 47 | 48 | it("getTermHash", () => { 49 | expect( 50 | Keys.getTermHash({ 51 | host: "northeastern", 52 | termId: "1234", 53 | subject: "computer science", 54 | }), 55 | ).toBe("northeastern/1234"); 56 | expect(Keys.getTermHash({})).toBeNull(); 57 | }); 58 | 59 | it("getSubjectHash", () => { 60 | expect( 61 | Keys.getSubjectHash({ 62 | host: "northeastern", 63 | termId: "1234", 64 | subject: "computer science", 65 | }), 66 | ).toBe("northeastern/1234/computer_science"); 67 | expect(Keys.getSubjectHash({})).toBeNull(); 68 | }); 69 | 70 | it("getClassHash", () => { 71 | expect( 72 | Keys.getClassHash({ 73 | host: "neu", 74 | termId: "1234", 75 | subject: "cs", 76 | classId: "id", 77 | }), 78 | ).toBe("neu/1234/cs/id"); 79 | expect(Keys.getClassHash({})).toBeNull(); 80 | }); 81 | 82 | it("getSectionHash", () => { 83 | expect( 84 | Keys.getSectionHash({ 85 | host: "neu", 86 | termId: "1234", 87 | subject: "cs", 88 | classId: "id", 89 | crn: "crn", 90 | }), 91 | ).toBe("neu/1234/cs/id/crn"); 92 | expect(Keys.getSectionHash({})).toBeNull(); 93 | }); 94 | 95 | it("parseSectionHash", () => { 96 | const hash1 = { 97 | host: "neu", 98 | termId: "1234", 99 | subject: "cs", 100 | classId: "id", 101 | crn: "crn", 102 | }; 103 | 104 | expect(Keys.parseSectionHash(Keys.getSectionHash(hash1))).toStrictEqual( 105 | hash1, 106 | ); 107 | expect(Keys.parseSectionHash("")).toBeNull; 108 | expect(Keys.parseSectionHash("neu/1234")).toBeNull; 109 | }); 110 | -------------------------------------------------------------------------------- /tests/unit/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | name: "unittest", 3 | displayName: "Unit Tests", 4 | rootDir: "../../", 5 | moduleFileExtensions: ["js", "json", "node", "ts"], 6 | testMatch: ["**/*.(spec|test).unit.[jt]s?(x)"], 7 | }; 8 | -------------------------------------------------------------------------------- /tests/unit/scrapers/request.test.unit.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Search NEU and licensed under AGPL3. 3 | * See the license file in the root folder for details. 4 | */ 5 | 6 | import Request from "../../../scrapers/request"; 7 | // Give extra time to ensure that the initial DNS lookup works 8 | jest.setTimeout(10_000); 9 | 10 | const request = new Request("request_test", { cacheRequests: false }); 11 | 12 | it("get should work", async () => { 13 | const response = await request.get("https://httpbin.org/get"); 14 | 15 | expect(JSON.parse(response.body)["url"]).toBe("https://httpbin.org/get"); 16 | }); 17 | 18 | it("post should work", async () => { 19 | const response = await request.post("https://httpbin.org/post", { 20 | form: { 21 | arg1: "data", 22 | }, 23 | }); 24 | 25 | expect(JSON.parse(response.body).form).toEqual({ 26 | arg1: "data", 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /tsconfig.eslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "include": ["**/*.js", "**/*.ts", ".eslintrc.js"] 4 | } 5 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | // Don't output the built files - Babel will do that. Tsc just for checking. 4 | "noEmit": true, 5 | "esModuleInterop": true, 6 | "resolveJsonModule": true, 7 | "lib": ["es2021"], 8 | "skipLibCheck": true, 9 | "downlevelIteration": true 10 | }, 11 | "exclude": ["node_modules"] 12 | } 13 | -------------------------------------------------------------------------------- /twilio/client.ts: -------------------------------------------------------------------------------- 1 | import twilio from "twilio"; 2 | 3 | /* Don't merge this with the `./notifs.ts` file 4 | 5 | Keeing this separate allows us to easily mock the Twilio client for tests */ 6 | const twilioClient = twilio( 7 | process.env.TWILIO_ACCOUNT_SID, 8 | process.env.TWILIO_AUTH_TOKEN, 9 | ); 10 | 11 | export { twilioClient }; 12 | -------------------------------------------------------------------------------- /types/notifTypes.ts: -------------------------------------------------------------------------------- 1 | export interface UserInfo { 2 | phoneNumber: string; 3 | courseIds: string[]; 4 | sectionIds: string[]; 5 | } 6 | 7 | // Stores information for all changes to a course or section 8 | export interface NotificationInfo { 9 | updatedCourses: CourseNotificationInfo[]; 10 | updatedSections: SectionNotificationInfo[]; 11 | } 12 | 13 | // marks new sections being added to a Course 14 | export interface CourseNotificationInfo { 15 | subject: string; 16 | courseId: string; 17 | termId: string; 18 | courseHash: string; 19 | numberOfSectionsAdded: number; 20 | campus: string; 21 | } 22 | 23 | // marks seats becoming available in a section 24 | export interface SectionNotificationInfo { 25 | subject: string; 26 | courseId: string; 27 | sectionHash: string; 28 | termId: string; 29 | seatsRemaining: number; 30 | crn: string; 31 | campus: string; 32 | } 33 | -------------------------------------------------------------------------------- /types/requestTypes.ts: -------------------------------------------------------------------------------- 1 | import http from "http"; 2 | import { OptionsOfTextResponseBody, Agents } from "got"; 3 | 4 | export interface HostAnalytics { 5 | totalBytesDownloaded: number; 6 | totalErrors: number; 7 | totalGoodRequests: number; 8 | startTime: number | null; 9 | } 10 | 11 | export type RequestAnalytics = Record; 12 | 13 | export interface RequestPool { 14 | options: http.AgentOptions; 15 | agents: Agents | false; 16 | } 17 | 18 | export interface AgentAnalytics { 19 | socketCount: number; 20 | requestCount: number; 21 | maxSockets: number; 22 | } 23 | 24 | export interface CustomOptions extends OptionsOfTextResponseBody { 25 | url: string; 26 | // defaults to true 27 | cacheRequests?: boolean; 28 | cacheName?: string; 29 | pool?: RequestPool; 30 | } 31 | 32 | export type AmplitudeEvent = Partial & { 33 | totalBytesDownloaded?: number; 34 | totalErrors?: number; 35 | totalGoodRequests?: number; 36 | startTime?: number | null; 37 | hostname: string; 38 | }; 39 | 40 | export type DoRequestReturn = { items: unknown; totalCount: number }; 41 | -------------------------------------------------------------------------------- /types/serializerTypes.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Course as PrismaCourse, 3 | Section as PrismaSection, 4 | Professor as PrismaProfessor, 5 | } from "@prisma/client"; 6 | import { BackendMeeting } from "./types"; 7 | 8 | export interface PrismaCourseWithSections extends PrismaCourse { 9 | sections?: PrismaSection[]; 10 | } 11 | 12 | export interface SerializedSection 13 | extends Omit { 14 | lastUpdateTime: number; 15 | meetings: BackendMeeting[]; 16 | } 17 | 18 | export type FinishedSerializedSection = Omit< 19 | SerializedSection, 20 | "id" | "classHash" 21 | >; 22 | 23 | export type SerializedProfessor = { 24 | employee: T; 25 | type: string; 26 | }; 27 | 28 | export type SerializedCourse = { 29 | type: "class"; 30 | class: C; 31 | sections: S[]; 32 | }; 33 | 34 | export type ESProfessor = Pick; 35 | export type ESCourse = Pick< 36 | PrismaCourse, 37 | "host" | "name" | "subject" | "classId" | "termId" | "nupath" 38 | >; 39 | export type ESSection = Pick< 40 | PrismaSection, 41 | "profs" | "classType" | "crn" | "campus" 42 | >; 43 | --------------------------------------------------------------------------------