├── .eslintignore ├── .eslintrc.json ├── .gitignore ├── LICENSE ├── README.md ├── __tests__ ├── .eslintrc.json ├── npm │ └── gimbap.test.ts ├── server │ ├── models │ │ ├── clusterModel.test.ts │ │ └── endpointBucketsModel.test.ts │ └── utils │ │ ├── endpoints.test.ts │ │ └── loadDePaulEndpointData.test.ts ├── shared │ ├── models │ │ └── endpointModels.test.ts │ └── utils │ │ └── dataGenerator.test.ts └── testUtils │ └── index.ts ├── babel.config.json ├── package.json ├── src ├── client │ ├── App.scss │ ├── App.tsx │ ├── assets │ │ ├── 404.html │ │ └── index.html │ ├── components │ │ ├── clusters │ │ │ ├── Clusters.tsx │ │ │ ├── EndpointList.tsx │ │ │ └── TreeGraph.tsx │ │ ├── common │ │ │ ├── ChipSelector.tsx │ │ │ ├── NavItem.tsx │ │ │ ├── NavigationBar.scss │ │ │ ├── NavigationBar.tsx │ │ │ └── Splash.tsx │ │ ├── documentation │ │ │ ├── ApiDoc.tsx │ │ │ ├── ApiTableOfContents.tsx │ │ │ ├── CodeComment.tsx │ │ │ ├── Contributors.tsx │ │ │ ├── Documentation.tsx │ │ │ ├── Installation.tsx │ │ │ ├── Intro.tsx │ │ │ ├── TableOfContents.scss │ │ │ ├── TableOfContents.tsx │ │ │ └── VisualizingYourData.tsx │ │ └── metrics │ │ │ ├── ClusterLoad.tsx │ │ │ ├── LoadGraph.scss │ │ │ ├── LoadGraph.tsx │ │ │ ├── Metrics.tsx │ │ │ └── RouteLoad.tsx │ ├── hooks │ │ └── useWindowDimensions.ts │ ├── index.scss │ ├── index.tsx │ ├── theme.ts │ ├── tsconfig.json │ ├── types.ts │ └── utils │ │ └── ajax.ts ├── npm │ ├── index.ts │ ├── package.json │ └── tsconfig.json ├── server │ ├── controllers │ │ ├── cacheController.ts │ │ └── dataController.ts │ ├── index.ts │ ├── models │ │ ├── clusterModel.ts │ │ └── endpointBucketsModel.ts │ ├── routes │ │ ├── apiRouter.ts │ │ ├── clusterRouter.ts │ │ └── endpointRouter.ts │ ├── tsconfig.json │ └── utils │ │ ├── MiddlewareError.ts │ │ ├── endpoints.ts │ │ └── loadDePaulEndpointData.ts └── shared │ ├── models │ ├── endpointModel.ts │ └── mongoSetup.ts │ ├── types.ts │ └── utils │ └── dataGenerator.ts ├── webpack.config.js └── whitebg.jpg /.eslintignore: -------------------------------------------------------------------------------- 1 | build -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "@typescript-eslint/parser", 3 | "parserOptions": { 4 | "ecmaVersion": 2021, 5 | "sourceType": "module" 6 | }, 7 | "plugins": [ 8 | "@typescript-eslint", 9 | "react-hooks" 10 | ], 11 | "extends": [ 12 | "plugin:react/recommended", 13 | "plugin:@typescript-eslint/recommended", 14 | "plugin:react/jsx-runtime", 15 | "plugin:react-hooks/recommended" 16 | ], 17 | "env": { 18 | "es2021": true, 19 | "node": true, 20 | "jest": true, 21 | "browser": true 22 | }, 23 | "rules": { 24 | "react-hooks/rules-of-hooks": "error", 25 | "react-hooks/exhaustive-deps": "warn", 26 | "react/prop-types": "off", 27 | "@typescript-eslint/no-var-requires": 0, 28 | "@typescript-eslint/no-require-imports": 1, 29 | "react/jsx-uses-react": 1, 30 | "quotes": [ 31 | 1, 32 | "single", 33 | { 34 | "avoidEscape": true 35 | } 36 | ], 37 | "jsx-quotes": [ 38 | 1, 39 | "prefer-single" 40 | ], 41 | "semi": 1, 42 | "eol-last": 1 43 | }, 44 | "settings": { 45 | "react": { 46 | "pragma": "React", 47 | "version": "detect" 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Aa][Rr][Mm]/ 27 | [Aa][Rr][Mm]64/ 28 | bld/ 29 | [Bb]in/ 30 | [Oo]bj/ 31 | [Ll]og/ 32 | [Ll]ogs/ 33 | 34 | # Visual Studio 2015/2017 cache/options directory 35 | .vs/ 36 | # Uncomment if you have tasks that create the project's static files in wwwroot 37 | #wwwroot/ 38 | 39 | # Visual Studio 2017 auto generated files 40 | Generated\ Files/ 41 | 42 | # MSTest test Results 43 | [Tt]est[Rr]esult*/ 44 | [Bb]uild[Ll]og.* 45 | 46 | # NUnit 47 | *.VisualState.xml 48 | TestResult.xml 49 | nunit-*.xml 50 | 51 | # Build Results of an ATL Project 52 | [Dd]ebugPS/ 53 | [Rr]eleasePS/ 54 | dlldata.c 55 | 56 | # Benchmark Results 57 | BenchmarkDotNet.Artifacts/ 58 | 59 | # .NET Core 60 | project.lock.json 61 | project.fragment.lock.json 62 | artifacts/ 63 | 64 | # StyleCop 65 | StyleCopReport.xml 66 | 67 | # Files built by Visual Studio 68 | *_i.c 69 | *_p.c 70 | *_h.h 71 | *.ilk 72 | *.meta 73 | *.obj 74 | *.iobj 75 | *.pch 76 | *.pdb 77 | *.ipdb 78 | *.pgc 79 | *.pgd 80 | *.rsp 81 | *.sbr 82 | *.tlb 83 | *.tli 84 | *.tlh 85 | *.tmp 86 | *.tmp_proj 87 | *_wpftmp.csproj 88 | *.log 89 | *.vspscc 90 | *.vssscc 91 | .builds 92 | *.pidb 93 | *.svclog 94 | *.scc 95 | 96 | # Chutzpah Test files 97 | _Chutzpah* 98 | 99 | # Visual C++ cache files 100 | ipch/ 101 | *.aps 102 | *.ncb 103 | *.opendb 104 | *.opensdf 105 | *.sdf 106 | *.cachefile 107 | *.VC.db 108 | *.VC.VC.opendb 109 | 110 | # Visual Studio profiler 111 | *.psess 112 | *.vsp 113 | *.vspx 114 | *.sap 115 | 116 | # Visual Studio Trace Files 117 | *.e2e 118 | 119 | # TFS 2012 Local Workspace 120 | $tf/ 121 | 122 | # Guidance Automation Toolkit 123 | *.gpState 124 | 125 | # ReSharper is a .NET coding add-in 126 | _ReSharper*/ 127 | *.[Rr]e[Ss]harper 128 | *.DotSettings.user 129 | 130 | # TeamCity is a build add-in 131 | _TeamCity* 132 | 133 | # DotCover is a Code Coverage Tool 134 | *.dotCover 135 | 136 | # AxoCover is a Code Coverage Tool 137 | .axoCover/* 138 | !.axoCover/settings.json 139 | 140 | # Visual Studio code coverage results 141 | *.coverage 142 | *.coveragexml 143 | 144 | # NCrunch 145 | _NCrunch_* 146 | .*crunch*.local.xml 147 | nCrunchTemp_* 148 | 149 | # MightyMoose 150 | *.mm.* 151 | AutoTest.Net/ 152 | 153 | # Web workbench (sass) 154 | .sass-cache/ 155 | 156 | # Installshield output folder 157 | [Ee]xpress/ 158 | 159 | # DocProject is a documentation generator add-in 160 | DocProject/buildhelp/ 161 | DocProject/Help/*.HxT 162 | DocProject/Help/*.HxC 163 | DocProject/Help/*.hhc 164 | DocProject/Help/*.hhk 165 | DocProject/Help/*.hhp 166 | DocProject/Help/Html2 167 | DocProject/Help/html 168 | 169 | # Click-Once directory 170 | publish/ 171 | 172 | # Publish Web Output 173 | *.[Pp]ublish.xml 174 | *.azurePubxml 175 | # Note: Comment the next line if you want to checkin your web deploy settings, 176 | # but database connection strings (with potential passwords) will be unencrypted 177 | *.pubxml 178 | *.publishproj 179 | 180 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 181 | # checkin your Azure Web App publish settings, but sensitive information contained 182 | # in these scripts will be unencrypted 183 | PublishScripts/ 184 | 185 | # NuGet Packages 186 | *.nupkg 187 | # NuGet Symbol Packages 188 | *.snupkg 189 | # The packages folder can be ignored because of Package Restore 190 | **/[Pp]ackages/* 191 | # except build/, which is used as an MSBuild target. 192 | !**/[Pp]ackages/build/ 193 | # Uncomment if necessary however generally it will be regenerated when needed 194 | #!**/[Pp]ackages/repositories.config 195 | # NuGet v3's project.json files produces more ignorable files 196 | *.nuget.props 197 | *.nuget.targets 198 | 199 | # Microsoft Azure Build Output 200 | csx/ 201 | *.build.csdef 202 | 203 | # Microsoft Azure Emulator 204 | ecf/ 205 | rcf/ 206 | 207 | # Windows Store app package directories and files 208 | AppPackages/ 209 | BundleArtifacts/ 210 | Package.StoreAssociation.xml 211 | _pkginfo.txt 212 | *.appx 213 | *.appxbundle 214 | *.appxupload 215 | 216 | # Visual Studio cache files 217 | # files ending in .cache can be ignored 218 | *.[Cc]ache 219 | # but keep track of directories ending in .cache 220 | !?*.[Cc]ache/ 221 | 222 | # Others 223 | ClientBin/ 224 | ~$* 225 | *~ 226 | *.dbmdl 227 | *.dbproj.schemaview 228 | *.jfm 229 | *.pfx 230 | *.publishsettings 231 | orleans.codegen.cs 232 | 233 | # Including strong name files can present a security risk 234 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 235 | #*.snk 236 | 237 | # Since there are multiple workflows, uncomment next line to ignore bower_components 238 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 239 | #bower_components/ 240 | 241 | # RIA/Silverlight projects 242 | Generated_Code/ 243 | 244 | # Backup & report files from converting an old project file 245 | # to a newer Visual Studio version. Backup files are not needed, 246 | # because we have git ;-) 247 | _UpgradeReport_Files/ 248 | Backup*/ 249 | UpgradeLog*.XML 250 | UpgradeLog*.htm 251 | ServiceFabricBackup/ 252 | *.rptproj.bak 253 | 254 | # SQL Server files 255 | *.mdf 256 | *.ldf 257 | *.ndf 258 | 259 | # Business Intelligence projects 260 | *.rdl.data 261 | *.bim.layout 262 | *.bim_*.settings 263 | *.rptproj.rsuser 264 | *- [Bb]ackup.rdl 265 | *- [Bb]ackup ([0-9]).rdl 266 | *- [Bb]ackup ([0-9][0-9]).rdl 267 | 268 | # Microsoft Fakes 269 | FakesAssemblies/ 270 | 271 | # GhostDoc plugin setting file 272 | *.GhostDoc.xml 273 | 274 | # Node.js Tools for Visual Studio 275 | .ntvs_analysis.dat 276 | node_modules/ 277 | 278 | # Visual Studio 6 build log 279 | *.plg 280 | 281 | # Visual Studio 6 workspace options file 282 | *.opt 283 | 284 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 285 | *.vbw 286 | 287 | # Visual Studio LightSwitch build output 288 | **/*.HTMLClient/GeneratedArtifacts 289 | **/*.DesktopClient/GeneratedArtifacts 290 | **/*.DesktopClient/ModelManifest.xml 291 | **/*.Server/GeneratedArtifacts 292 | **/*.Server/ModelManifest.xml 293 | _Pvt_Extensions 294 | 295 | # Paket dependency manager 296 | .paket/paket.exe 297 | paket-files/ 298 | 299 | # FAKE - F# Make 300 | .fake/ 301 | 302 | # CodeRush personal settings 303 | .cr/personal 304 | 305 | # Python Tools for Visual Studio (PTVS) 306 | __pycache__/ 307 | *.pyc 308 | 309 | # Cake - Uncomment if you are using it 310 | # tools/** 311 | # !tools/packages.config 312 | 313 | # Tabs Studio 314 | *.tss 315 | 316 | # Telerik's JustMock configuration file 317 | *.jmconfig 318 | 319 | # BizTalk build output 320 | *.btp.cs 321 | *.btm.cs 322 | *.odx.cs 323 | *.xsd.cs 324 | 325 | # OpenCover UI analysis results 326 | OpenCover/ 327 | 328 | # Azure Stream Analytics local run output 329 | ASALocalRun/ 330 | 331 | # MSBuild Binary and Structured Log 332 | *.binlog 333 | 334 | # NVidia Nsight GPU debugger configuration file 335 | *.nvuser 336 | 337 | # MFractors (Xamarin productivity tool) working folder 338 | .mfractor/ 339 | 340 | # Local History for Visual Studio 341 | .localhistory/ 342 | 343 | # BeatPulse healthcheck temp database 344 | healthchecksdb 345 | 346 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 347 | MigrationBackup/ 348 | 349 | # Ionide (cross platform F# VS Code tools) working folder 350 | .ionide/ 351 | 352 | # Custom 353 | build 354 | package-lock.json 355 | secrets.json 356 | *.txt 357 | *.zip 358 | .DS_Store 359 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 OSLabs Beta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gimbap 2 | A monolithic architecture is the traditional application design where the whole product is conveniently compiled together and deployed in a single artifact. The approach is simple but becomes problematic as an application scales horizontally. Most successful applications tend to grow in size and complexity. At a high scale, this would result in difficulty in understanding and updating the application. In addition, the size could significantly slow down the start-up time of the application. Since all modules are compiled at once, the monolithic approach is unreliable as a bug in a single module may break the entire application. 3 | 4 | When it comes to working with largely scaled applications, the microservices architecture is more efficient. Splitting a complex application into a smaller set of interconnected services is incredibly impactful. This allows accessible and efficient continuous deployment as each service can be scaled independently. 5 | 6 | It can be laborious and challenging to know exactly how to break up an application effectively. If done incorrectly, the business may end up with high costs and loss of resources. 7 | 8 | With Gimbap, developers simply need to install an NPM package and let the tool handle the rest. First, the Express application object will be modified to allow Gimbap to track endpoint performance. This data will be stored in a database, allowing developers to monitor individual endpoints and view microservice clustering recommendations based upon endpoints’ similar covariant scores. Lastly, the information is presented in easy-to-read dendrograms and line charts. 9 | 10 | 11 | # Installation 12 | ## Integrate Into Monolith 13 | The gimbap npm package is responsible for collecting server response data and sending that data to your MongoDB. This database must be set up before you launch your monolith using gimbap. 14 | 15 | Gimbap is a function that mutates express’ app object to inject route data logging. 16 | 17 | First, install gimbap as a dependency of your monolithic application: 18 | 19 | ``` 20 | npm install -g gimbap 21 | ``` 22 | 23 | Next, import gimbap into your main server file and add call it before any other middleware function: 24 | 25 | ``` 26 | const app: Express = express(); 27 | gimbap(app, 'mongodb', MONGODB_URI); 28 | ``` 29 | ## Visualization Backend 30 | To launch the visualization app, a React front-end with an Express backend, first fork and clone the gimbap git repository. 31 | 32 | Next, in the root directory, create a file called `secrets.json.` This JSON file must contain an object with a property called `MONGODB_URI` whose value is the URI string to connect to your MongoDB instance. 33 | 34 | Note, your MongoDB must be running as a replica set. Use these instructions to convert a local database into a replica set. 35 | 36 | Next, build and launch the backend: 37 | 38 | ``` 39 | npm run build 40 | npm run start-server 41 | ``` 42 | 43 | You can access the backend at localhost:3000. 44 | 45 | Or, in development mode, to launch a Webpack dev server, simply run: 46 | 47 | ``` 48 | npm run dev 49 | ``` 50 | 51 | You can access the backend at localhost:8080. 52 | 53 | # Visualizing Your Data 54 | 55 | ## Cluster Tree 56 | Data is processed through Gimbap’s clustering algorithm and a dendrogram is generated that visualizes endpoint clustering recommendations based on similar covariant scores. 57 | 58 | ## Load Graph 59 | Dive into more in-depth metrics using Gimbap’s load graphs. Developers are able to analyze recommended clusters in addition to monitoring the call times of individual routes within their application. 60 | 61 | # Contributors 62 | - [Angelynn Truong](https://github.com/vngelynn) 63 | - [Khandker Islam](https://github.com/khandkerislam) 64 | - [Miguel Hernandez](https://github.com/miguelh72) 65 | - [Parker Hutcheson](https://github.com/Parker9706) 66 | - [Sebastien Fauque](https://github.com/SebastienFauque) 67 | -------------------------------------------------------------------------------- /__tests__/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "@typescript-eslint/parser", 3 | "parserOptions": { 4 | "ecmaVersion": 2021, 5 | "sourceType": "module" 6 | }, 7 | "plugins": [ 8 | "@typescript-eslint", 9 | "react-hooks" 10 | ], 11 | "extends": [ 12 | "plugin:react/recommended", 13 | "plugin:@typescript-eslint/recommended", 14 | "plugin:react/jsx-runtime", 15 | "plugin:react-hooks/recommended" 16 | ], 17 | "env": { 18 | "es2021": true, 19 | "node": true, 20 | "jest": true, 21 | "browser": true 22 | }, 23 | "rules": { 24 | "react-hooks/rules-of-hooks": "error", 25 | "react-hooks/exhaustive-deps": "warn", 26 | "react/prop-types": "off", 27 | "@typescript-eslint/no-var-requires": 0, 28 | "@typescript-eslint/no-require-imports": 0, 29 | "react/jsx-uses-react": 1, 30 | "quotes": [1, "single", { "avoidEscape": true }], 31 | "semi": 1, 32 | "eol-last": 1 33 | }, 34 | "settings": { 35 | "react": { 36 | "pragma": "React", 37 | "version": "detect" 38 | } 39 | } 40 | } -------------------------------------------------------------------------------- /__tests__/npm/gimbap.test.ts: -------------------------------------------------------------------------------- 1 | import express, { Application } from 'express'; 2 | import request from 'supertest'; 3 | import gimbap from './../../src/npm'; 4 | import { EndpointModel, Endpoint } from './../../src/shared/models/endpointModel'; 5 | 6 | xdescribe('gimbap logs route request to MongoDB', () => { 7 | let app: Application; 8 | 9 | beforeAll(async () => { 10 | app = express(); 11 | 12 | await gimbap(app, 'mongodb', 'mongodb://localhost:27017/gimbap-test'); 13 | 14 | app.get('/', (req, res) => res.send('Hello World!')); 15 | app.get('/api/user', (req, res) => res.send('Miguel')); 16 | app.get('/api/message', (req, res) => res.send('TDD!')); 17 | }); 18 | 19 | afterAll(async () => { 20 | await EndpointModel.deleteMany({}); 21 | return gimbap.stop(); 22 | }); 23 | 24 | beforeEach(async () => { 25 | await EndpointModel.deleteMany({}); 26 | }); 27 | 28 | test('single endpoint correctly logged to database', async () => { 29 | const callTime: number = Date.now(); 30 | 31 | return request(app) 32 | .get('/') 33 | .expect(200) 34 | .then(async (response) => { 35 | expect(response.text).toBe('Hello World!'); 36 | 37 | const endpoints: Endpoint[] = await EndpointModel.find({}); 38 | expect(endpoints).toHaveLength(1); 39 | expect(endpoints[0]).toMatchObject({ method: 'GET', endpoint: '/' }); 40 | expect(endpoints[0].callTime - callTime).toBeLessThan(100); // allow 100 ms difference 41 | }); 42 | }); 43 | 44 | test('multiple endpoints correctly logged to database', async () => { 45 | const routes = ['/', '/api/user', '/api/message']; 46 | 47 | return Promise.all(routes.map(endpoint => { 48 | return request(app) 49 | .get(endpoint) 50 | .expect(200); 51 | })).then(async () => { 52 | const endpoints: Endpoint[] = await EndpointModel.find({}); 53 | expect(endpoints).toHaveLength(3); 54 | expect(endpoints).toMatchObject(routes.map(route => ({ method: 'GET', endpoint: route }))); 55 | }); 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /__tests__/server/models/clusterModel.test.ts: -------------------------------------------------------------------------------- 1 | import { delay } from '../../testUtils'; 2 | 3 | import { connect, disconnect } from '../../../src/shared/models/mongoSetup'; 4 | import { 5 | ClusterModel, 6 | startWatchingClusterModel, 7 | stopWatchingClusterModel, 8 | forceUpdate, 9 | getClusters, 10 | } from '../../../src/server/models/clusterModel'; 11 | import { 12 | MIN_NUM_CHANGES_TO_UPDATE, 13 | EndpointBucketsModel, 14 | stopWatchingEndpointModel, 15 | startWatchingEndpointModel, 16 | forceAllPendingUpdates 17 | } from './../../../src/server/models/endpointBucketsModel'; 18 | import { EndpointModel, logAllEndpoints, Endpoint } from '../../../src/shared/models/endpointModel'; 19 | import { simulateServerResponses, EndpointPDF, DistributionFunction } from '../../../src/shared/utils/dataGenerator'; 20 | import { Cluster } from '../../../src/shared/types'; 21 | 22 | // Note to user: to make this test work, follow instructions here to convert your database to a replica set 23 | // https://docs.mongodb.com/manual/tutorial/convert-standalone-to-replica-set/ 24 | 25 | 26 | jest.setTimeout(10 * 1000); 27 | 28 | 29 | describe('ClusterModel tests', () => { 30 | beforeAll(async () => { 31 | await connect('mongodb://localhost:27017/gimbap-test'); 32 | }); 33 | 34 | afterAll(async () => { 35 | await ClusterModel.deleteMany({}); 36 | await EndpointBucketsModel.deleteMany({}); 37 | await EndpointModel.deleteMany({}); 38 | stopWatchingClusterModel(); 39 | await stopWatchingEndpointModel(); 40 | return await disconnect(); 41 | }); 42 | 43 | beforeEach(async () => { 44 | stopWatchingClusterModel(); 45 | await stopWatchingEndpointModel(); 46 | startWatchingEndpointModel(); 47 | startWatchingClusterModel(); 48 | }); 49 | 50 | afterEach(async () => { 51 | jest.useRealTimers(); 52 | await ClusterModel.deleteMany({}); 53 | await EndpointBucketsModel.deleteMany({}); 54 | await EndpointModel.deleteMany({}); 55 | }); 56 | 57 | describe('Test storing cluster model', () => { 58 | test('Waiting less than the required timeout should not trigger update functionality', async () => { 59 | let result = await ClusterModel.find({}); 60 | expect(result).toHaveLength(0); // sanity check 61 | const result2 = await EndpointBucketsModel.find({}); 62 | expect(result2).toHaveLength(0); // sanity check 63 | 64 | // Date at beginning of day will add all calls to bucket at index 0. 65 | const callTime: number = new Date(new Date().toDateString()).getTime(); 66 | const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE }, () => ({ method: 'GET', endpoint: '/test', callTime })); 67 | await logAllEndpoints(endpoints); 68 | await delay(200); 69 | 70 | await forceAllPendingUpdates(); // endpoint buckets 71 | 72 | result = await ClusterModel.find({}); 73 | // expect no update to have occurred 74 | expect(result).toHaveLength(0); 75 | }); 76 | 77 | test('Forcing update should trigger update functionality', async () => { 78 | // Date at beginning of day will add all calls to bucket at index 0. 79 | const callTime: number = new Date(new Date().toDateString()).getTime(); 80 | const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE }, () => ({ method: 'GET', endpoint: '/test', callTime })); 81 | await logAllEndpoints(endpoints); 82 | await delay(200); 83 | 84 | await forceAllPendingUpdates(); // endpoint buckets 85 | 86 | await forceUpdate(); // cluster model 87 | 88 | const result = await ClusterModel.find({}); 89 | // expect no update to have occurred 90 | expect(result).toHaveLength(1); 91 | expect(result[0].clusters).toBeInstanceOf(Array); 92 | expect(result[0].clusters).toHaveLength(1); 93 | expect(result[0].clusters[0]).toMatchObject([{ method: 'GET', endpoint: '/test' }]); 94 | }); 95 | 96 | test('Should cluster endpoints correctly', async () => { 97 | // Only weakly testing this here, full testing of clustering algorithm is done in endpoints.test.ts 98 | 99 | const endpointsPDF: EndpointPDF[] = [ 100 | { method: 'GET', endpoint: '/api/1', pdf: () => 1 / 24 }, 101 | { method: 'GET', endpoint: '/api/2', pdf: (x) => x / 24 }, 102 | { method: 'GET', endpoint: '/api/3', pdf: (x) => x / 24 }, 103 | { method: 'POST', endpoint: '/api/4', pdf: () => 1 / 24 } 104 | ]; 105 | const callDist: DistributionFunction = () => 100; 106 | const endpoints: Endpoint[] = simulateServerResponses(endpointsPDF, callDist, 2); 107 | 108 | await logAllEndpoints(endpoints); 109 | await delay(200); 110 | 111 | await forceAllPendingUpdates(); // endpoint buckets 112 | 113 | await forceUpdate(); // cluster model 114 | 115 | const result = await ClusterModel.find({}); 116 | 117 | // expect no update to have occurred 118 | expect(result).toHaveLength(1); 119 | expect(result[0].clusters).toBeInstanceOf(Array); 120 | 121 | const clusters: Cluster[] = result[0].clusters.map(dbCluster => dbCluster.map(dbRoute => ({ method: dbRoute.method, endpoint: dbRoute.endpoint }))); 122 | expect(clusters).toHaveLength(2); 123 | // no guarantee on return clustering order 124 | if (clusters[0][1].endpoint === '/api/1' || clusters[0][1].endpoint === '/api/4') { 125 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/1' }); 126 | expect(clusters[0]).toContainEqual({ method: 'POST', endpoint: '/api/4' }); 127 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/2' }); 128 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/3' }); 129 | } else { 130 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/1' }); 131 | expect(clusters[1]).toContainEqual({ method: 'POST', endpoint: '/api/4' }); 132 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/2' }); 133 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/3' }); 134 | } 135 | }); 136 | 137 | // TODO figure out how to mock the timer to test the time based update 138 | }); 139 | 140 | 141 | describe('Test retrieving cluster model', () => { 142 | test('Retrieve buckets for an empty database', async () => { 143 | const result: Cluster[] | null = await getClusters(); 144 | expect(result).toBeNull(); 145 | }); 146 | 147 | test('Retrieve clusters for a filled database', async () => { 148 | // Date at beginning of day will add all calls to bucket at index 0. 149 | const callTime: number = new Date(new Date().toDateString()).getTime(); 150 | const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE }, () => ({ method: 'GET', endpoint: '/test', callTime })); 151 | await logAllEndpoints(endpoints); 152 | await delay(200); 153 | 154 | await forceAllPendingUpdates(); // endpoint buckets 155 | 156 | await forceUpdate(); // cluster model 157 | 158 | const result: Cluster[] | null = await getClusters(); 159 | 160 | expect(result).toBeInstanceOf(Array); 161 | expect(result).toHaveLength(1); 162 | expect(result[0]).toMatchObject([{ method: 'GET', endpoint: '/test' }]); 163 | }); 164 | 165 | test('Test retrieving clusters before an update has occurred. An update should be forced when retrieval is attempted on a database without cluster model.', async () => { 166 | // Date at beginning of day will add all calls to bucket at index 0. 167 | const callTime: number = new Date(new Date().toDateString()).getTime(); 168 | const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE }, () => ({ method: 'GET', endpoint: '/test', callTime })); 169 | await logAllEndpoints(endpoints); 170 | await delay(200); 171 | 172 | await forceAllPendingUpdates(); // endpoint buckets 173 | 174 | const result: Cluster[] | null = await getClusters(); 175 | 176 | expect(result).toBeInstanceOf(Array); 177 | expect(result).toHaveLength(1); 178 | expect(result[0]).toMatchObject([{ method: 'GET', endpoint: '/test' }]); 179 | }); 180 | }); 181 | }); 182 | -------------------------------------------------------------------------------- /__tests__/server/models/endpointBucketsModel.test.ts: -------------------------------------------------------------------------------- 1 | import { delay } from '../../testUtils'; 2 | 3 | import { connect, disconnect } from './../../../src/shared/models/mongoSetup'; 4 | import { 5 | EndpointBucketsModel, 6 | EndpointBuckets, 7 | startWatchingEndpointModel, 8 | stopWatchingEndpointModel, 9 | getEndpointBuckets, 10 | getAllEndpointBuckets, 11 | getDistinctRoutes, 12 | forceAllPendingUpdates, 13 | MIN_NUM_CHANGES_TO_UPDATE, 14 | NUM_DAILY_DIVISIONS, 15 | } from './../../../src/server/models/endpointBucketsModel'; 16 | import { EndpointModel, logEndpoint, logAllEndpoints, Endpoint } from './../../../src/shared/models/endpointModel'; 17 | import { Route } from './../../../src/shared/types'; 18 | 19 | // Note to user: to make this test work, follow instructions here to convert your database to a replica set 20 | // https://docs.mongodb.com/manual/tutorial/convert-standalone-to-replica-set/ 21 | 22 | describe('EndpointBuckets tests', () => { 23 | beforeAll(async () => { 24 | await connect('mongodb://localhost:27017/gimbap-test'); 25 | }); 26 | 27 | afterAll(async () => { 28 | await EndpointModel.deleteMany({}); 29 | await EndpointBucketsModel.deleteMany({}); 30 | await stopWatchingEndpointModel(); 31 | return await disconnect(); 32 | }); 33 | 34 | beforeEach(async () => { 35 | await stopWatchingEndpointModel(); 36 | startWatchingEndpointModel(); 37 | }); 38 | 39 | afterEach(async () => { 40 | jest.useRealTimers(); 41 | await EndpointModel.deleteMany({}); 42 | await EndpointBucketsModel.deleteMany({}); 43 | }); 44 | 45 | describe('Test storing endpoint buckets', () => { 46 | test('Updating less than the required number of changes should not trigger update functionality', async () => { 47 | let result = await EndpointBucketsModel.find({}); 48 | expect(result).toHaveLength(0); // sanity check 49 | 50 | // add one less than needed number of change events to trigger update 51 | for (let i = 0; i < MIN_NUM_CHANGES_TO_UPDATE - 1; i++) { 52 | await logEndpoint('GET', '/test', Date.now()); 53 | } 54 | 55 | result = await EndpointBucketsModel.find({}); 56 | // expect no update to have occurred 57 | expect(result).toHaveLength(0); 58 | }); 59 | 60 | test('Updating more than the required number of changes should trigger update functionality', async () => { 61 | let result: EndpointBuckets[] = await EndpointBucketsModel.find({}); 62 | expect(result).toHaveLength(0); // sanity check 63 | 64 | // Date at beginning of day will add all calls to bucket at index 0. 65 | const callTime: number = new Date(new Date().toDateString()).getTime(); 66 | 67 | const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE }, () => ({ method: 'GET', endpoint: '/test', callTime })); 68 | await logAllEndpoints(endpoints); 69 | 70 | await delay(200); 71 | 72 | result = await EndpointBucketsModel.find({}); 73 | // expect no update to have occurred 74 | expect(result).toHaveLength(1); 75 | expect(result[0].buckets).toHaveLength(NUM_DAILY_DIVISIONS); 76 | expect(result[0].buckets[0]).toBe(MIN_NUM_CHANGES_TO_UPDATE); 77 | }); 78 | 79 | test('Multiple updates correctly updates endpoint bucket', async () => { 80 | const numUpdates = 3; 81 | // Date at beginning of day will add all calls to bucket at index 0. 82 | const callTime: number = new Date(new Date().toDateString()).getTime(); 83 | 84 | const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE }, () => ({ method: 'GET', endpoint: '/test', callTime })); 85 | for (let i = 0; i < numUpdates; i++) { 86 | await logAllEndpoints(endpoints); 87 | } 88 | 89 | await delay(200); 90 | 91 | const result = await EndpointBucketsModel.find({}); 92 | 93 | // expect no update to have occurred 94 | expect(result).toHaveLength(1); 95 | expect(result[0].buckets).toHaveLength(NUM_DAILY_DIVISIONS); 96 | expect(result[0].buckets[0]).toBe(MIN_NUM_CHANGES_TO_UPDATE * numUpdates); 97 | }); 98 | 99 | // TODO figure out how to mock the timer to test the time based update 100 | // test('Test storing endpoint buckets because 5 minute timeout has occurred, even before enough data entry events have caused an update', async () => { 101 | // // Date at beginning of day will add all calls to bucket at index 0. 102 | // const callTime: number = new Date(new Date().toDateString()).getTime(); 103 | // await stopWatchingEndpointModel(); 104 | 105 | // jest.useFakeTimers('legacy'); 106 | // Promise.resolve().then(() => jest.advanceTimersByTime(5 * 60 * 1000)); // because jest timer mocks are broken https://stackoverflow.com/questions/51126786/jest-fake-timers-with-promises 107 | 108 | // startWatchingEndpointModel(); 109 | 110 | // const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE - 1 }, () => ({ method: 'GET', endpoint: '/test', callTime })); 111 | // await logAllEndpoints(endpoints); 112 | // await delay(200); 113 | 114 | // jest.spyOn(global, 'setTimeout'); 115 | 116 | // const inDatabase: EndpointBuckets[] = await EndpointBucketsModel.find({}); 117 | // expect(inDatabase).toHaveLength(0); // sanity check 118 | 119 | // const result: EndpointBuckets[] = await EndpointBucketsModel.find({}); 120 | // // expect no update to have occurred 121 | // expect(result).toHaveLength(1); 122 | // expect(result[0].buckets).toHaveLength(NUM_DAILY_DIVISIONS); 123 | // expect(result[0].buckets[0]).toBe(MIN_NUM_CHANGES_TO_UPDATE - 1); 124 | // }); 125 | }); 126 | 127 | 128 | describe('Test retrieving endpoint buckets', () => { 129 | test('Retrieve buckets for an empty database', async () => { 130 | const result: EndpointBuckets = await getEndpointBuckets('GET', '/test'); 131 | expect(result).toBeNull(); 132 | }); 133 | 134 | test('Retrieve buckets for a filled database', async () => { 135 | // Date at beginning of day will add all calls to bucket at index 0. 136 | const callTime: number = new Date(new Date().toDateString()).getTime(); 137 | 138 | const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE - 1 }, () => ({ method: 'GET', endpoint: '/test', callTime })); 139 | await logAllEndpoints(endpoints); 140 | await logEndpoint('GET', '/test', callTime + 1); 141 | await delay(200); 142 | 143 | const result: EndpointBuckets = await getEndpointBuckets('GET', '/test'); 144 | // expect no update to have occurred 145 | expect(result.method).toBe('GET'); 146 | expect(result.endpoint).toBe('/test'); 147 | expect(result.oldestDate).toBe(callTime); 148 | expect(result.newestDate).toBe(callTime + 1); 149 | expect(result.buckets).toHaveLength(NUM_DAILY_DIVISIONS); 150 | expect(result.buckets[0]).toBe(MIN_NUM_CHANGES_TO_UPDATE); 151 | }); 152 | 153 | test('Test retrieving endpoint buckets before an update has occurred. An update should be forced when retrieval is attempted on an empty database.', async () => { 154 | // Date at beginning of day will add all calls to bucket at index 0. 155 | const callTime: number = new Date(new Date().toDateString()).getTime(); 156 | 157 | const endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE - 1 }, () => ({ method: 'GET', endpoint: '/test', callTime })); 158 | await logAllEndpoints(endpoints); 159 | await delay(200); 160 | 161 | const inDatabase: EndpointBuckets[] = await EndpointBucketsModel.find({}); 162 | expect(inDatabase).toHaveLength(0); // sanity check 163 | 164 | const result: EndpointBuckets = await getEndpointBuckets('GET', '/test'); 165 | // expect no update to have occurred 166 | expect(result.buckets).toHaveLength(NUM_DAILY_DIVISIONS); 167 | expect(result.buckets[0]).toBe(MIN_NUM_CHANGES_TO_UPDATE - 1); 168 | }); 169 | 170 | test('Test retrieving all endpoint buckets from the database.', async () => { 171 | const method1 = 'GET', method2 = 'POST'; 172 | const endpoint1 = '/api/1', endpoint2 = '/api/2'; 173 | // Date at beginning of day will add all calls to bucket at index 0. 174 | const callTime: number = new Date(new Date().toDateString()).getTime(); 175 | 176 | let endpoints: Endpoint[] = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE }, () => ({ method: method1, endpoint: endpoint1, callTime })); 177 | await logAllEndpoints(endpoints); 178 | await delay(200); 179 | 180 | endpoints = Array.from({ length: MIN_NUM_CHANGES_TO_UPDATE }, () => ({ method: method2, endpoint: endpoint2, callTime })); 181 | await logAllEndpoints(endpoints); 182 | await delay(200); 183 | 184 | const result: EndpointBuckets[] = await getAllEndpointBuckets(); 185 | expect(result).toHaveLength(2); 186 | expect(result[0].buckets).toHaveLength(NUM_DAILY_DIVISIONS); 187 | expect(result[0].buckets[0]).toBe(MIN_NUM_CHANGES_TO_UPDATE); 188 | expect(result[1].buckets).toHaveLength(NUM_DAILY_DIVISIONS); 189 | expect(result[1].buckets[0]).toBe(MIN_NUM_CHANGES_TO_UPDATE); 190 | }); 191 | 192 | test('Test retrieving all endpoint buckets from an empty database.', async () => { 193 | const result: EndpointBuckets[] = await getAllEndpointBuckets(); 194 | expect(result).toHaveLength(0); 195 | }); 196 | 197 | test('Get distinct endpoints', async () => { 198 | const getApi1: Endpoint[] = [ 199 | { method: 'GET', endpoint: 'api/1', callTime: 1 }, 200 | { method: 'GET', endpoint: 'api/1', callTime: 2 }, 201 | { method: 'GET', endpoint: 'api/1', callTime: 3 }, 202 | ]; 203 | const deleteApi1: Endpoint[] = [ 204 | { method: 'DELETE', endpoint: 'api/1', callTime: 4 }, 205 | ]; 206 | const getApi2: Endpoint[] = [ 207 | { method: 'GET', endpoint: 'api/2', callTime: 5 }, 208 | { method: 'GET', endpoint: 'api/2', callTime: 6 }, 209 | ]; 210 | const postApi2: Endpoint[] = [ 211 | { method: 'POST', endpoint: 'api/2', callTime: 7 }, 212 | { method: 'POST', endpoint: 'api/2', callTime: 8 }, 213 | ]; 214 | 215 | await logAllEndpoints([...getApi1, ...deleteApi1, ...getApi2, ...postApi2]); 216 | await delay(200); 217 | 218 | await forceAllPendingUpdates(); 219 | 220 | const result: Route[] = await getDistinctRoutes(); 221 | 222 | expect(result).toHaveLength(4); 223 | expect(result).toMatchObject([ 224 | { method: 'GET', endpoint: 'api/1' }, 225 | { method: 'DELETE', endpoint: 'api/1' }, 226 | { method: 'GET', endpoint: 'api/2' }, 227 | { method: 'POST', endpoint: 'api/2' }, 228 | ]); 229 | }); 230 | }); 231 | }); 232 | -------------------------------------------------------------------------------- /__tests__/server/utils/endpoints.test.ts: -------------------------------------------------------------------------------- 1 | import { getLoadData, determineClusters } from '../../../src/server/utils/endpoints'; 2 | import { EndpointModel, Endpoint } from '../../../src/shared/models/endpointModel'; 3 | import { calculateEndpointBuckets, EndpointBuckets, stopWatchingEndpointModel } from './../../../src/server/models/endpointBucketsModel'; 4 | import { connect, disconnect } from '../../../src/shared/models/mongoSetup'; 5 | import { simulateServerResponses, EndpointPDF, DistributionFunction } from '../../../src/shared/utils/dataGenerator'; 6 | import { Cluster, LoadData } from '../../../src/shared/types'; 7 | 8 | 9 | describe('Generate LoadData from array of endpoints', () => { 10 | test('Should return empty array if an empty array is passed in', () => { 11 | const result = getLoadData({ 12 | method: 'GET', 13 | endpoint: '/test', 14 | buckets: [], 15 | lastEndpointId: 1, 16 | oldestDate: 1, 17 | newestDate: 1, 18 | }); 19 | expect(result).toBeInstanceOf(Array); 20 | expect(result).toHaveLength(0); 21 | }); 22 | 23 | test('Should return array with correct time division', () => { 24 | const granularity = 30; 25 | // Date at beginning of day will add all calls to bucket at index 0. 26 | const callTime: number = new Date(new Date().toDateString()).getTime(); 27 | const nextDayCallTime: Date = new Date(callTime); 28 | nextDayCallTime.setDate(nextDayCallTime.getDate() + 2); 29 | 30 | const buckets = Array.from({ length: (24 * 60) / granularity }, () => 1); 31 | buckets[0] = 1; 32 | const result: LoadData = getLoadData({ 33 | method: 'GET', 34 | endpoint: '/test', 35 | buckets, 36 | lastEndpointId: 1, 37 | oldestDate: callTime, 38 | newestDate: nextDayCallTime.getTime(), 39 | }, granularity); 40 | 41 | expect(result).toBeInstanceOf(Array); 42 | expect(result).toHaveLength((24 * 60) / granularity); 43 | result.forEach(([, value]) => expect(value).toBe(0.5)); 44 | }); 45 | }); 46 | 47 | describe('Correctness of clustering algorithm using step function pdf', () => { 48 | beforeAll(async () => await connect('mongodb://localhost:27017/gimbap-test')); 49 | afterAll(async () => { 50 | await stopWatchingEndpointModel(); 51 | await disconnect(); 52 | }); 53 | 54 | beforeEach(async () => { 55 | await EndpointModel.deleteMany(); 56 | }); 57 | 58 | test('Should return a single cluster for a single endpoint', async () => { 59 | const endpointsPDF: EndpointPDF[] = [ 60 | { method: 'GET', endpoint: '/api', pdf: () => 1 / 24 } 61 | ]; 62 | const callDist: DistributionFunction = () => 10; 63 | const singleEndpointServerResponses: Endpoint[] = simulateServerResponses(endpointsPDF, callDist, 5, 60); 64 | // const buckets = vectorizeEndpoints(endpoints); 65 | 66 | const endpointBuckets: EndpointBuckets = calculateEndpointBuckets(singleEndpointServerResponses); 67 | 68 | const clusters: Cluster[] = determineClusters([endpointBuckets]); 69 | 70 | expect(clusters).toHaveLength(1); 71 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api' }); 72 | }); 73 | 74 | test('Should return that endpoints with same pdf belongs in the same cluster', async () => { 75 | const endpointsPDF: EndpointPDF[] = [ 76 | { method: 'GET', endpoint: '/api/1', pdf: () => 1 / 24 }, 77 | { method: 'GET', endpoint: '/api/2', pdf: (x) => x / 24 }, 78 | { method: 'GET', endpoint: '/api/3', pdf: (x) => x / 24 }, 79 | { method: 'POST', endpoint: '/api/4', pdf: () => 1 / 24 } 80 | ]; 81 | const callDist: DistributionFunction = () => 100; 82 | const singleEndpointServerResponses: Endpoint[] = simulateServerResponses(endpointsPDF, callDist, 5, 60); 83 | 84 | // TODO place this repeated code in a function 85 | // group by routes 86 | const sortedEndpoints: Endpoint[][] = singleEndpointServerResponses.reduce((sorted, endpoint) => { 87 | const index = endpointsPDF.findIndex(pdf => pdf.method === endpoint.method && pdf.endpoint === endpoint.endpoint); 88 | sorted[index].push(endpoint); 89 | return sorted; 90 | }, Array.from({ length: endpointsPDF.length }, () => [])); 91 | 92 | const allEndpointBuckets: EndpointBuckets[] = sortedEndpoints.map(endpoints => calculateEndpointBuckets(endpoints)); 93 | 94 | const clusters: Cluster[] = determineClusters(allEndpointBuckets); 95 | 96 | expect(clusters).toHaveLength(2); 97 | // no guarantee on return clustering order 98 | if (clusters[0][1].endpoint === '/api/1' || clusters[0][1].endpoint === '/api/4') { 99 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/1' }); 100 | expect(clusters[0]).toContainEqual({ method: 'POST', endpoint: '/api/4' }); 101 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/2' }); 102 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/3' }); 103 | } else { 104 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/1' }); 105 | expect(clusters[1]).toContainEqual({ method: 'POST', endpoint: '/api/4' }); 106 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/2' }); 107 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/3' }); 108 | } 109 | }); 110 | 111 | test('Should return that endpoints with similar pdf belongs in the same cluster', async () => { 112 | const endpointsPDF: EndpointPDF[] = [ 113 | { method: 'GET', endpoint: '/api/1', pdf: () => 0.8 / 24 }, 114 | { method: 'GET', endpoint: '/api/2', pdf: (x) => 0.8 * x / 24 }, 115 | { method: 'GET', endpoint: '/api/3', pdf: (x) => x / 24 }, 116 | { method: 'POST', endpoint: '/api/4', pdf: () => 1 / 24 } 117 | ]; 118 | const callDist: DistributionFunction = () => 100; 119 | const singleEndpointServerResponses: Endpoint[] = simulateServerResponses(endpointsPDF, callDist, 5, 60); 120 | 121 | // group by routes 122 | const sortedEndpoints: Endpoint[][] = singleEndpointServerResponses.reduce((sorted, endpoint) => { 123 | const index = endpointsPDF.findIndex(pdf => pdf.method === endpoint.method && pdf.endpoint === endpoint.endpoint); 124 | sorted[index].push(endpoint); 125 | return sorted; 126 | }, Array.from({ length: endpointsPDF.length }, () => [])); 127 | 128 | const allEndpointBuckets: EndpointBuckets[] = sortedEndpoints.map(endpoints => calculateEndpointBuckets(endpoints)); 129 | 130 | const clusters: Cluster[] = determineClusters(allEndpointBuckets); 131 | 132 | expect(clusters).toHaveLength(2); 133 | // no guarantee on return clustering order 134 | if (clusters[0][1].endpoint === '/api/1' || clusters[0][1].endpoint === '/api/4') { 135 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/1' }); 136 | expect(clusters[0]).toContainEqual({ method: 'POST', endpoint: '/api/4' }); 137 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/2' }); 138 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/3' }); 139 | } else { 140 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/1' }); 141 | expect(clusters[1]).toContainEqual({ method: 'POST', endpoint: '/api/4' }); 142 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/2' }); 143 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/3' }); 144 | } 145 | }); 146 | 147 | test('Should return that endpoints with same total calls but different timing pdf belongs in the same cluster', async () => { 148 | function leftStep(x) { 149 | if (x < 12) return 1 / 12; 150 | return 0; 151 | } 152 | function rightStep(x) { 153 | if (x > 12) return 1 / 12; 154 | return 0; 155 | } 156 | 157 | const endpointsPDF: EndpointPDF[] = [ 158 | { method: 'GET', endpoint: '/api/1', pdf: leftStep }, 159 | { method: 'GET', endpoint: '/api/2', pdf: leftStep }, 160 | { method: 'GET', endpoint: '/api/3', pdf: rightStep }, 161 | { method: 'POST', endpoint: '/api/4', pdf: rightStep } 162 | ]; 163 | const callDist: DistributionFunction = () => 100; 164 | const singleEndpointServerResponses: Endpoint[] = simulateServerResponses(endpointsPDF, callDist, 5, 60); 165 | 166 | // group by routes 167 | const sortedEndpoints: Endpoint[][] = singleEndpointServerResponses.reduce((sorted, endpoint) => { 168 | const index = endpointsPDF.findIndex(pdf => pdf.method === endpoint.method && pdf.endpoint === endpoint.endpoint); 169 | sorted[index].push(endpoint); 170 | return sorted; 171 | }, Array.from({ length: endpointsPDF.length }, () => [])); 172 | 173 | const allEndpointBuckets: EndpointBuckets[] = sortedEndpoints.map(endpoints => calculateEndpointBuckets(endpoints)); 174 | 175 | const clusters: Cluster[] = determineClusters(allEndpointBuckets); 176 | 177 | expect(clusters).toHaveLength(2); 178 | // no guarantee on return clustering order 179 | if (clusters[0][1].endpoint === '/api/1' || clusters[0][1].endpoint === '/api/2') { 180 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/1' }); 181 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/2' }); 182 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/3' }); 183 | expect(clusters[1]).toContainEqual({ method: 'POST', endpoint: '/api/4' }); 184 | } else { 185 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/1' }); 186 | expect(clusters[1]).toContainEqual({ method: 'GET', endpoint: '/api/2' }); 187 | expect(clusters[0]).toContainEqual({ method: 'GET', endpoint: '/api/3' }); 188 | expect(clusters[0]).toContainEqual({ method: 'POST', endpoint: '/api/4' }); 189 | } 190 | }); 191 | 192 | test('Should return that endpoints with similar pdf belongs in the same cluster, even if just one cluster. No need for microservice result.', async () => { 193 | const endpointsPDF: EndpointPDF[] = [ 194 | { method: 'GET', endpoint: '/api/1', pdf: () => 1 }, 195 | { method: 'GET', endpoint: '/api/2', pdf: () => 0.9 }, 196 | { method: 'GET', endpoint: '/api/3', pdf: () => 1.2 }, 197 | { method: 'POST', endpoint: '/api/4', pdf: () => 0.85 } 198 | ]; 199 | const callDist: DistributionFunction = () => 100; 200 | const singleEndpointServerResponses: Endpoint[] = simulateServerResponses(endpointsPDF, callDist, 5, 60); 201 | 202 | // group by routes 203 | const sortedEndpoints: Endpoint[][] = singleEndpointServerResponses.reduce((sorted, endpoint) => { 204 | const index = endpointsPDF.findIndex(pdf => pdf.method === endpoint.method && pdf.endpoint === endpoint.endpoint); 205 | sorted[index].push(endpoint); 206 | return sorted; 207 | }, Array.from({ length: endpointsPDF.length }, () => [])); 208 | 209 | const allEndpointBuckets: EndpointBuckets[] = sortedEndpoints.map(endpoints => calculateEndpointBuckets(endpoints)); 210 | 211 | const clusters: Cluster[] = determineClusters(allEndpointBuckets); 212 | 213 | expect(clusters).toHaveLength(1); 214 | }); 215 | 216 | // TODO thoroughly test clustering results to different types of biasing pdf functions 217 | }); 218 | -------------------------------------------------------------------------------- /__tests__/server/utils/loadDePaulEndpointData.test.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs/promises'; 2 | import path from 'path'; 3 | 4 | import loadDePaulEndpointData from './../../../src/server/utils/loadDePaulEndpointData'; 5 | import { EndpointModel, Endpoint } from './../../../src/shared/models/endpointModel'; 6 | import { startWatchingEndpointModel, EndpointBucketsModel, forceAllPendingUpdates } from './../../../src/server/models/endpointBucketsModel'; 7 | import { startWatchingClusterModel, forceUpdate } from '../../../src/server/models/clusterModel'; 8 | import { connect, disconnect } from '../../../src/shared/models/mongoSetup'; 9 | import { delay } from './../../testUtils'; 10 | 11 | import { MONGODB_URI } from './../../../src/server/secrets.json'; 12 | 13 | xdescribe('Populate database with DePaul CTI data and verify data is in DB', () => { 14 | jest.setTimeout(60 * 60 * 1000); 15 | 16 | beforeAll(async () => { 17 | await connect(MONGODB_URI); 18 | await EndpointModel.deleteMany({}); 19 | await EndpointBucketsModel.deleteMany({}); 20 | startWatchingEndpointModel(); 21 | startWatchingClusterModel(); 22 | }); 23 | 24 | afterAll(async () => { 25 | await delay(60 * 1000); // wait a minute for update calls to finish, increase if you got errors at the end 26 | await forceAllPendingUpdates(); 27 | await forceUpdate(); 28 | await disconnect(); 29 | }); 30 | 31 | test('Check for a specific data point', async () => { 32 | const batchSize = 5000; 33 | 34 | await loadDePaulEndpointData(batchSize); 35 | 36 | const data = await fs.readFile(path.resolve('cti-april02-log.txt'), 'utf8'); 37 | const entries: string[] = data.split('\n').slice(4); 38 | 39 | for (let i = 0; i < entries.length; i += batchSize) { 40 | const entry: string[] = data.split('\n').slice(4); 41 | const timeStamp: number = new Date(entry[0] + ' ' + entry[1]).getTime(); 42 | const expected: Endpoint = { 43 | method: entry[8], 44 | endpoint: entry[9], 45 | callTime: timeStamp, 46 | }; 47 | 48 | if (isNaN(timeStamp) || !entry[8] || !entry[9]) continue; 49 | 50 | const check: boolean = await EndpointModel.exists(expected); 51 | expect(check).toBe(true); // expected endpoint to be in DB 52 | } 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /__tests__/shared/models/endpointModels.test.ts: -------------------------------------------------------------------------------- 1 | import { connect, disconnect } from './../../../src/shared/models/mongoSetup'; 2 | import { EndpointModel, Endpoint, logEndpoint, getAllEndpoints } from './../../../src/shared/models/endpointModel'; 3 | 4 | describe('Test storing endpoints', () => { 5 | beforeAll(async () => { 6 | await connect('mongodb://localhost:27017/gimbap-test'); 7 | }); 8 | 9 | afterAll(async () => { 10 | await EndpointModel.deleteMany({}); 11 | return await disconnect(); 12 | }); 13 | 14 | beforeEach(async () => { 15 | await EndpointModel.deleteMany({}); 16 | }); 17 | 18 | test('Log a single endpoint to database', async () => { 19 | const method = 'GET'; 20 | const endpoint = 'api/fun'; 21 | const callTime: number = Date.now(); 22 | 23 | await logEndpoint(method, endpoint, callTime); 24 | 25 | const result: Endpoint[] = await EndpointModel.find({}); 26 | expect(result).toHaveLength(1); 27 | expect(result[0]).toMatchObject({ method, endpoint, callTime }); 28 | }); 29 | 30 | test('Log a multiple endpoint to database', async () => { 31 | const methods: string[] = ['GET', 'POST', 'DELETE']; 32 | const endpoints: string[] = ['api/1', 'api/2', 'api/3']; 33 | const callTimes: number[] = [new Date('1/1/01').getTime(), new Date('1/2/01').getTime(), new Date('1/3/01').getTime()]; 34 | 35 | for (let i = 0; i < methods.length; i++) 36 | await logEndpoint(methods[i], endpoints[i], callTimes[i]); 37 | 38 | const result = await EndpointModel.find({}); 39 | 40 | expect(result).toHaveLength(methods.length); 41 | for (let i = 0; i < methods.length; i++) 42 | expect(result[i]).toMatchObject({ method: methods[i], endpoint: endpoints[i], callTime: callTimes[i] }); 43 | }); 44 | }); 45 | 46 | describe('Test retrieving endpoints', () => { 47 | beforeAll(async () => { 48 | await connect('mongodb://localhost:27017/gimbap-test'); 49 | }); 50 | 51 | afterAll(async () => { 52 | await EndpointModel.deleteMany({}); 53 | return await disconnect(); 54 | }); 55 | 56 | beforeEach(async () => { 57 | await EndpointModel.deleteMany({}); 58 | }); 59 | 60 | test('Get a list of all endpoints', async () => { 61 | const endpoints: Endpoint[] = [ 62 | { method: 'GET', endpoint: 'api/1', callTime: 1 }, 63 | { method: 'GET', endpoint: 'api/1', callTime: 2 }, 64 | { method: 'GET', endpoint: 'api/1', callTime: 3 }, 65 | { method: 'DELETE', endpoint: 'api/1', callTime: 4 }, 66 | { method: 'GET', endpoint: 'api/2', callTime: 5 }, 67 | { method: 'GET', endpoint: 'api/2', callTime: 6 }, 68 | { method: 'POST', endpoint: 'api/2', callTime: 7 }, 69 | { method: 'POST', endpoint: 'api/2', callTime: 8 }, 70 | ]; 71 | 72 | await EndpointModel.insertMany(endpoints); 73 | 74 | const result: Endpoint[] = await getAllEndpoints(); 75 | expect(result).toHaveLength(endpoints.length); 76 | expect(result).toMatchObject(endpoints); 77 | }); 78 | 79 | test('Get endpoints matching specific method and route', async () => { 80 | const getApi1: Endpoint[] = [ 81 | { method: 'GET', endpoint: 'api/1', callTime: 1 }, 82 | { method: 'GET', endpoint: 'api/1', callTime: 2 }, 83 | { method: 'GET', endpoint: 'api/1', callTime: 3 }, 84 | ]; 85 | const deleteApi1: Endpoint[] = [ 86 | { method: 'DELETE', endpoint: 'api/1', callTime: 4 }, 87 | ]; 88 | const getApi2: Endpoint[] = [ 89 | { method: 'GET', endpoint: 'api/2', callTime: 5 }, 90 | { method: 'GET', endpoint: 'api/2', callTime: 6 }, 91 | ]; 92 | const postApi2: Endpoint[] = [ 93 | { method: 'POST', endpoint: 'api/2', callTime: 7 }, 94 | { method: 'POST', endpoint: 'api/2', callTime: 8 }, 95 | ]; 96 | 97 | await EndpointModel.insertMany([...getApi1, ...deleteApi1, ...getApi2, ...postApi2]); 98 | 99 | for (const api of [getApi1, deleteApi1, getApi2, postApi2]) { 100 | const result: Endpoint[] = await getAllEndpoints(api[0].method, api[0].endpoint); 101 | expect(result).toHaveLength(api.length); 102 | for (let i = 1; i < api.length; i++) { 103 | expect(result[i]).toMatchObject(api[i]); 104 | } 105 | } 106 | }); 107 | }); 108 | -------------------------------------------------------------------------------- /__tests__/shared/utils/dataGenerator.test.ts: -------------------------------------------------------------------------------- 1 | import chiSquaredTest from 'chi-squared-test'; 2 | 3 | import { simulateServerResponses, DistributionFunction, EndpointPDF } from './../../../src/shared/utils/dataGenerator'; 4 | import { Endpoint } from './../../../src/shared/models/endpointModel'; 5 | 6 | describe('Simulate server responses using uniform distributions', () => { 7 | test('For a single endpoint with uniform pdf and uniform call distribution of 1', () => { 8 | const endpointPDF: EndpointPDF = { 9 | method: 'GET', 10 | endpoint: '/api', 11 | pdf: () => 1 / 24, 12 | }; 13 | const callDist: DistributionFunction = () => 1; 14 | const startDayTime: number = new Date(new Date().toLocaleDateString()).getTime(); 15 | 16 | // one interval per hour, with call dist of 1 call per hour 17 | let result: Endpoint[] = simulateServerResponses([endpointPDF], callDist, 1, 60); 18 | 19 | expect(result).toHaveLength(24); 20 | 21 | result.forEach((response: Endpoint, i: number) => { 22 | expect(response).toMatchObject({ method: endpointPDF.method, endpoint: endpointPDF.endpoint }); 23 | expect(response.callTime).toBeGreaterThanOrEqual(startDayTime + i * 60 * 60 * 1000); 24 | expect(response.callTime).toBeLessThanOrEqual(startDayTime + (i + 1) * 60 * 60 * 1000); 25 | }); 26 | 27 | // 4 interval per hour, with call dist of 1 call per hour 28 | // When granularity is too small for dist, number of calls in each interval will be less than one and 29 | // no calls will be returned. 30 | result = simulateServerResponses([endpointPDF], callDist, 1, 15); 31 | expect(result).toHaveLength(0); 32 | 33 | // one interval per hour, with call dist of 1 call per hour, for multiple days 34 | const numDays = 7; 35 | result = simulateServerResponses([endpointPDF], callDist, numDays, 60); 36 | 37 | expect(result).toHaveLength(24 * numDays); 38 | 39 | result.forEach((response: Endpoint, i: number) => { 40 | expect(response).toMatchObject({ method: endpointPDF.method, endpoint: endpointPDF.endpoint }); 41 | expect(response.callTime).toBeGreaterThanOrEqual(startDayTime + i * 60 * 60 * 1000); 42 | expect(response.callTime).toBeLessThanOrEqual(startDayTime + (i + 1) * 60 * 60 * 1000); 43 | }); 44 | }); 45 | 46 | test('For a single endpoint with uniform pdf and large uniform call distribution', () => { 47 | const endpointPDF: EndpointPDF = { 48 | method: 'GET', 49 | endpoint: '/api', 50 | pdf: () => 1 / 24, 51 | }; 52 | const callsPerHour = 100; 53 | const callDist: DistributionFunction = () => callsPerHour; 54 | const startDayTime: number = new Date(new Date().toLocaleDateString()).getTime(); 55 | 56 | // one interval per hour, with call dist of 100 call per hour 57 | const result: Endpoint[] = simulateServerResponses([endpointPDF], callDist, 1, 60); 58 | expect(result).toHaveLength(24 * callsPerHour); 59 | 60 | result.forEach((response: Endpoint, i: number) => { 61 | const j = Math.floor(i / callsPerHour); // since 100 endpoints per hour 62 | 63 | expect(response).toMatchObject({ method: endpointPDF.method, endpoint: endpointPDF.endpoint }); 64 | expect(response.callTime).toBeGreaterThanOrEqual(startDayTime + j * 60 * 60 * 1000); 65 | expect(response.callTime).toBeLessThanOrEqual(startDayTime + (j + 1) * 60 * 60 * 1000); 66 | }); 67 | }); 68 | 69 | test('Works for different granularity', () => { 70 | const endpointPDF: EndpointPDF = { 71 | method: 'GET', 72 | endpoint: '/api', 73 | pdf: () => 1 / 24, 74 | }; 75 | const callsPerHour = 4; 76 | const granularity = 15; 77 | const callDist: DistributionFunction = () => callsPerHour; 78 | const startDayTime: number = new Date(new Date().toLocaleDateString()).getTime(); 79 | 80 | // one interval per hour, with call dist of 100 call per hour 81 | const result: Endpoint[] = simulateServerResponses([endpointPDF], callDist, 1, granularity); 82 | expect(result).toHaveLength(24 * callsPerHour); 83 | 84 | result.forEach((response: Endpoint, i: number) => { 85 | expect(response).toMatchObject({ method: endpointPDF.method, endpoint: endpointPDF.endpoint }); 86 | expect(response.callTime).toBeGreaterThanOrEqual(startDayTime + i * granularity * 60 * 1000); 87 | expect(response.callTime).toBeLessThanOrEqual(startDayTime + (i + 1) * granularity * 60 * 1000); 88 | }); 89 | }); 90 | 91 | test('Returned times in an interval follow a uniform distribution', () => { 92 | const endpoints: EndpointPDF[] = [ 93 | { 94 | method: 'GET', 95 | endpoint: '/api/1', 96 | pdf: () => 1 / 24, 97 | }, 98 | ]; 99 | const numCallsPerInterval = 480; // make it divisible by 24 100 | const callDist: DistributionFunction = () => numCallsPerInterval / 24; 101 | const startDayTime: number = new Date(new Date().toLocaleDateString()).getTime(); 102 | 103 | // single interval day long 104 | const result: Endpoint[] = simulateServerResponses(endpoints, callDist, 1, 24 * 60); 105 | 106 | expect(result).toHaveLength(numCallsPerInterval); 107 | 108 | const numBuckets = 10, observedFrequencies: number[] = []; 109 | const bucketIntervalLength = 24 * 60 * 60 * 1000 / numBuckets; 110 | for (let i = 0; i < numBuckets; i++) { 111 | observedFrequencies[i] = result.filter(endpoint => endpoint.callTime >= startDayTime + i * bucketIntervalLength && endpoint.callTime <= startDayTime + (i + 1) * bucketIntervalLength).length; 112 | } 113 | 114 | const expected: number[] = Array.from({ length: numBuckets }, () => numCallsPerInterval / numBuckets); 115 | 116 | // We use a Chi Squared Test to test probability that interval times are uniformly distributed 117 | const probability = chiSquaredTest(observedFrequencies, expected, 1); 118 | expect(probability.probability).toBeGreaterThan(0.10); 119 | }); 120 | 121 | test('For a multiple endpoint with uniform pdf and uniform call distribution of 1', () => { 122 | const endpoints: EndpointPDF[] = [ 123 | { 124 | method: 'GET', 125 | endpoint: '/api/1', 126 | pdf: () => 1 / 24, 127 | }, 128 | { 129 | method: 'GET', 130 | endpoint: '/api/2', 131 | pdf: () => 1 / 24, 132 | }, 133 | { 134 | method: 'GET', 135 | endpoint: '/api/3', 136 | pdf: () => 1 / 24, 137 | }, 138 | { 139 | method: 'GET', 140 | endpoint: '/api/4', 141 | pdf: () => 1 / 24, 142 | }, 143 | { 144 | method: 'GET', 145 | endpoint: '/api/5', 146 | pdf: () => 1 / 24, 147 | }, 148 | ]; 149 | const callDist: DistributionFunction = () => 1; 150 | const numDays = 100; 151 | 152 | // one interval per hour, with call dist of 1 call per hour 153 | const result: Endpoint[] = simulateServerResponses(endpoints, callDist, numDays, 60); 154 | 155 | expect(result).toHaveLength(24 * numDays); 156 | 157 | // Expect distribution of endpoints to be roughly uniform 158 | const endpointFrequencies: { [key: string]: number } = result.reduce( 159 | (frequencies: { [key: string]: number }, endpoint: Endpoint): { [key: string]: number } => { 160 | frequencies[endpoint.endpoint] ??= 0; 161 | frequencies[endpoint.endpoint]++; 162 | 163 | return frequencies; 164 | }, Object.create(null) 165 | ); 166 | 167 | // We use a Chi Squared Test to test probability that endpoint frequency is uniformly distributed 168 | const observed: number[] = Object.values(endpointFrequencies); 169 | const expected: number[] = Array.from({ length: observed.length }, () => (24 * numDays) / endpoints.length); 170 | 171 | const probability = chiSquaredTest(observed, expected, 1); 172 | expect(probability.probability).toBeGreaterThan(0.10); 173 | }); 174 | }); 175 | -------------------------------------------------------------------------------- /__tests__/testUtils/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Delay for a set number of milliseconds. 3 | * 4 | * @param ms milliseconds to pause for 5 | */ 6 | export async function delay(ms: number): Promise { 7 | return new Promise(resolve => { 8 | setTimeout(resolve, ms); 9 | }); 10 | } 11 | -------------------------------------------------------------------------------- /babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | ["@babel/preset-env", {"targets": {"node": "current"}}], 4 | "@babel/preset-typescript" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gimbap-dev", 3 | "version": "0.0.1", 4 | "private": true, 5 | "description": "", 6 | "scripts": { 7 | "clean": "cross-env rm -rf build", 8 | "build": "cross-env NODE_ENV=production concurrently \"npm run build-client\" \"npm run build-server\" \"npm run build-npm\"", 9 | "build-client": "webpack", 10 | "build-server": "tsc -p ./src/server/tsconfig.json", 11 | "build-npm": "tsc -p ./src/npm/tsconfig.json; cp ./src/npm/package.json ./build/npm/package.json", 12 | "dev": "cross-env NODE_ENV=development concurrently \"webpack serve\" \"nodemon --watch build/server build/server/index.js\" \"npm run watch-build-server\"", 13 | "watch-build-server": "tsc -p ./src/server/tsconfig.json --watch", 14 | "watch-build-npm": "tsc -p ./src/middleware/tsconfig.json --watch", 15 | "start-server": "cross-env NODE_ENV=production node build/server/index.js", 16 | "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --runInBand" 17 | }, 18 | "jest": { 19 | "transform": { 20 | "^.+\\.[t|j]sx?$": "babel-jest" 21 | }, 22 | "modulePathIgnorePatterns": [ 23 | "/testUtils/" 24 | ] 25 | }, 26 | "repository": { 27 | "type": "git", 28 | "url": "git+https://github.com/oslabs-beta/gimbap.git" 29 | }, 30 | "keywords": [ 31 | "monolith", 32 | "microservice", 33 | "express", 34 | "typescript", 35 | "node", 36 | "react", 37 | "MUI", 38 | "gimbap", 39 | "mongodb", 40 | "mongoose" 41 | ], 42 | "author": "Miguel Hernandez (miguelh72@outlook.com), Sebastien Fauque (sbfauque@gmail.com), Angelynn Truong (angelynn.trng@gmail.com), Parker Hutcheson (pdhutcheson@gmail.com)", 43 | "license": "MIT", 44 | "bugs": { 45 | "url": "https://github.com/oslabs-beta/gimbap/issues" 46 | }, 47 | "homepage": "https://github.com/oslabs-beta/gimbap#readme", 48 | "devDependencies": { 49 | "@babel/core": "^7.15.8", 50 | "@babel/plugin-transform-runtime": "^7.15.8", 51 | "@babel/preset-env": "^7.15.8", 52 | "@babel/preset-react": "^7.14.5", 53 | "@babel/preset-typescript": "^7.15.0", 54 | "@babel/runtime": "^7.15.4", 55 | "@types/compression": "^1.7.2", 56 | "@types/density-clustering": "^1.3.0", 57 | "@types/express": "^4.17.13", 58 | "@types/jest": "^27.0.2", 59 | "@types/memory-cache": "^0.2.1", 60 | "@types/react": "^17.0.30", 61 | "@types/react-dom": "^17.0.9", 62 | "@types/react-window": "^1.8.5", 63 | "@types/supertest": "^2.0.11", 64 | "@typescript-eslint/eslint-plugin": "^5.0.0", 65 | "@typescript-eslint/parser": "^5.0.0", 66 | "@visx/visx": "^2.2.1", 67 | "babel-jest": "^27.2.5", 68 | "babel-loader": "^8.2.2", 69 | "chi-squared-test": "^1.1.0", 70 | "concurrently": "^6.3.0", 71 | "cross-env": "^7.0.3", 72 | "css-loader": "^6.4.0", 73 | "css-minimizer-webpack-plugin": "^3.1.1", 74 | "d3-shape": "^3.0.1", 75 | "eslint": "^7.32.0", 76 | "eslint-plugin-react": "^7.26.1", 77 | "eslint-plugin-react-hooks": "^4.2.0", 78 | "file-loader": "^6.2.0", 79 | "html-webpack-plugin": "^5.4.0", 80 | "image-webpack-loader": "^8.0.1", 81 | "jest": "^27.2.5", 82 | "mini-css-extract-plugin": "^2.4.2", 83 | "nodemon": "^2.0.13", 84 | "react-spring": "^9.3.0", 85 | "sass": "^1.43.2", 86 | "sass-loader": "^12.2.0", 87 | "style-loader": "^3.3.0", 88 | "supertest": "^6.1.6", 89 | "ts-node": "^10.4.0", 90 | "typescript": "^4.4.4", 91 | "webpack": "^5.58.2", 92 | "webpack-cli": "^4.9.0", 93 | "webpack-dev-server": "^3.11.2" 94 | }, 95 | "dependencies": { 96 | "@emotion/react": "^11.5.0", 97 | "@emotion/styled": "^11.3.0", 98 | "@mui/icons-material": "^5.0.4", 99 | "@mui/material": "^5.0.4", 100 | "@mui/styled-engine-sc": "^5.0.3", 101 | "compression": "^1.7.4", 102 | "density-clustering": "^1.3.0", 103 | "express": "^4.17.1", 104 | "gimbap": "^0.0.3", 105 | "memory-cache": "^0.2.0", 106 | "mongoose": "^6.0.11", 107 | "mongoose-plugin-autoinc": "^1.1.9", 108 | "normalize.css": "^8.0.1", 109 | "react": "^17.0.2", 110 | "react-code-blocks": "0.0.9-0", 111 | "react-dom": "^17.0.2", 112 | "react-window": "^1.8.6", 113 | "styled-components": "^5.3.3" 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/client/App.scss: -------------------------------------------------------------------------------- 1 | #app { 2 | -webkit-background-size: cover; 3 | -moz-background-size: cover; 4 | -o-background-size: cover; 5 | background-size: cover; 6 | } 7 | -------------------------------------------------------------------------------- /src/client/App.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useCallback, useEffect } from 'react'; 2 | 3 | import { ThemeProvider } from '@mui/material/styles'; 4 | import Stack from '@mui/material/Stack'; 5 | 6 | import { Cluster, Route } from './../../src/shared/types'; 7 | import { Page, SubPage } from './types'; 8 | import { fetchRoutes, fetchClusters } from './utils/ajax'; 9 | import { darkTheme, lightTheme } from './theme'; 10 | import useWindowDimensions from './hooks/useWindowDimensions'; 11 | import NavigationBar from './components/common/NavigationBar'; 12 | import Clusters from './components/clusters/Clusters'; 13 | import Metrics from './components/metrics/Metrics'; 14 | import Documentation from './components/documentation/Documentation'; 15 | 16 | import './App.scss'; 17 | 18 | 19 | export default function App() { 20 | const [useLightTheme, setUseLightTheme] = useState(true); 21 | const { height } = useWindowDimensions(); 22 | 23 | const [isNavBarOpen, setIsNavBarOpen] = React.useState(true); 24 | const [page, setPage] = useState(Page.Clusters); 25 | const [metricSubPage, setMetricSubPage] = useState(SubPage.ClusterLoad); 26 | const [docSubPage, setDocSubPage] = useState(SubPage.None); 27 | 28 | const [clusters, setClusters] = useState(null); 29 | const [routes, setRoutes] = useState(null); 30 | 31 | const showApiDocPage: () => void = useCallback(() => setDocSubPage(SubPage.ApiDoc), [setDocSubPage]); 32 | 33 | 34 | // pre-load routes and clusters 35 | useEffect(() => { 36 | fetchRoutes((routes) => { 37 | routes = routes as Route[]; 38 | routes.sort((r1: Route, r2: Route) => r1.method + r1.endpoint > r2.method + r2.endpoint ? 1 : -1); 39 | setRoutes(routes); 40 | }); 41 | 42 | fetchClusters(setClusters); 43 | }, []); 44 | 45 | return ( 46 |
54 | 55 | 56 | 66 | {page === Page.Clusters && } 67 | {page === Page.Metrics && 68 | 75 | } 76 | {page === Page.Documentation && 77 | 78 | } 79 | 80 | 81 |
82 | ); 83 | } 84 | -------------------------------------------------------------------------------- /src/client/assets/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 404 6 | 7 | 8 | This is the 404 error page🚀🚀🚀🚀🚀🚀🚀🚀🚀🚀. 9 | 10 | -------------------------------------------------------------------------------- /src/client/assets/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | gimbap 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 | 18 | 19 | -------------------------------------------------------------------------------- /src/client/components/clusters/Clusters.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Cluster } from './../../../shared/types'; 4 | import { drawerWidth, closedDrawerWidth } from './../common/NavigationBar'; 5 | import getWindowDimensions from './../../hooks/useWindowDimensions'; 6 | import TreeGraph from './TreeGraph'; 7 | import Splash from './../common/Splash'; 8 | 9 | 10 | export default function Clusters({ 11 | isNavBarOpen, 12 | clusters, 13 | useLightTheme, 14 | }: { 15 | isNavBarOpen: boolean; 16 | clusters: Cluster[] | null; 17 | }) { 18 | 19 | const { width, height } = getWindowDimensions(); 20 | 21 | // TODO adjust as needed I removed percent cuts 22 | return (<> 23 | {clusters === null && } 24 | {clusters !== null && } 25 | ); 26 | } 27 | -------------------------------------------------------------------------------- /src/client/components/clusters/EndpointList.tsx: -------------------------------------------------------------------------------- 1 | import React, { useCallback } from 'react'; 2 | import { FixedSizeList, ListChildComponentProps } from 'react-window'; 3 | 4 | import Box from '@mui/material/Box'; 5 | import ListItem from '@mui/material/ListItem'; 6 | import ListItemButton from '@mui/material/ListItemButton'; 7 | import ListItemText from '@mui/material/ListItemText'; 8 | import Typography from '@mui/material/Typography'; 9 | 10 | export default function EndpointList({ 11 | width, 12 | height, 13 | methodName, 14 | clusterName, 15 | endpointList, 16 | }: { 17 | width: number; 18 | height: number; 19 | methodName: string; 20 | clusterName: string; 21 | endpointList: string[]; 22 | }) { 23 | const renderRow = useCallback(({ index, style }: ListChildComponentProps) => { 24 | return ( 25 | 26 | 27 | {endpointList.length === 0 ? 'No endpoints to show...' : endpointList[index]}} /> 28 | 29 | 30 | ); 31 | }, [endpointList]); 32 | 33 | return ( 34 | 46 | 47 | {endpointList.length ? `${clusterName} - ${methodName} - ${endpointList.length} ` : null} 48 | 49 | 50 | 57 | {renderRow} 58 | 59 | 60 | ); 61 | } 62 | -------------------------------------------------------------------------------- /src/client/components/clusters/TreeGraph.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | 3 | import { Group } from '@visx/group'; 4 | import { hierarchy, Tree } from '@visx/hierarchy'; 5 | import { LinearGradient } from '@visx/gradient'; 6 | import { LinkHorizontal, } from '@visx/shape'; 7 | import Stack from '@mui/material/Stack'; 8 | 9 | import { TreeNode } from './../../../shared/types'; 10 | import { fetchClusterTree } from '../../utils/ajax'; 11 | import EndpointList from './EndpointList'; 12 | import Splash from './../common/Splash'; 13 | // import { useLightTheme } from '' 14 | 15 | 16 | const defaultMargin = { top: 30, left: 30, right: 30, bottom: 70 }; // TODO reconciliate this with component Tree attribute size 17 | 18 | export default function TreeGraph({ 19 | width, 20 | height, 21 | margin = defaultMargin, 22 | useLightTheme, 23 | setUseLightTheme, 24 | }: { 25 | width: number; 26 | height: number; 27 | margin?: { top: number; right: number; bottom: number; left: number }; 28 | }) { 29 | const [trees, setTreeGraphData] = useState(null); 30 | const [endpoints, setEndPoints] = useState<{ clusterName: string, methodName: string, endpointList: string[] }>({ clusterName: 'No cluster selected', methodName: 'No method selected', endpointList: [] }); 31 | 32 | const treeWidth = (width * 0.6) - (24 * 3); 33 | const treeHeight = height - (24 * 2); 34 | 35 | // Fetch request zone 36 | useEffect(() => { 37 | fetchClusterTree(setTreeGraphData); 38 | }, []); 39 | 40 | return (<> 41 | {trees === null && } 42 | {trees !== null && 43 | 44 | 45 | 46 | 47 | 48 | d.children)} 50 | size={[treeHeight - 56, treeWidth - 35]} 51 | separation={(a, b) => (a.parent === b.parent ? 2 : 100) / a.depth} 52 | > 53 | {(tree) => ( 54 | 55 | {tree.links().map((link, i) => ( 56 | 63 | ))} 64 | {tree.descendants().map((node, key) => { 65 | const width = 60; 66 | const height = 20; 67 | const top: number = node.x; 68 | const left: number = node.y; 69 | 70 | return ( 71 | 72 | {node.depth === 0 && ( 73 | 77 | )} 78 | {node.depth > 0 && node.depth !== 3 && ( 79 | { 91 | if (node.depth == 2) { 92 | 93 | //To see what cluster this method belongs to , we have to access node.data.parent.name 94 | //To get the name of the method, we have to access node.data.name 95 | const clusterName = node.parent !== null ? node.parent.data.name : ''; 96 | const methodName = node.data.name; 97 | 98 | if (node.data.children) node.data.children.sort((a, b) => a.name.toLowerCase() > b.name.toLowerCase() ? 1 : -1); 99 | 100 | const endpointList: TreeNode[] = node.data.children ? node.data.children : []; 101 | 102 | setEndPoints({ clusterName: clusterName, methodName: methodName, endpointList: endpointList.map(node => node.name) }); 103 | } 104 | }} 105 | /> 106 | )} 107 | {node.depth !== 3 && ( 108 | 116 | {node.data.name} 117 | 118 | )} 119 | 120 | ); 121 | })} 122 | 123 | )} 124 | 125 | 126 | 127 | 128 | 129 | } 130 | ); 131 | } 132 | -------------------------------------------------------------------------------- /src/client/components/common/ChipSelector.tsx: -------------------------------------------------------------------------------- 1 | import React, { useCallback } from 'react'; 2 | 3 | import { Theme, useTheme } from '@mui/material/styles'; 4 | import Box from '@mui/material/Box'; 5 | import OutlinedInput from '@mui/material/OutlinedInput'; 6 | import InputLabel from '@mui/material/InputLabel'; 7 | import MenuItem from '@mui/material/MenuItem'; 8 | import FormControl from '@mui/material/FormControl'; 9 | import Select, { SelectChangeEvent } from '@mui/material/Select'; 10 | import Chip from '@mui/material/Chip'; 11 | 12 | const ITEM_HEIGHT = 48; 13 | const ITEM_PADDING_TOP = 8; 14 | 15 | /** 16 | * Used to style chip menu item. 17 | * 18 | * @param index - Index of current item in routes 19 | * @param selected - List of indices of selected items in routes 20 | * @param theme - MUI theme 21 | * 22 | * @returns Typography theme settings 23 | * 24 | * @private 25 | */ 26 | function getStyles(index: number, selected: number[], theme: Theme) { 27 | return { 28 | fontWeight: selected.indexOf(index) === -1 ? theme.typography.fontWeightRegular : theme.typography.fontWeightMedium, 29 | }; 30 | } 31 | 32 | export default function ChipSelector({ 33 | itemLabels, 34 | selected, 35 | label, 36 | setSelected, 37 | }: { 38 | itemLabels: string[], 39 | selected: number[]; 40 | label: string; 41 | setSelected: React.Dispatch> 42 | }): JSX.Element { 43 | 44 | const theme = useTheme(); 45 | 46 | const handleChange = useCallback((event: SelectChangeEvent) => { 47 | const { target: { value } } = event; 48 | setSelected( 49 | // On autofill we get a stringified value 50 | typeof value === 'string' ? value.split(',').map(val => parseInt(val)) : value, 51 | ); 52 | }, [setSelected]); 53 | 54 | return ( 55 | 56 | {label} 57 | 100 | 101 | ); 102 | } 103 | -------------------------------------------------------------------------------- /src/client/components/common/NavItem.tsx: -------------------------------------------------------------------------------- 1 | import React, { FunctionComponent } from 'react'; 2 | 3 | import List from '@mui/material/List'; 4 | import ListItem from '@mui/material/ListItem'; 5 | import ListItemIcon from '@mui/material/ListItemIcon'; 6 | import ListItemText from '@mui/material/ListItemText'; 7 | 8 | export default function NavItem({ 9 | isActive, 10 | title, 11 | subLinks, 12 | icon, 13 | onClick, 14 | }: { 15 | isActive: boolean; 16 | title: string; 17 | subLinks: { title: string, onClick: () => void }[]; 18 | icon: FunctionComponent; 19 | onClick: (event: React.MouseEvent) => void; 20 | }) { 21 | return ( 22 | 23 | 24 | {React.createElement(icon)} 25 | 26 | 27 | {isActive && subLinks.length > 0 && 28 | 29 | {subLinks.map(({ title, onClick }): JSX.Element => { 30 | return ( 31 | 32 | 33 | 34 | ); 35 | })} 36 | 37 | } 38 | 39 | ); 40 | } 41 | -------------------------------------------------------------------------------- /src/client/components/common/NavigationBar.scss: -------------------------------------------------------------------------------- 1 | #theme-select { 2 | display: flex; 3 | align-items: center; 4 | justify-content: flex-end; 5 | padding: 10px 10px 0 0; 6 | } 7 | -------------------------------------------------------------------------------- /src/client/components/common/NavigationBar.tsx: -------------------------------------------------------------------------------- 1 | import React, { useCallback } from 'react'; 2 | 3 | import { styled, Theme } from '@mui/material/styles'; 4 | import Box from '@mui/material/Box'; 5 | import MuiDrawer from '@mui/material/Drawer'; 6 | import Divider from '@mui/material/Divider'; 7 | import Typography from '@mui/material/Typography'; 8 | import IconButton from '@mui/material/IconButton'; 9 | import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; 10 | import MenuIcon from '@mui/icons-material/Menu'; 11 | import BubbleChartIcon from '@mui/icons-material/BubbleChart'; 12 | import AssessmentIcon from '@mui/icons-material/Assessment'; 13 | import DescriptionIcon from '@mui/icons-material/Description'; 14 | import Switch from '@mui/material/Switch'; 15 | import Stack from '@mui/material/Stack'; 16 | import LightModeIcon from '@mui/icons-material/LightMode'; 17 | import NightlightIcon from '@mui/icons-material/Nightlight'; 18 | import useWindowDimensions from './../../hooks/useWindowDimensions'; 19 | 20 | import { Page, SubPage } from './../../types'; 21 | import NavItem from './NavItem'; 22 | 23 | import './NavigationBar.scss'; 24 | 25 | export const drawerWidth = 240; 26 | export const closedDrawerWidth = 73; 27 | 28 | const openedMixin = (theme: Theme) => ({ 29 | width: drawerWidth, 30 | transition: theme.transitions.create('width', { 31 | easing: theme.transitions.easing.sharp, 32 | duration: theme.transitions.duration.enteringScreen, 33 | }), 34 | overflowX: 'hidden', 35 | }); 36 | 37 | const closedMixin = (theme: Theme) => ({ 38 | transition: theme.transitions.create('width', { 39 | easing: theme.transitions.easing.sharp, 40 | duration: theme.transitions.duration.leavingScreen, 41 | }), 42 | overflowX: 'hidden', 43 | width: `calc(${theme.spacing(7)} + 1px)`, 44 | [theme.breakpoints.up('sm')]: { 45 | width: `calc(${theme.spacing(9)} + 1px)`, 46 | }, 47 | }); 48 | 49 | const DrawerHeader = styled('div')(({ theme }) => ({ 50 | display: 'flex', 51 | alignItems: 'center', 52 | justifyContent: 'flex-end', 53 | padding: theme.spacing(0, 1), 54 | // necessary for content to be below app bar 55 | ...theme.mixins.toolbar, 56 | })); 57 | 58 | const Drawer = styled(MuiDrawer, { shouldForwardProp: (prop) => prop !== 'open' })( 59 | ({ theme, open }) => ({ 60 | width: drawerWidth, 61 | flexShrink: 0, 62 | whiteSpace: 'nowrap', 63 | boxSizing: 'border-box', 64 | ...(open && { 65 | ...openedMixin(theme), 66 | '& .MuiDrawer-paper': openedMixin(theme), 67 | }), 68 | ...(!open && { 69 | ...closedMixin(theme), 70 | '& .MuiDrawer-paper': closedMixin(theme), 71 | }), 72 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 73 | }) as any, // remove typescript complaint together with above line TODO determine how to fix this 74 | ); 75 | 76 | export default function NavigationBar({ 77 | useLightTheme, 78 | page, 79 | open, 80 | setOpen, 81 | setMetricSubPage, 82 | setDocSubPage, 83 | setPage, 84 | showApiDocPage, 85 | setUseLightTheme, 86 | }: { 87 | page: Page; 88 | open: boolean; 89 | useLightTheme: boolean; 90 | setOpen: React.Dispatch>; 91 | setMetricSubPage: React.Dispatch>; 92 | setDocSubPage: React.Dispatch>; 93 | setPage: React.Dispatch>; 94 | showApiDocPage: () => void; 95 | setUseLightTheme: React.Dispatch>; 96 | }): JSX.Element { 97 | 98 | const handleThemeChange: (event: React.ChangeEvent) => void = useCallback((event) => { 99 | setUseLightTheme(event.target.checked); 100 | }, [setUseLightTheme]); 101 | 102 | const handleDrawerOpen: () => void = useCallback(() => setOpen(true), [setOpen]); 103 | const handleDrawerClose: () => void = useCallback(() => setOpen(false), [setOpen]); 104 | 105 | const showClustersPage: () => void = useCallback(() => setPage(Page.Clusters), [setPage]); 106 | 107 | const showMetricsPage: () => void = useCallback(() => setPage(Page.Metrics), [setPage]); 108 | const showClusterLoadPage: () => void = useCallback(() => setMetricSubPage(SubPage.ClusterLoad), [setMetricSubPage]); 109 | const showRouteLoadPage: () => void = useCallback(() => setMetricSubPage(SubPage.RouteLoads), [setMetricSubPage]); 110 | 111 | const showDocumentationPage: (event: React.MouseEvent) => void = useCallback((event: React.SyntheticEvent) => { 112 | if ((event.target as HTMLElement).innerHTML === 'Documentation') setDocSubPage(SubPage.None); 113 | setPage(Page.Documentation); 114 | }, [setPage, setDocSubPage]); 115 | 116 | const { height } = useWindowDimensions(); 117 | 118 | return ( 119 | 120 | 121 | 122 | {open && <> 123 | gimbap 124 | 125 | 126 | 127 | } 128 | {!open && 129 | 138 | 139 | {/* */} 140 | 141 | } 142 | 143 | 144 | 145 | 146 | 147 | 148 | 154 | 155 | 156 | 157 | 164 | 180 | { 189 | setDocSubPage(SubPage.None); 190 | window.location.href = '#intro'; 191 | }, 192 | }, 193 | { 194 | title: 'Installation', 195 | onClick: () => { 196 | setDocSubPage(SubPage.None); 197 | window.location.href = '#installation'; 198 | }, 199 | }, 200 | { 201 | title: 'Visualizing Your Data', 202 | onClick: () => { 203 | setDocSubPage(SubPage.None); 204 | window.location.href = '#visualizing-your-data'; 205 | }, 206 | }, 207 | { 208 | title: 'API', 209 | onClick: showApiDocPage, 210 | }, 211 | { 212 | title: 'Contributors', 213 | onClick: () => { 214 | setDocSubPage(SubPage.None); 215 | window.location.href = '#contributors'; 216 | }, 217 | }, 218 | ]} 219 | onClick={showDocumentationPage} 220 | /> 221 | 222 | 223 | ); 224 | } 225 | -------------------------------------------------------------------------------- /src/client/components/common/Splash.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import CircularProgress from '@mui/material/CircularProgress'; 4 | import Box from '@mui/material/Box'; 5 | 6 | export default function Splash() { 7 | return ( 8 | 11 | < CircularProgress /> 12 | 13 | ); 14 | } 15 | -------------------------------------------------------------------------------- /src/client/components/documentation/ApiDoc.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | 5 | import CodeComment from './CodeComment'; 6 | import ApiTableOfContents from './ApiTableOfContents'; 7 | 8 | export default function ApiDoc() { 9 | return (<> 10 | 11 | API Documentation 12 | 13 | 14 | 15 | 16 | 17 | Types 18 | 19 | 23 | 27 | 35 | 36 | 37 | Gimbap 38 | 39 | '} 41 | explanation={[ 42 | 'Mutates express app objects to inject route data logging.', 43 | 'Note: Using PostgreSQL database is not currently supported.' 44 | ]} 45 | parameters={[ 46 | '{Application} app - express application object. Usually created with `const app = express();`.', 47 | '{\'mongodb\' | \'postgresql\'} database - choose between using `mongodb` or `postgresql` database to send data to.', 48 | ]} 49 | returns='{Promise} Promise returns when connection to database has been established.' 50 | /> 51 | 52 | 53 | Setup 54 | 55 | 60 | 64 | 65 | 66 | EndpointModel 67 | 68 | 78 | 88 | 89 | 90 | EndpointBucketsModel 91 | 92 | 104 | 110 | 114 | 118 | 123 | 132 | 136 | 140 | 141 | 142 | ClusterModel 143 | 144 | 148 | 153 | 157 | 161 | 162 | 163 | Endpoints Utility 164 | 165 | 171 | 180 | 186 | 195 | 196 | 197 | Data Generator 198 | 199 | 205 | 214 | 225 | ); 226 | } 227 | -------------------------------------------------------------------------------- /src/client/components/documentation/ApiTableOfContents.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Link from '@mui/material/Link'; 4 | import Stack from '@mui/material/Stack'; 5 | 6 | export default function ApiTableOfContents() { 7 | return ( 8 | Types 9 | Gimbap 10 | Setup 11 | EndpointModel 12 | EndpointBucketsModel 13 | ClusterModel 14 | Endpoints Utility 15 | Data Generator 16 | ); 17 | } 18 | -------------------------------------------------------------------------------- /src/client/components/documentation/CodeComment.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | import Stack from '@mui/material/Stack'; 5 | 6 | export default function CodeComment({ 7 | declaration, 8 | explanation, 9 | properties, 10 | parameters, 11 | returns, 12 | }: { 13 | declaration: string, 14 | explanation: string | string[], 15 | properties?: string | string[], 16 | parameters?: string | string[], 17 | returns?: string 18 | }): React.ReactElement { 19 | if (typeof explanation === 'string') explanation = [explanation]; 20 | if (typeof properties === 'string') properties = [properties]; 21 | if (typeof parameters === 'string') parameters = [parameters]; 22 | 23 | return ( 24 | {declaration} 25 | {explanation.map((paragraph, i) => {paragraph})} 26 | {properties && properties.map((property, i) => @property {property})} 27 | {parameters && parameters.map((parameter, i) => @param {parameter})} 28 | {returns && @returns {returns}} 29 | ); 30 | } 31 | -------------------------------------------------------------------------------- /src/client/components/documentation/Contributors.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | import Link from '@mui/material/Link'; 5 | 6 | 7 | export default function Contributors() { 8 | return (<> 9 | Contributors 10 | Angelynn Truong 11 | Khandker Islam 12 | Miguel Hernandez 13 | Parker Hutcheson 14 | Sebastien Fauque 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /src/client/components/documentation/Documentation.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Stack from '@mui/material/Stack'; 4 | 5 | import { SubPage } from './../../types'; 6 | import TableOfContents from './TableOfContents'; 7 | import Intro from './Intro'; 8 | import Installation from './Installation'; 9 | import VisualizingYourData from './VisualizingYourData'; 10 | import ApiDoc from './ApiDoc'; 11 | import Contributors from './Contributors'; 12 | 13 | export default function Documentation({ 14 | useLightTheme, 15 | subPage, 16 | showApiDocPage, 17 | }: { 18 | useLightTheme: boolean; 19 | subPage: SubPage; 20 | showApiDocPage: () => void; 21 | }): JSX.Element { 22 | 23 | return (<> 24 | 25 | {subPage === SubPage.ApiDoc && } 26 | {subPage === SubPage.None && <> 27 | 28 | 29 | 30 | 31 | 32 | } 33 | 34 | ); 35 | } 36 | -------------------------------------------------------------------------------- /src/client/components/documentation/Installation.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | import { CopyBlock, dracula, a11yLight } from 'react-code-blocks'; 5 | 6 | 7 | export default function Installation({ 8 | useLightTheme 9 | }: { 10 | useLightTheme: boolean; 11 | }) { 12 | 13 | 14 | return (<> 15 | 16 | Installation 17 | 18 | 19 | 20 | Integrate Into Monolith 21 | 22 | 23 | The gimbap npm package is responsible for collecting server response data and sending that data to your MongoDB. This database must be set up before you launch your monolith using gimbap. 24 | 25 | 26 | Gimbap is a function that mutates express’ app object to inject route data logging. 27 | 28 | 29 | First, install gimbap as a dependency of your monolithic application: 30 | 31 | 32 | 39 | 40 | 41 | Next, import gimbap into your main server file and add call it before any other middleware function: 42 | 43 | 44 | 52 | 53 | 54 | Visualization Backend 55 | 56 | 57 | 58 | To launch the visualization app, a React front-end with an Express backend, first fork and clone the gimbap git repository. 59 | 60 | 61 | 62 | Next, in the root directory, create a file called `secrets.json.` This JSON file must contain an object with a property called `MONGODB_URI` whose value is the URI string to connect to your MongoDB instance. 63 | 64 | 65 | 66 | Note, your MongoDB must be running as a replica set. Use these instructions to convert a local database into a replica set. 67 | 68 | 69 | 70 | Next, build and launch the backend: 71 | 72 | 73 | 81 | 82 | 83 | You can access the backend at localhost:3000. 84 | 85 | 86 | 87 | Or, in development mode, to launch a Webpack dev server, simply run: 88 | 89 | 90 | 97 | 98 | 99 | You can access the backend at localhost:8080. 100 | 101 | ); 102 | } 103 | -------------------------------------------------------------------------------- /src/client/components/documentation/Intro.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | 5 | export default function Intro() { 6 | return (<> 7 | Intro 8 | 9 | 10 | A monolithic architecture is the traditional application design where the whole product is conveniently compiled together and deployed in a single artifact. The approach is simple but becomes problematic as an application scales horizontally. Most successful applications tend to grow in size and complexity. At a high scale, this would result in difficulty in understanding and updating the application. In addition, the size could significantly slow down the start-up time of the application. Since all modules are compiled at once, the monolithic approach is unreliable as a bug in a single module may break the entire application. 11 | 12 | 13 | 14 | When it comes to working with largely scaled applications, the microservices architecture is more efficient. Splitting a complex application into a smaller set of interconnected services is incredibly impactful. This allows accessible and efficient continuous deployment as each service can be scaled independently. 15 | 16 | 17 | 18 | It can be laborious and challenging to know exactly how to break up an application effectively. If done incorrectly, the business may end up with high costs and loss of resources. 19 | 20 | 21 | 22 | With Gimbap, developers simply need to install an NPM package and let the tool handle the rest. First, the Express application object will be modified to allow Gimbap to track endpoint performance. This data will be stored in a database, allowing developers to monitor individual endpoints and view microservice clustering recommendations based upon endpoints’ similar covariant scores. Lastly, the information is presented in easy-to-read dendrograms and line charts. 23 | 24 | ); 25 | } 26 | -------------------------------------------------------------------------------- /src/client/components/documentation/TableOfContents.scss: -------------------------------------------------------------------------------- 1 | #doc-link-api { 2 | cursor: pointer; 3 | } 4 | -------------------------------------------------------------------------------- /src/client/components/documentation/TableOfContents.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | import Link from '@mui/material/Link'; 5 | import Stack from '@mui/material/Stack'; 6 | 7 | import './TableOfContents.scss'; 8 | 9 | export default function TableOfContents({ 10 | showApiDocPage 11 | }: { 12 | showApiDocPage: () => void; 13 | }) { 14 | return (<> 15 | Documentation 16 | 17 | Intro 18 | Installation 19 | Visualizing Your Data 20 | API 21 | Contributors 22 | 23 | ); 24 | } 25 | -------------------------------------------------------------------------------- /src/client/components/documentation/VisualizingYourData.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | 5 | export default function VisualizingYourData() { 6 | return (<> 7 | Visualizing Your Data 8 | 9 | 10 | Cluster Tree 11 | 12 | 13 | Data is processed through Gimbap’s clustering algorithm and a dendrogram is generated that visualizes endpoint clustering recommendations based on similar covariant scores. 14 | 15 | 16 | 17 | Load Graph 18 | 19 | 20 | Dive into more in-depth metrics using Gimbap’s load graphs. Developers are able to analyze recommended clusters in addition to monitoring the call times of individual routes within their application. 21 | 22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /src/client/components/metrics/ClusterLoad.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | import Stack from '@mui/material/Stack'; 5 | import Splash from './../common/Splash'; 6 | 7 | import { Cluster, LoadData } from './../../../shared/types'; 8 | import { fetchClusterLoadData } from './../../utils/ajax'; 9 | import { drawerWidth } from './../common/NavigationBar'; 10 | import useWindowDimensions from './../../hooks/useWindowDimensions'; 11 | import ChipSelector from './../common/ChipSelector'; 12 | import LoadGraph from './LoadGraph'; 13 | 14 | 15 | export default function ClusterLoad({ 16 | clusters, 17 | isNavBarOpen, 18 | useLightTheme 19 | }: { 20 | clusters: Cluster[] | null; 21 | isNavBarOpen: boolean; 22 | useLightTheme: boolean; 23 | }): JSX.Element { 24 | 25 | const { width: windowWidth } = useWindowDimensions(); 26 | 27 | const [selectedClusters, setSelectedClusters] = useState([]); // indices in clusters 28 | const [clustersLoadData, setClustersLoadData] = useState<{ [key: number]: LoadData }>(Object.create(null)); 29 | 30 | 31 | // load route load data on user selecting an endpoint 32 | useEffect(() => { 33 | if (clusters === null) return; 34 | 35 | for (const index of selectedClusters) { 36 | if (!clustersLoadData[index]) { 37 | fetchClusterLoadData(index, setClustersLoadData); 38 | } 39 | } 40 | }, [clusters, clustersLoadData, selectedClusters]); 41 | 42 | 43 | // will be undefined if data has not finished transferring from back-end 44 | const selectedLoadData: { [key: number]: LoadData | undefined } = selectedClusters.reduce((selectedData: { [key: number]: LoadData | undefined }, index: number) => { 45 | selectedData[index] = clustersLoadData[index]; 46 | return selectedData; 47 | }, Object.create(null)); 48 | 49 | const clusterLabels = clusters ? clusters.map((_, i: number) => `Cluster ${i}`) : []; 50 | 51 | return (<> 52 | {!clusters && } 53 | {clusters && 54 | 55 | Cluster Load Graphs 56 | Average number of server calls to a particular cluster per 24-hour time period. 57 | Select clusters to view graphs. 58 | 59 | 65 | 66 | {Object.entries(selectedLoadData).map(([index, loadData]) => { 67 | if (!loadData) return ; 68 | 69 | const i: number = parseInt(index); 70 | const label = clusterLabels[i]; 71 | return ( 72 | ); 80 | })} 81 | 82 | } 83 | ); 84 | } 85 | -------------------------------------------------------------------------------- /src/client/components/metrics/LoadGraph.scss: -------------------------------------------------------------------------------- 1 | .load-graph { 2 | display: flex; 3 | flex-direction: column; 4 | align-items: center; 5 | 6 | svg { 7 | margin-top: -30px; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/client/components/metrics/LoadGraph.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { LoadData } from '../../../shared/types'; 4 | 5 | import { Axis, LineSeries, XYChart, Tooltip, lightTheme, darkTheme } from '@visx/xychart'; 6 | import { curveLinear } from '@visx/curve'; 7 | import Stack from '@mui/material/Stack'; 8 | import Typography from '@mui/material/Typography'; 9 | 10 | import './LoadGraph.scss'; 11 | 12 | export default function LoadGraph({ 13 | useLightTheme, 14 | height, 15 | width, 16 | loadData, 17 | label, 18 | }: { 19 | useLightTheme: boolean; 20 | height: number; 21 | width: number; 22 | loadData: LoadData; 23 | label: string 24 | }): JSX.Element { 25 | 26 | const accessors = { 27 | xAccessor: (d: [number, number]): number => d[0], 28 | yAccessor: (d: [number, number]): number => d[1], 29 | }; 30 | 31 | return ( 32 | 33 | {label} 34 | 41 | 46 | 51 | 57 | { 63 | if (tooltipData && colorScale && tooltipData.nearestDatum) { 64 | const hour: number = Math.floor(accessors.xAccessor(tooltipData.nearestDatum.datum as [number, number])); 65 | const minutes: number = Math.round((accessors.xAccessor(tooltipData.nearestDatum.datum as [number, number]) % 1) * 60); 66 | 67 | return ( 68 |
69 |
70 | {tooltipData.nearestDatum.key} 71 |
72 | {`time: ${hour < 10 ? '0' : ''}${hour}:${minutes}`} 73 | {', '} 74 | {'# calls: ' + accessors.yAccessor(tooltipData.nearestDatum.datum as [number, number])} 75 |
76 | ); 77 | } 78 | }} 79 | /> 80 | 81 | 82 | ); 83 | } 84 | -------------------------------------------------------------------------------- /src/client/components/metrics/Metrics.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { Route, Cluster } from './../../../shared/types'; 4 | import { SubPage } from './../../types'; 5 | 6 | import RouteLoad from './RouteLoad'; 7 | import ClusterLoad from './ClusterLoad'; 8 | 9 | export default function Metrics({ 10 | routes, 11 | clusters, 12 | useLightTheme, 13 | isNavBarOpen, 14 | metricSubPage, 15 | }: { 16 | routes: Route[] | null; 17 | clusters: Cluster[] | null; 18 | useLightTheme: boolean; 19 | isNavBarOpen: boolean; 20 | metricSubPage: SubPage; 21 | }) { 22 | return (<> 23 | {metricSubPage === SubPage.RouteLoads && 24 | 25 | } 26 | {metricSubPage === SubPage.ClusterLoad && 27 | 28 | } 29 | ); 30 | } 31 | -------------------------------------------------------------------------------- /src/client/components/metrics/RouteLoad.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | 3 | import Typography from '@mui/material/Typography'; 4 | import Stack from '@mui/material/Stack'; 5 | import Splash from './../common/Splash'; 6 | import LoadGraph from './LoadGraph'; 7 | 8 | import { Route, LoadData } from './../../../shared/types'; 9 | import { fetchRouteLoadData } from './../../utils/ajax'; 10 | import { drawerWidth } from './../common/NavigationBar'; 11 | import useWindowDimensions from './../../hooks/useWindowDimensions'; 12 | import ChipSelector from './../common/ChipSelector'; 13 | 14 | export default function RouteLoad({ 15 | routes, 16 | isNavBarOpen, 17 | useLightTheme, 18 | }: { 19 | routes: Route[] | null; 20 | isNavBarOpen: boolean; 21 | useLightTheme: boolean 22 | }): JSX.Element { 23 | 24 | const { width: windowWidth } = useWindowDimensions(); 25 | 26 | const [selectedRoutes, setSelectedRoutes] = useState([]); // indices in routes 27 | const [routesLoadData, setRoutesLoadData] = useState<{ [key: number]: LoadData }>(Object.create(null)); 28 | 29 | // load route load data on user selecting an endpoint 30 | useEffect(() => { 31 | if (routes === null) return; 32 | 33 | for (const index of selectedRoutes) { 34 | if (!routesLoadData[index]) { 35 | fetchRouteLoadData(routes[index], index, setRoutesLoadData); 36 | } 37 | } 38 | }, [routes, routesLoadData, selectedRoutes]); 39 | 40 | // TODO when user has graph selected, but data is not available, show splash 41 | 42 | const selectedLoadData: { [key: number]: LoadData | undefined } = selectedRoutes.reduce((selectedData: { [key: number]: LoadData | undefined }, index: number) => { 43 | selectedData[index] = routesLoadData[index]; 44 | return selectedData; 45 | }, Object.create(null)); 46 | 47 | const routeLabels = routes ? routes.map(route => `${route.method} ${route.endpoint}`) : []; 48 | 49 | return (<> 50 | {!routes && } 51 | {routes && 52 | 53 | Route Load Graphs 54 | Average number of server calls to a particular endpoint per 24-hour time period. 55 | Select routes to view graphs. 56 | 57 | 63 | 64 | {Object.entries(selectedLoadData).map(([index, loadData]) => { 65 | if (!loadData) return ; 66 | 67 | const i: number = parseInt(index); 68 | const label = routeLabels[i]; 69 | 70 | return ( 71 | ); 79 | })} 80 | 81 | } 82 | ); 83 | } 84 | -------------------------------------------------------------------------------- /src/client/hooks/useWindowDimensions.ts: -------------------------------------------------------------------------------- 1 | import { useState, useEffect } from 'react'; 2 | 3 | function getWindowDimensions() { 4 | const { innerWidth: width, innerHeight: height } = window; 5 | return { width, height }; 6 | } 7 | 8 | /** 9 | * Custom hook for using current window size. 10 | * 11 | * @returns {{width: number, height: number}} object with width and height property for current window size 12 | */ 13 | export default function useWindowDimensions(): { width: number, height: number } { 14 | const [windowDimensions, setWindowDimensions] = useState(getWindowDimensions()); 15 | 16 | useEffect(() => { 17 | function handleResize() { 18 | setWindowDimensions(getWindowDimensions()); 19 | } 20 | 21 | window.addEventListener('resize', handleResize); 22 | return () => window.removeEventListener('resize', handleResize); 23 | }, []); 24 | 25 | return windowDimensions; 26 | } 27 | -------------------------------------------------------------------------------- /src/client/index.scss: -------------------------------------------------------------------------------- 1 | body { 2 | overflow-x: hidden; 3 | } -------------------------------------------------------------------------------- /src/client/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | 4 | import './index.scss'; 5 | import 'normalize.css'; 6 | 7 | import App from './App'; 8 | 9 | ReactDOM.render(, document.getElementById('root')); 10 | -------------------------------------------------------------------------------- /src/client/theme.ts: -------------------------------------------------------------------------------- 1 | import { createTheme, Theme } from '@mui/material/styles'; 2 | 3 | export const lightTheme: Theme = createTheme({ 4 | palette: { 5 | mode: 'light', 6 | primary: { 7 | main: '#3559A4', 8 | }, 9 | secondary: { 10 | main: '#3559A4', 11 | }, 12 | error: { 13 | main: '#3559A4', 14 | }, 15 | warning: { 16 | main: '#3559A4', 17 | }, 18 | divider: '#28A78D', 19 | }, 20 | }); 21 | 22 | export const darkTheme: Theme = createTheme({ 23 | palette: { 24 | mode: 'dark', 25 | primary: { 26 | main:'#fff', 27 | }, 28 | divider: '#242423', 29 | background: { 30 | default: '#E24B44', 31 | paper: '#242423', 32 | }, 33 | text: { 34 | primary: '#fff', 35 | secondary: '#fff', 36 | }, 37 | 38 | } 39 | }); 40 | -------------------------------------------------------------------------------- /src/client/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["dom", "dom.iterable", "esnext"], 4 | "allowJs": true, 5 | "esModuleInterop": true, 6 | "strict": true, 7 | "noImplicitAny": true, 8 | "strictNullChecks": true, 9 | "forceConsistentCasingInFileNames": true, 10 | "moduleResolution": "node", 11 | "resolveJsonModule": true, 12 | "noEmit": true, 13 | "jsx": "react", 14 | "paths": { 15 | "*.ts": ["./*"] 16 | } 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /src/client/types.ts: -------------------------------------------------------------------------------- 1 | export enum Page { 2 | Clusters, 3 | Metrics, 4 | Documentation, 5 | } 6 | 7 | export enum SubPage { 8 | None, 9 | RouteLoads, 10 | ClusterLoad, 11 | ApiDoc 12 | } 13 | -------------------------------------------------------------------------------- /src/client/utils/ajax.ts: -------------------------------------------------------------------------------- 1 | import { ClientError, Cluster, Route, LoadData, TreeNode } from './../../shared/types'; 2 | 3 | /** 4 | * Makes a GET fetch request. 5 | * 6 | * @param url - URL to make fetch request to. 7 | * @returns Promise to generic data type or void if fetch failed. 8 | * 9 | * @private 10 | */ 11 | async function fetchWrapper(url: string): Promise { 12 | const response: Response = await fetch(url, { 13 | method: 'GET', 14 | headers: { 15 | 'Accept': 'application/json' 16 | } 17 | }); 18 | 19 | const body: T | ClientError = await response.json(); 20 | 21 | if (response.status !== 200) { 22 | console.error(`Server responded with status ${response.status}`); 23 | } 24 | if (typeof body === 'object' && (body as object).hasOwnProperty('error')) return console.error((body as ClientError).error); 25 | 26 | return body as T; 27 | } 28 | 29 | /** 30 | * Make a fetch request to backend to load unique routes. 31 | * 32 | * @param setRoutes - state setter function for Route[] 33 | * 34 | * @public 35 | */ 36 | export async function fetchRoutes(setRoutes: React.Dispatch>): Promise { 37 | const routes: Route[] | void = await fetchWrapper('/api/graph/endpoint'); 38 | if (routes) setRoutes(routes); 39 | } 40 | 41 | /** 42 | * Make a fetch request to backend for LoadData for a particular route. 43 | * 44 | * @param {Route} route - Route to fetch LoadData for. 45 | * @param {number} index - index in setRoutesLoadData object to store the LoadData at 46 | * @param {React.Dispatch> 54 | ): Promise { 55 | const loadData: LoadData | void = await fetchWrapper(`/api/graph/endpoint/load?method=${encodeURIComponent(route.method)}&route=${encodeURIComponent(route.endpoint)}`); 56 | if (loadData) setRoutesLoadData(routesLoadData => { 57 | const nextRoutesLoadData = Object.assign(Object.create(null), routesLoadData); 58 | nextRoutesLoadData[index] = loadData; 59 | return nextRoutesLoadData; 60 | }); 61 | } 62 | 63 | /** 64 | * Make fetch request to backend to load clusters. 65 | * 66 | * @param setClusters - updates state with Clusters 67 | * 68 | * @public 69 | */ 70 | export async function fetchClusters(setClusters: React.Dispatch>): Promise { 71 | const clusters: Cluster[] | void = await fetchWrapper('api/graph/cluster'); 72 | if (clusters) setClusters(clusters); 73 | } 74 | 75 | /** 76 | * Make a fetch request to backend for LoadData for a particular cluster. 77 | * 78 | * @param {number} index - index in setClusterLoadData object to store the LoadData at 79 | * @param {React.Dispatch> 86 | ): Promise { 87 | const loadData: LoadData | void = await fetchWrapper(`/api/graph/cluster/load/${index}`); 88 | if (loadData) setClusterLoadData(clusterLoadData => { 89 | const nextClustersLoadData = Object.assign(Object.create(null), clusterLoadData); 90 | nextClustersLoadData[index] = loadData; 91 | return nextClustersLoadData; 92 | }); 93 | } 94 | 95 | /** 96 | * Make a fetch request to backend for LoadData for a particular cluster. 97 | * 98 | * @param {number} index - index in setClusterLoadData object to store the LoadData at 99 | * @param {React.Dispatch>): Promise { 104 | const trees: TreeNode | void = await fetchWrapper('api/graph/cluster/tree'); 105 | if (trees) setTreeGraphData(trees); 106 | } 107 | -------------------------------------------------------------------------------- /src/npm/index.ts: -------------------------------------------------------------------------------- 1 | import { Application, Request, Response } from 'express'; 2 | import { logEndpoint } from '../shared/models/endpointModel'; 3 | import { connect, disconnect } from '../shared/models/mongoSetup'; 4 | 5 | let appRef: InternalApplication | null = null; 6 | let originalHandleFunction: HandleFunction | null = null; 7 | 8 | type DoneCallback = () => void; 9 | type HandleFunction = (req: Request, res: Response, callback: DoneCallback) => void; 10 | interface InternalApplication extends Application { 11 | handle: HandleFunction; 12 | } 13 | 14 | /** 15 | * Mutates express app object to inject route data logging. 16 | * 17 | * @param {Application} app - express application object. Usually created with `const app = express();`. 18 | * @param {'mongodb' | 'postgresql'} database - choose between using `mongodb` or `postgresql` database to send data to. 19 | * @param {string} dbURI - URI to connect to database. 20 | * 21 | * @return {Promise} Promise returns when connection to database has been established. 22 | * 23 | * @public 24 | */ 25 | export default function gimbap( 26 | app: Application, 27 | database: 'mongodb' | 'postgresql', 28 | dbURI: string 29 | ): Promise { 30 | // TODO implement PostgreSQL support. 31 | if (database === 'postgresql') throw new Error('PostgreSQL is not currently supported.'); 32 | 33 | appRef = app as InternalApplication; 34 | originalHandleFunction = appRef.handle; 35 | 36 | appRef.handle = function handleGimbapMiddleware(req: Request, res: Response, callback: DoneCallback): void { 37 | logEndpoint(req.method, req.originalUrl || req.url, Date.now()); 38 | 39 | return (originalHandleFunction as HandleFunction).call(appRef, req, res, callback); 40 | }; 41 | 42 | return connect(dbURI); 43 | } 44 | 45 | /** 46 | * Close connection with database and stop logging endpoints. 47 | * 48 | * @public 49 | */ 50 | gimbap.stop = async (): Promise => { 51 | if (appRef && originalHandleFunction) { 52 | appRef.handle = originalHandleFunction; 53 | return disconnect(); 54 | } 55 | }; 56 | -------------------------------------------------------------------------------- /src/npm/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gimbap", 3 | "version": "0.0.3", 4 | "description": "Express endpoint logging tool.", 5 | "main": "./npm/index.js", 6 | "keywords": [ 7 | "express", 8 | "gimbap", 9 | "logging" 10 | ], 11 | "author": "Miguel Hernandez (miguelh72@outlook.com)", 12 | "license": "MIT", 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/oslabs-beta/gimbap.git" 16 | }, 17 | "bugs": { 18 | "url": "https://github.com/oslabs-beta/gimbap/issues" 19 | }, 20 | "homepage": "https://github.com/oslabs-beta/gimbap#readme" 21 | } -------------------------------------------------------------------------------- /src/npm/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./../../build/npm", 4 | 5 | "module": "commonjs", 6 | "esModuleInterop": true, 7 | "target": "es2021", 8 | "declaration": true, 9 | "strict": true, 10 | "noImplicitAny": true, 11 | "strictNullChecks": true, 12 | "moduleResolution": "node", 13 | "forceConsistentCasingInFileNames": true, 14 | "baseUrl": ".", 15 | "sourceMap": true, 16 | "paths": { 17 | "*": [ 18 | "node_modules/*" 19 | ] 20 | } 21 | }, 22 | "include": [ 23 | "./**/*", 24 | "../shared/models" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /src/server/controllers/cacheController.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, NextFunction } from 'express'; 2 | import mcache from 'memory-cache'; 3 | 4 | export function cache(duration: number) { 5 | return function cachingMiddleware(req: Request, res: Response, next: NextFunction) { 6 | const key = `__express__${req.method}__${req.originalUrl || req.url}`; 7 | const cachedResponse = mcache.get(key); 8 | if (cachedResponse) { 9 | return res.send(cachedResponse); 10 | } else { 11 | const originalSendFunc = res.send; 12 | res.send = (body: string) => { 13 | if (res.statusCode === 200) { 14 | mcache.put(key, body, duration); 15 | } 16 | 17 | return originalSendFunc.call(res, body); 18 | }; 19 | } 20 | return next(); 21 | }; 22 | } 23 | -------------------------------------------------------------------------------- /src/server/controllers/dataController.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, NextFunction } from 'express'; 2 | import { getDistinctRoutes, getEndpointBuckets, EndpointBuckets } from './../models/endpointBucketsModel'; 3 | import { getClusters } from './../models/clusterModel'; 4 | import { getLoadData, getClusterTreeNode } from './../utils/endpoints'; 5 | import { Route, Cluster } from './../../shared/types'; 6 | 7 | // ! remove let clusters: Cluster[] | undefined = undefined; 8 | 9 | /** 10 | * Middleware: If successful, `res.locals.endpoints` will contain Route[]. 11 | * 12 | * @param {Request} req - express's HTTP request object 13 | * @param {Response} res - express's HTTP response object 14 | * @param {NextFunction} next - express's next function 15 | * 16 | * @public 17 | */ 18 | export async function getEndpointList(req: Request, res: Response, next: NextFunction): Promise { 19 | try { 20 | const routes: Route[] = await getDistinctRoutes(); 21 | 22 | res.locals.endpoints = routes; 23 | } catch (error) { 24 | return next(Object.assign(error, { 25 | status: 500, 26 | error: 'Internal server error reading database.' 27 | })); 28 | } 29 | 30 | return next(); 31 | } 32 | 33 | /** 34 | * Middleware: Depends on query parameter method and route to be set in request object. 35 | * If successful, `res.locals.loadGraphData` will contain LoadData. 36 | * 37 | * @param {Request} req - express's HTTP request object 38 | * @param {Response} res - express's HTTP response object 39 | * @param {NextFunction} next - express's next function 40 | * 41 | * @public 42 | */ 43 | export async function getEndpointLoadGraphData(req: Request, res: Response, next: NextFunction): Promise { 44 | const { method, route } = req.query; 45 | 46 | if (typeof method !== 'string' || typeof route !== 'string') return next({ 47 | status: 500, 48 | message: 'Reached getEndpointLoadGraphData middleware without method and route parameters set in request object.', 49 | error: 'Internal server error.' 50 | }); 51 | 52 | try { 53 | const endpointBuckets: EndpointBuckets | null = await getEndpointBuckets(method, route); 54 | 55 | res.locals.loadGraphData = endpointBuckets !== null ? getLoadData(endpointBuckets) : []; 56 | } catch (error) { 57 | return next(Object.assign(error, { 58 | status: 500, 59 | error: 'Internal server error reading database.' 60 | })); 61 | } 62 | 63 | return next(); 64 | } 65 | 66 | /** 67 | * Middleware: If successful, `res.locals.clusters` will contain Cluster[]. 68 | * 69 | * @param {Request} req - express's HTTP request object 70 | * @param {Response} res - express's HTTP response object 71 | * @param {NextFunction} next - express's next function 72 | * 73 | * @public 74 | */ 75 | export async function getClusterList(req: Request, res: Response, next: NextFunction): Promise { 76 | try { 77 | const clusters: Cluster[] | null = await getClusters(); 78 | res.locals.clusters = clusters === null ? [] : clusters; 79 | } catch (error) { 80 | return next(Object.assign(error, { 81 | status: 500, 82 | error: 'Internal server error reading database.' 83 | })); 84 | } 85 | 86 | return next(); 87 | } 88 | 89 | /** 90 | * Middleware: Depends on parameter clusterId to be set in request object. 91 | * If successful, `res.locals.loadGraphData` will contain LoadData. 92 | * 93 | * @param {Request} req - express's HTTP request object 94 | * @param {Response} res - express's HTTP response object 95 | * @param {NextFunction} next - express's next function 96 | * 97 | * @public 98 | */ 99 | export async function getClusterLoadGraphData(req: Request, res: Response, next: NextFunction): Promise { 100 | const { clusterId: clusterIdStr } = req.params; 101 | 102 | if (!clusterIdStr) return next({ 103 | status: 500, 104 | message: 'Reached getClusterLoadGraphData middleware without clusterId parameters set in request object.', 105 | error: 'Internal server error.' 106 | }); 107 | 108 | const clusterId: number = parseInt(clusterIdStr); 109 | 110 | if (isNaN(clusterId)) next({ 111 | status: 400, 112 | error: 'ClusterId parameter must be a number.' 113 | }); 114 | 115 | try { 116 | let clusters: Cluster[] | null = await getClusters(); 117 | if (clusters === null) clusters = []; 118 | const routes: Route[] = clusters[clusterId]; 119 | 120 | const allEndpointBucketsInCluster: EndpointBuckets[] = []; 121 | for (const route of routes) { 122 | const endpointBuckets: EndpointBuckets | null = await getEndpointBuckets(route.method, route.endpoint); 123 | 124 | if (endpointBuckets !== null) allEndpointBucketsInCluster.push(endpointBuckets); 125 | } 126 | 127 | // combine all endpoint buckets into a single buckets vector for getLoadData to use 128 | const buckets: number[] = allEndpointBucketsInCluster.reduce((allBuckets: number[], endpointBuckets: EndpointBuckets) => { 129 | endpointBuckets.buckets.forEach((value, i) => allBuckets[i] += value); 130 | return allBuckets; 131 | }, Array.from({ length: allEndpointBucketsInCluster[0].buckets.length }, () => 0)); 132 | 133 | res.locals.loadGraphData = getLoadData({ 134 | method: allEndpointBucketsInCluster[0].method, 135 | endpoint: allEndpointBucketsInCluster[0].endpoint, 136 | lastEndpointId: allEndpointBucketsInCluster[0].lastEndpointId, 137 | newestDate: allEndpointBucketsInCluster[0].newestDate, 138 | oldestDate: allEndpointBucketsInCluster[0].oldestDate, 139 | buckets, 140 | }); 141 | } catch (error) { 142 | return next(Object.assign(error, { 143 | status: 500, 144 | error: 'Internal server error reading database.' 145 | })); 146 | } 147 | 148 | return next(); 149 | } 150 | 151 | /** 152 | * If successful, `res.locals.treeGraphData` will contain treeNode. 153 | * 154 | * @param {Request} req - express's HTTP request object 155 | * @param {Response} res - express's HTTP response object 156 | * @param {NextFunction} next - express's next function 157 | * 158 | * @public 159 | */ 160 | export async function getClusterTreeGraphData(req: Request, res: Response, next: NextFunction): Promise { 161 | try { 162 | let clusters: Cluster[] | null = await getClusters(); 163 | if (clusters === null) clusters = []; 164 | 165 | res.locals.treeGraphData = getClusterTreeNode(clusters); 166 | } catch (error) { 167 | return next(Object.assign(error, { 168 | status: 500, 169 | error: 'Internal server error reading database.' 170 | })); 171 | } 172 | 173 | return next(); 174 | } 175 | -------------------------------------------------------------------------------- /src/server/index.ts: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import express, { Request, Response, NextFunction, Express } from 'express'; 3 | import compression from 'compression'; 4 | import { connect } from './../shared/models/mongoSetup'; 5 | // import gimbap from 'gimbap'; 6 | 7 | import { MONGODB_URI } from './secrets.json'; 8 | 9 | import MiddlewareError from './utils/MiddlewareError'; 10 | import apiRouter from './routes/apiRouter'; 11 | 12 | //import { MONGODB_URI } from './secrets.json'; 13 | const PORT: number = process.env.PORT ? parseInt(process.env.PORT) : 3000; 14 | const HOST: string = process.env.HOST || 'localhost'; 15 | 16 | const app: Express = express(); 17 | //gimbap(app, 'mongodb', MONGODB_URI); // TODO remove before merge with main 18 | // TODO figure out why this causes Endpoint to be overwritten and conflict 19 | 20 | 21 | /* MIDDLEWARE */ 22 | app.use(express.json()); 23 | app.use(compression()); 24 | 25 | /* STATIC SERVER */ 26 | if (process.env.NODE_ENV === 'production') { 27 | app.use('/', express.static(path.resolve(__dirname, './../client'))); 28 | } 29 | 30 | 31 | /* ROUTES */ 32 | app.use('/api', apiRouter); 33 | 34 | 35 | /* GLOBAL 404 */ 36 | app.use('*', (req: Request, res: Response) => res.status(404).sendFile(path.resolve(__dirname, './../client/404.html'))); 37 | // TODO improve 404 layout 38 | 39 | 40 | /* GLOBAL ERROR HANDLER */ 41 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 42 | app.use((err: MiddlewareError, req: Request, res: Response, next: NextFunction) => { 43 | console.error(err); 44 | 45 | const defaultClientError = new MiddlewareError('Unknown server error. Please check server log.', 500); 46 | const clientError = Object.assign(defaultClientError, err); 47 | res.status(clientError.status).send(JSON.stringify({ error: clientError.error })); 48 | }); 49 | 50 | 51 | /* INIT SERVER */ 52 | if (process.env.NODE_ENV !== 'test') { 53 | connect(MONGODB_URI).then(() => 54 | app.listen(PORT, HOST, () => console.log(`Server listening on http://${HOST}:${PORT}`)) 55 | ); 56 | } 57 | 58 | export default app; 59 | -------------------------------------------------------------------------------- /src/server/models/clusterModel.ts: -------------------------------------------------------------------------------- 1 | import mongoose from 'mongoose'; 2 | 3 | import { Cluster } from './../../shared/types'; 4 | import { getLastEndpoint, Endpoint } from './../../shared/models/endpointModel'; 5 | import { getAllEndpointBuckets, EndpointBuckets } from './../models/endpointBucketsModel'; 6 | import { determineClusters } from './../utils/endpoints'; 7 | 8 | 9 | export type DBClusterSchema = { 10 | clusters: Cluster[], 11 | lastEndpointId: number, 12 | } 13 | 14 | 15 | const FORCED_UPDATE_TIMEOUT = 30 * 60 * 1000; 16 | 17 | let intervalId: NodeJS.Timer | null = null; 18 | 19 | const ClusterSchema = new mongoose.Schema({ 20 | clusters: { type: [[{ method: String, endpoint: String }]], required: true }, 21 | lastEndpointId: { type: Number, required: true }, 22 | }); 23 | export const ClusterModel: mongoose.Model = mongoose.model('Cluster', ClusterSchema); 24 | 25 | 26 | /** 27 | * Initiate the watching for new data to recalculate ClusterModel. 28 | * 29 | * @public 30 | */ 31 | export function startWatchingClusterModel(): void { 32 | // set interval if it does not already exist 33 | if (intervalId !== null) { 34 | intervalId = setInterval(() => { 35 | updateClusters(); 36 | }, FORCED_UPDATE_TIMEOUT); 37 | } 38 | } 39 | 40 | /** 41 | * Stop watching for new data. 42 | * 43 | * @public 44 | */ 45 | export function stopWatchingClusterModel(): void { 46 | // clear the interval 47 | if (intervalId !== null) { 48 | clearInterval(intervalId); 49 | intervalId = null; 50 | } 51 | } 52 | 53 | /** 54 | * Forces cluster recalculation and update if new data is available. 55 | * 56 | * @public 57 | */ 58 | export async function forceUpdate(): Promise { 59 | await updateClusters(); 60 | } 61 | 62 | /** 63 | * Get current cluster model in database. Note, this will only attempt a cluster calculation if there is no 64 | * cluster model saved in the database, use `forceUpdate` before this call if a recalculation is needed. 65 | * 66 | * @returns Cluster[] or null if no cluster model exist in database. 67 | * 68 | * @public 69 | */ 70 | export async function getClusters(): Promise { 71 | let clusterModel: DBClusterSchema | null = await ClusterModel.findOne({}); 72 | 73 | if (clusterModel === null) { 74 | // if no cluster model exist in database, force update and retry 75 | await updateClusters(); 76 | clusterModel = await ClusterModel.findOne({}); 77 | } 78 | 79 | return clusterModel === null ? null : clusterModel.clusters; 80 | } 81 | 82 | /** 83 | * Update cluster model. 84 | * 85 | * @private 86 | */ 87 | async function updateClusters(): Promise { 88 | const lastEndpoint: Endpoint & { _id: number } | null = await getLastEndpoint(); 89 | 90 | if (lastEndpoint !== null) { 91 | const clusterModel: DBClusterSchema & { _id: mongoose.Types.ObjectId; } | null = await ClusterModel.findOne({}); 92 | await recalculateClusters(lastEndpoint._id, clusterModel === null ? undefined : clusterModel._id); 93 | } 94 | } 95 | 96 | /** 97 | * Recalculates the clusters by getting all endpoint and determining clusters. Updates cluster recommendations 98 | * in database. 99 | * 100 | * @param lastEndpointId - _id of the last endpoint in database 101 | * @param clusterId - _id of the clusterModel in database to be replaced with new calculation. 102 | * @returns Promise of the last endpoint or null if no endpoints in collection 103 | * 104 | * @private 105 | */ 106 | async function recalculateClusters(lastEndpointId: number, clusterId?: mongoose.Types.ObjectId): Promise { 107 | const endpointBuckets: EndpointBuckets[] = await getAllEndpointBuckets(); 108 | const clusters: Cluster[] = determineClusters(endpointBuckets); 109 | 110 | await ClusterModel.findOneAndUpdate({ _id: clusterId }, { clusters, lastEndpointId }, { upsert: true }); 111 | } 112 | 113 | startWatchingClusterModel(); 114 | -------------------------------------------------------------------------------- /src/server/models/endpointBucketsModel.ts: -------------------------------------------------------------------------------- 1 | import mongoose from 'mongoose'; 2 | import { ChangeStream } from 'mongodb'; 3 | 4 | import { EndpointModel, Endpoint, getAllEndpoints } from './../../shared/models/endpointModel'; 5 | import { vectorizeEndpoints } from './../utils/endpoints'; 6 | import { Route } from './../../shared/types'; 7 | 8 | // TODO abstract so it can work with either MongoDB or PostgreSQL depending on how setup is called. 9 | 10 | /** 11 | * Interface defining a time-vectorized collection of server responses for a specific route. 12 | * 13 | * @property method - HTTP method type 14 | * @property endpoint - HTTP request relative endpoint 15 | * @property buckets - an array of numbers signifying the number of server responses that occurred in that time interval. 16 | * @property lastEndpointId - the _id of the last endpoints document used in the buckets calculation 17 | * @property oldestDate - UNIX timestamp of the oldest server response taken into account in buckets 18 | * @property newestDate - UNIX timestamp of the newest server response taken into account in buckets 19 | */ 20 | export type EndpointBuckets = { 21 | method: string, 22 | endpoint: string, 23 | buckets: number[], 24 | lastEndpointId: number, 25 | oldestDate: number, 26 | newestDate: number, 27 | }; 28 | 29 | const FORCED_UPDATE_TIMEOUT = 5 * 60 * 1000; 30 | export const NUM_DAILY_DIVISIONS = 24 * 2; 31 | export const MIN_NUM_CHANGES_TO_UPDATE = 100; 32 | let changeStream: ChangeStream> | null = null; 33 | let updateCounter: { [key: string]: number } = Object.create(null); 34 | 35 | // cache of active timeouts 36 | let timeoutHandles: { [key: string]: NodeJS.Timeout } = Object.create(null); 37 | 38 | 39 | const EndpointBucketsSchema = new mongoose.Schema({ 40 | method: { type: String, required: true }, 41 | endpoint: { type: String, required: true }, 42 | buckets: { type: [Number], required: true }, 43 | lastEndpointId: { type: Number, required: true }, 44 | oldestDate: { type: Number, required: true }, 45 | newestDate: { type: Number, required: true }, 46 | }); 47 | export const EndpointBucketsModel = mongoose.model('EndpointBuckets', EndpointBucketsSchema); 48 | 49 | /** 50 | * Close the ChangeStream watching EndpointModel. 51 | * 52 | * @public 53 | */ 54 | export async function stopWatchingEndpointModel(): Promise { 55 | // clear all timeouts 56 | for (const timeoutHandle of Object.values(timeoutHandles)) { 57 | clearTimeout(timeoutHandle); 58 | } 59 | timeoutHandles = Object.create(null); 60 | 61 | if (changeStream !== null) changeStream.close(); 62 | changeStream = null; 63 | } 64 | 65 | /** 66 | * Initiate the ChangeStream watching EndpointModel. 67 | * 68 | * @public 69 | */ 70 | export function startWatchingEndpointModel(): void { 71 | if (changeStream !== null) return; 72 | 73 | updateCounter = Object.create(null); 74 | changeStream = EndpointModel.watch(); 75 | 76 | changeStream.on('change', change => { 77 | if (change.operationType !== 'insert' || !change.fullDocument) return; 78 | 79 | const key: string = getKey(change.fullDocument.method, change.fullDocument.endpoint); 80 | updateCounter[key] ??= 0; 81 | updateCounter[key]++; 82 | if (updateCounter[key] === MIN_NUM_CHANGES_TO_UPDATE) { 83 | updateCounter[key] = 0; 84 | 85 | // remove timeout if it was already set 86 | if (timeoutHandles[key]) { 87 | clearTimeout(timeoutHandles[key]); 88 | delete timeoutHandles[key]; 89 | } 90 | 91 | updateEndpointBuckets(change.fullDocument.method, change.fullDocument.endpoint); 92 | } else { 93 | // set timeout if it does not already exist 94 | if (!timeoutHandles[key]) timeoutHandles[key] = setTimeout(() => { 95 | if (change.fullDocument) updateEndpointBuckets(change.fullDocument.method, change.fullDocument.endpoint); 96 | }, FORCED_UPDATE_TIMEOUT); 97 | } 98 | }); 99 | } 100 | 101 | /** 102 | * Get EndpointBuckets for a particular route from the database. Forces a calculation update if there are no endpoint buckets in the database. 103 | * 104 | * @param {String} method - HTTP method type 105 | * @param {String} endpoint - HTTP request relative endpoint 106 | * @returns EndpointBuckets for that route, or null if no data exists for that route 107 | * 108 | * @public 109 | */ 110 | export async function getEndpointBuckets(method: string, endpoint: string): Promise { 111 | let endpointBuckets: EndpointBuckets | null = await EndpointBucketsModel.findOne({ method, endpoint }); 112 | 113 | // if no data has been calculated, force an update and check again 114 | if (endpointBuckets === null) { 115 | await updateEndpointBuckets(method, endpoint); 116 | endpointBuckets = await EndpointBucketsModel.findOne({ method, endpoint }); 117 | } 118 | 119 | return endpointBuckets; 120 | } 121 | 122 | /** 123 | * Get all EndpointBuckets in the database. This does not force a calculation update. 124 | * 125 | * @public 126 | */ 127 | export async function getAllEndpointBuckets(): Promise { 128 | return await EndpointBucketsModel.find({}); 129 | } 130 | 131 | /** 132 | * Get current endpoint buckets, get endpoints pushed to database since last update, recalculate buckets and push update to database. 133 | * 134 | * @param {String} method - HTTP method type 135 | * @param {String} endpoint - HTTP request relative endpoint 136 | * 137 | * @private 138 | */ 139 | async function updateEndpointBuckets(method: string, endpoint: string): Promise { 140 | const endpointBuckets = await getBuckets(method, endpoint); 141 | if (endpointBuckets === null) return; 142 | 143 | await EndpointBucketsModel.findOneAndUpdate({ method, endpoint }, endpointBuckets, { upsert: true, new: true }); 144 | } 145 | 146 | /** 147 | * Forces all pending updates with timeout handle to complete. 148 | * 149 | * @public 150 | */ 151 | export async function forceAllPendingUpdates(): Promise { 152 | for (const key in timeoutHandles) { 153 | const { method, endpoint } = getRouteFromKey(key); 154 | clearTimeout(timeoutHandles[key]); 155 | delete timeoutHandles[key]; 156 | await updateEndpointBuckets(method, endpoint); 157 | } 158 | } 159 | 160 | /** 161 | * Calculate endpoint buckets for an array of server response endpoints. 162 | * 163 | * @param endpoints - Array of server response endpoints 164 | * @returns EndpointBuckets object for those endpoints. 165 | */ 166 | export function calculateEndpointBuckets(endpoints: Endpoint[]): EndpointBuckets { 167 | let lastEndpointId = 0, oldestDate: number = endpoints[0].callTime, newestDate: number = endpoints[0].callTime; 168 | endpoints.forEach(endpoint => { 169 | if (endpoint._id && endpoint._id > lastEndpointId) lastEndpointId = endpoint._id; 170 | if (endpoint.callTime < oldestDate) oldestDate = endpoint.callTime; 171 | if (endpoint.callTime > newestDate) newestDate = endpoint.callTime; 172 | }); 173 | 174 | const buckets: number[] = vectorizeEndpoints(endpoints, (24 * 60) / NUM_DAILY_DIVISIONS); 175 | 176 | return { method: endpoints[0].method, endpoint: endpoints[0].endpoint, buckets, lastEndpointId, oldestDate, newestDate }; 177 | } 178 | 179 | /** 180 | * Get a distinct list of distinct routes. 181 | * 182 | * @returns Promise of an array of Route objects 183 | * 184 | * @public 185 | */ 186 | export async function getDistinctRoutes(): Promise { 187 | return await EndpointBucketsModel.find({}, 'method endpoint'); 188 | } 189 | 190 | /** 191 | * Get current endpoint buckets, get endpoints pushed to database since last update, recalculate buckets and new last endpoint id. 192 | * 193 | * @param method - HTTP method type 194 | * @param endpoint - HTTP request relative endpoint 195 | * @returns Object with buckets, the last endpoint id, oldestDate, and newestDate in the calculation, or null if no endpoints exist for that route. 196 | * 197 | * @private 198 | */ 199 | async function getBuckets(method: string, endpoint: string): Promise { 200 | const endpointBuckets: EndpointBuckets | null = await EndpointBucketsModel.findOne({ method, endpoint }); 201 | 202 | // grab all matching endpoints 203 | const endpoints: Endpoint[] = await getAllEndpoints(method, endpoint, endpointBuckets !== null ? endpointBuckets.lastEndpointId : undefined); 204 | if (endpoints.length === 0) return null; 205 | 206 | const newDataEndpointBuckets = calculateEndpointBuckets(endpoints); 207 | 208 | if (endpointBuckets !== null) { 209 | newDataEndpointBuckets.oldestDate = endpointBuckets.oldestDate; 210 | // update bucket with new data 211 | newDataEndpointBuckets.buckets = newDataEndpointBuckets.buckets.map((numCalls: number, i: number) => endpointBuckets.buckets[i] + numCalls); 212 | } 213 | 214 | return newDataEndpointBuckets; 215 | } 216 | 217 | /** 218 | * Generate a unique string from the combination of method and endpoint. 219 | * 220 | * @param method - HTTP method type 221 | * @param endpoint - HTTP request relative endpoint 222 | * @returns string representing unique combination of method and endpoint 223 | * 224 | * @private 225 | */ 226 | function getKey(method: string, endpoint: string): string { 227 | return JSON.stringify({ method, endpoint }); 228 | } 229 | 230 | /** 231 | * Return a route from the unique combination key string. 232 | * 233 | * @param key unique combination string of method and endpoint generated by a call to `getKey`. 234 | * @returns Route 235 | * 236 | * @private 237 | */ 238 | function getRouteFromKey(key: string): Route { 239 | return JSON.parse(key); 240 | } 241 | 242 | startWatchingEndpointModel(); 243 | -------------------------------------------------------------------------------- /src/server/routes/apiRouter.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | 3 | import endpointRouter from './endpointRouter'; 4 | import clusterRouter from './clusterRouter'; 5 | 6 | const router = Router(); 7 | 8 | router.use('/graph/endpoint', endpointRouter); 9 | router.use('/graph/cluster', clusterRouter); 10 | 11 | export default router; 12 | -------------------------------------------------------------------------------- /src/server/routes/clusterRouter.ts: -------------------------------------------------------------------------------- 1 | import { Router, Request, Response, NextFunction } from 'express'; 2 | 3 | import { getClusterList, getClusterLoadGraphData, getClusterTreeGraphData } from './../controllers/dataController'; 4 | import { cache } from './../controllers/cacheController'; 5 | 6 | const router = Router(); 7 | 8 | // To get list of all clusters 9 | router.get('/', 10 | cache(30 * 1000), 11 | getClusterList, 12 | (req: Request, res: Response, next: NextFunction): void => { 13 | if (!res.locals.clusters) return next({ 14 | status: 500, 15 | error: 'Middleware getClusterList did not return expected data.' 16 | }); 17 | 18 | res.json(res.locals.clusters); 19 | } 20 | ); 21 | 22 | // To get cluster load graph data 23 | // clusterId is the index of the cluster in the array. 24 | router.get('/load/:clusterId', 25 | cache(30 * 1000), 26 | getClusterLoadGraphData, 27 | (req: Request, res: Response, next: NextFunction): void => { 28 | if (!res.locals.loadGraphData) return next({ 29 | status: 500, 30 | error: 'Middleware getClusterLoadGraphData did not return expected data.' 31 | }); 32 | 33 | res.json(res.locals.loadGraphData); 34 | } 35 | ); 36 | 37 | // To get cluster tree graph data 38 | router.get('/tree', 39 | cache(30 * 1000), 40 | getClusterTreeGraphData, 41 | (req: Request, res: Response, next: NextFunction): void => { 42 | if (!res.locals.treeGraphData) return next({ 43 | status: 500, 44 | error: 'Middleware getClusterTreeGraphData did not return expected data.' 45 | }); 46 | 47 | res.json(res.locals.treeGraphData); 48 | } 49 | ); 50 | 51 | export default router; 52 | -------------------------------------------------------------------------------- /src/server/routes/endpointRouter.ts: -------------------------------------------------------------------------------- 1 | import { Router, Request, Response, NextFunction } from 'express'; 2 | 3 | import { getEndpointList, getEndpointLoadGraphData } from './../controllers/dataController'; 4 | import { cache } from './../controllers/cacheController'; 5 | 6 | const router = Router(); 7 | 8 | // To get list of all endpoints 9 | router.get('/', 10 | cache(30 * 1000), 11 | getEndpointList, 12 | (req: Request, res: Response, next: NextFunction): void => { 13 | if (!res.locals.endpoints) return next({ 14 | status: 500, 15 | error: 'Middleware getEndpointList did not return expected data.' 16 | }); 17 | 18 | res.json(res.locals.endpoints); 19 | } 20 | ); 21 | 22 | // To get endpoint load line graph data 23 | router.get('/load', 24 | getEndpointLoadGraphData, 25 | (req: Request, res: Response, next: NextFunction): void => { 26 | if (!res.locals.loadGraphData) return next({ 27 | status: 500, 28 | error: 'Middleware getEndpointLoadGraphData did not return expected data.' 29 | }); 30 | 31 | res.json(res.locals.loadGraphData); 32 | } 33 | ); 34 | 35 | export default router; 36 | -------------------------------------------------------------------------------- /src/server/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "outDir": "./../../build", 4 | 5 | "module": "commonjs", 6 | "esModuleInterop": true, 7 | "target": "es6", 8 | "strict": true, 9 | "noImplicitAny": true, 10 | "strictNullChecks": true, 11 | "forceConsistentCasingInFileNames": true, 12 | "baseUrl": ".", 13 | "sourceMap": true, 14 | "resolveJsonModule": true, 15 | "paths": { 16 | "*": [ 17 | "node_modules/*" 18 | ] 19 | } 20 | }, 21 | "include": [ 22 | "./**/*", 23 | "../shared/**/*" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /src/server/utils/MiddlewareError.ts: -------------------------------------------------------------------------------- 1 | export default class MiddlewareError extends Error { 2 | status: number; 3 | error: string; 4 | 5 | constructor(errorMessage: string, status = 500) { 6 | super(errorMessage); 7 | 8 | this.error = errorMessage; 9 | this.status = status; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/server/utils/endpoints.ts: -------------------------------------------------------------------------------- 1 | import clustering from 'density-clustering'; 2 | 3 | import { Endpoint } from '../../shared/models/endpointModel'; 4 | import { EndpointBuckets } from '../../server/models/endpointBucketsModel'; 5 | import { Route, Cluster, LoadData, TreeNode, DataPoint } from './../../shared/types'; 6 | 7 | interface TimeDomainEndpoint extends Endpoint { 8 | hour: number; 9 | } 10 | 11 | /** 12 | * Utilize OPTICS algorithm to cluster endpoints based on covariant time utilization. 13 | * https://en.wikipedia.org/wiki/OPTICS_algorithm 14 | * 15 | * @param allEndpointBuckets - Array of endpoint buckets to use for clustering calculation. 16 | * @returns Array of Cluster recommendations. 17 | * 18 | * @public 19 | */ 20 | export function determineClusters(allEndpointBuckets: EndpointBuckets[]): Cluster[] { 21 | // vectorize endpoint array into 24 data points with number of calls in that hour. 22 | // same order as endpointBuckets 23 | const vectors: number[][] = []; 24 | let totalNumCalls = 0; 25 | for (const endpointBuckets of allEndpointBuckets) { 26 | totalNumCalls += endpointBuckets.buckets.reduce((sum, val) => sum + val, 0); 27 | vectors.push(endpointBuckets.buckets); 28 | } 29 | 30 | const analyzer = new clustering.OPTICS(); 31 | 32 | const averageCallsPerBucket = totalNumCalls / vectors[0].length; // use as neighborhood radius 33 | const result: number[][] = analyzer.run(vectors, averageCallsPerBucket, 1); 34 | 35 | return result.map(clusterIndices => clusterIndices.map((i): Route => ({ method: allEndpointBuckets[i].method, endpoint: allEndpointBuckets[i].endpoint }))); 36 | } 37 | 38 | /** 39 | * Generate DataPoints for a load data graph for a pre-processed route buckets vector. 40 | * 41 | * @param endpointBuckets - EndpointBuckets object for a particular route. 42 | * @param granularity - time interval in minutes between data points 43 | * @returns {LoadData} Graph data to construct load graph scatter plot. 44 | * 45 | * @public 46 | */ 47 | export function getLoadData(endpointBuckets: EndpointBuckets, granularity = 30): LoadData { 48 | const granularityInHours: number = granularity / 60; 49 | 50 | let numDays: number = (endpointBuckets.newestDate - endpointBuckets.oldestDate) / (24 * 60 * 60 * 1000); 51 | numDays = numDays === 0 ? 1 : Math.ceil(numDays); 52 | 53 | return endpointBuckets.buckets.map((numCalls: number, i: number): DataPoint => [granularityInHours * i, numCalls / numDays]); 54 | } 55 | 56 | /** 57 | * Create an array of buckets with the total number of endpoints that call into each bucket based on an interval size of granularity in minutes. 58 | * 59 | * @param endpoints - Array of Endpoint 60 | * @param granularity - time interval in minutes between data points 61 | * @returns array of buckets with the total number of endpoints that call into each bucket 62 | * 63 | * @public 64 | */ 65 | export function vectorizeEndpoints(endpoints: Endpoint[], granularity = 30): number[] { 66 | // break data into bucket 67 | const responses: TimeDomainEndpoint[] = endpoints.map(endpoint => { 68 | const date = new Date(endpoint.callTime); 69 | 70 | return { 71 | ...endpoint, 72 | hour: date.getHours() + (date.getMinutes() / 60) 73 | }; 74 | }); 75 | 76 | const vector: number[] = []; 77 | for (let hourStart = 0; hourStart < 24; hourStart += (granularity / 60)) { 78 | const numCalls: number = responses.filter(endpoint => endpoint.hour >= hourStart && endpoint.hour < hourStart + (granularity / 60)).length; 79 | 80 | vector.push(numCalls); 81 | } 82 | 83 | return vector; 84 | } 85 | 86 | /** 87 | * Generate a D3 compatible nested node object for graphing tree graphs (dendrogram). 88 | * 89 | * @param clusters - Array of Cluster recommendations. 90 | * @returns - TreeNode representation of cluster to be used with D3 dendrogram graph 91 | */ 92 | export function getClusterTreeNode(clusters: Cluster[]): TreeNode { 93 | const root: TreeNode = { 94 | name: 'Clusters', 95 | children: [] 96 | }; 97 | 98 | for (let i = 0; i < clusters.length; i++) { 99 | const clusterRoot: TreeNode = { 100 | name: 'Cluster ' + i, 101 | children: [] 102 | }; 103 | 104 | const cluster: Cluster = clusters[i]; 105 | const cache: { [key: string]: string[] } = Object.create(null); 106 | for (let j = 0; j < cluster.length; j++) { 107 | // check if cluster[j].method does not exist in cache object 108 | if (!cache[cluster[j].method]) cache[cluster[j].method] = []; 109 | // add method to cache object as key and the value as an empty array 110 | // push end point to array 111 | cache[cluster[j].method].push(cluster[j].endpoint); 112 | } 113 | 114 | for (const method in cache) { 115 | const methodCluster: TreeNode = { 116 | name: method, 117 | children: cache[method].map((endpoint: string): TreeNode => { 118 | return { 119 | name: endpoint 120 | }; 121 | }) 122 | }; 123 | 124 | clusterRoot.children?.push(methodCluster); 125 | } 126 | root.children?.push(clusterRoot); 127 | } 128 | 129 | return root; 130 | } 131 | -------------------------------------------------------------------------------- /src/server/utils/loadDePaulEndpointData.ts: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import fs, { promises as fsPromises } from 'fs'; 3 | 4 | import { logAllEndpoints, Endpoint } from '../../shared/models/endpointModel'; 5 | 6 | /** 7 | * Data from https://cds.cdm.depaul.edu/resources/datasets/ using Non-Preprocessed DePaul CTI Web Usage Data. 8 | * Download zip file, and extract cti-april02-log.txt to the root directory before running this function. 9 | * 10 | * @param batchSize - number of endpoints to send to database per transaction 11 | * @returns Returns undefined if cti-april02-log.txt file does not exist in root directory. 12 | * 13 | * @public 14 | */ 15 | export default async function loadDePaulEndpointData(batchSize = 5000): Promise { 16 | if (!fs.existsSync(path.resolve('cti-april02-log.txt'))) { 17 | console.error('cti-april02-log.txt not found in root directory. Please see documentation for loadDePaulEndpointData.'); 18 | return; 19 | } 20 | const data = await fsPromises.readFile(path.resolve('cti-april02-log.txt'), 'utf8'); 21 | 22 | const entries = data.split('\n').slice(4); 23 | 24 | //we now have an array of relevant data, but each entry is a string. We should split now based on spaces. We should use a for loop to iterate over each entry. 25 | const allEndpoints: Endpoint[] = []; 26 | for (const entry of entries) { 27 | const values: string[] = entry.split(' '); 28 | const timeStamp: number = new Date(values[0] + ' ' + values[1]).getTime(); 29 | 30 | if (isNaN(timeStamp) || !values[8] || !values[9]) continue; 31 | 32 | allEndpoints.push({ 33 | method: values[8], 34 | endpoint: values[9], 35 | callTime: timeStamp, 36 | }); 37 | } 38 | 39 | for (let begin = 0; begin < allEndpoints.length; begin += batchSize) { 40 | const batch = allEndpoints.slice(begin, begin + batchSize); 41 | await logAllEndpoints(batch); 42 | console.log(`Percent completed: ${(((begin + batchSize) / allEndpoints.length) * 100).toFixed(2)}`); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/shared/models/endpointModel.ts: -------------------------------------------------------------------------------- 1 | import mongoose, { QueryOptions } from 'mongoose'; 2 | import { autoIncrement } from 'mongoose-plugin-autoinc'; 3 | 4 | // TODO abstract so it can work with either MongoDB or PostgreSQL depending on how setup is called. 5 | 6 | /** 7 | * Interface defining a server response object type. 8 | * 9 | * @property method - HTTP method type 10 | * @property endpoint - HTTP request relative endpoint 11 | * @property callTime - UNIX timestamp of when the server received the request 12 | * @property _id - MongoDB unique id, will be a number for endpoints collection 13 | */ 14 | export interface Endpoint { method: string, endpoint: string, callTime: number, _id?: number } 15 | 16 | const EndpointSchema = new mongoose.Schema({ 17 | _id: { type: Number, required: true }, 18 | method: { type: String, required: true }, 19 | endpoint: { type: String, required: true }, 20 | callTime: { type: Number, required: true }, // unix timestamp 21 | }); 22 | // use auto incrementing _id of type Number 23 | EndpointSchema.plugin(autoIncrement, { 24 | model: 'Endpoint', 25 | startAt: 0, 26 | incrementBy: 1, 27 | }); 28 | // TODO add validation for call_time to be convertible to Date 29 | export const EndpointModel = mongoose.model('Endpoint', EndpointSchema); 30 | 31 | /** 32 | * Log an endpoint request data point to the database. 33 | * 34 | * @param {String} method - HTTP method type 35 | * @param {String} endpoint - HTTP request relative endpoint 36 | * @param {number} callTime - UNIX timestamp of when request first communicated with the server 37 | * 38 | * @public 39 | */ 40 | export async function logEndpoint(method: string, endpoint: string, callTime: number): Promise { 41 | // TODO validate inputs 42 | 43 | try { 44 | await EndpointModel.create({ 45 | method, 46 | endpoint, 47 | callTime, 48 | }); 49 | } catch (error) { 50 | console.error('\n\nERROR LOGGING RESPONSE TO DATABASE'); 51 | console.error(error); 52 | console.log('\n\n'); 53 | } 54 | } 55 | 56 | /** 57 | * Log an array of server response data points to the database. 58 | * 59 | * @param {Endpoint[]} endpoints - Array of endpoints to be added to database. 60 | * 61 | * @public 62 | */ 63 | export async function logAllEndpoints(endpoints: Endpoint[]): Promise { 64 | // TODO validate inputs 65 | 66 | try { 67 | await EndpointModel.insertMany(endpoints); 68 | } catch (error) { 69 | console.error('\n\nERROR LOGGING RESPONSES TO DATABASE - LOG MANY'); 70 | console.error(error); 71 | console.log('\n\n'); 72 | } 73 | } 74 | 75 | /** 76 | * Get a list of all endpoints. If no method or endpoint is specified, it will return all endpoints in the database. 77 | * 78 | * @param {string} method - (optional) HTTP method 79 | * @param {string} endpoint - (optional) HTTP request relative endpoint 80 | * @param {number} afterId - (optional) _id of EndpointModel used to filter result to include only _id greater than this value 81 | * @returns Promise of array of endpoints 82 | * 83 | * @public 84 | */ 85 | export async function getAllEndpoints(method?: string, endpoint?: string, afterId?: number): Promise { 86 | const query: QueryOptions = {}; 87 | if (method) query.method = method; 88 | if (endpoint) query.endpoint = endpoint; 89 | if (afterId) query._id = { $gt: afterId }; 90 | 91 | return await EndpointModel.find(query); 92 | } 93 | 94 | 95 | /** 96 | * Get the last endpoint. If no endpoints exist in the database, it will return null. 97 | * 98 | * @returns Promise of the last endpoint or null if no endpoints in collection 99 | * 100 | * @public 101 | */ 102 | export async function getLastEndpoint(): Promise { 103 | return await EndpointModel.findOne().sort({ _id: -1 }); 104 | } 105 | -------------------------------------------------------------------------------- /src/shared/models/mongoSetup.ts: -------------------------------------------------------------------------------- 1 | import mongoose from 'mongoose'; 2 | 3 | let isConnected = false; 4 | 5 | /** 6 | * Connect to MongoDB if a connection has not already been established. Used for lazy connecting. 7 | * 8 | * @param {string} mongoURI - URI to connect to database. 9 | * 10 | * @public 11 | */ 12 | export async function connect(mongoURI: string): Promise { 13 | if (!mongoURI.match(/mongodb/i)) throw new Error(`Invalid MongoDB URI: ${mongoURI}`); 14 | 15 | if (!isConnected) { 16 | isConnected = true; 17 | await mongoose.connect(mongoURI); 18 | } 19 | } 20 | 21 | /** 22 | * Disconnect from MongoDB. 23 | * 24 | * @public 25 | */ 26 | export async function disconnect(): Promise { 27 | if (isConnected) { 28 | await mongoose.disconnect(); 29 | isConnected = false; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/shared/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A cluster is an array of Route objects. It represents a recommended clustering of server routes. 3 | */ 4 | export type Cluster = Route[]; 5 | 6 | /** 7 | * Defined a server endpoint as the combination of the HTTP method and the relative URL. 8 | * 9 | * @property method - HTTP method type 10 | * @property endpoint - HTTP request relative endpoint 11 | */ 12 | export type Route = { 13 | method: string; 14 | endpoint: string; 15 | }; 16 | 17 | /** 18 | * Array of [number, number] tuples specifying the x and y values respectively of a scatter plot. 19 | */ 20 | export type LoadData = DataPoint[]; 21 | 22 | export type DataPoint = [number, number]; 23 | 24 | export type TreeNode = { 25 | name: string; 26 | children?: TreeNode[]; 27 | }; 28 | 29 | export type ClientError = { 30 | error: string; 31 | } 32 | -------------------------------------------------------------------------------- /src/shared/utils/dataGenerator.ts: -------------------------------------------------------------------------------- 1 | import { Endpoint } from './../models/endpointModel'; 2 | 3 | /** 4 | * A distribution function. A simple one-to-one mapping. 5 | * 6 | * @param {number} x - the independent variable 7 | * @returns {number} - the probability, a value between 0 and 1 8 | */ 9 | export type DistributionFunction = (x: number) => number; 10 | 11 | /** 12 | * An object specifying a route and a probability distribution function. 13 | * 14 | * @property method - HTTP method type 15 | * @property endpoint - HTTP request relative endpoint 16 | * @property pdf - A probability distribution function that should return the probability (a number between 0 and 1) at a particular independent value. 17 | */ 18 | export type EndpointPDF = { 19 | method: string; 20 | endpoint: string; 21 | pdf: DistributionFunction; 22 | } 23 | 24 | /** 25 | * Simulate the server responding to client calls at the specified endpoints. Algorithm is probabilistic. You specify the probability distribution function (pdf) as a function 26 | * for each endpoint in a 24 hour period as well as the overall server call distribution in a 24 hour period and calls are made randomly taking into account the pdf of each 27 | * endpoint. 28 | * 29 | * @param {EndpointPDF[]} endpoints - Array of EndpointPDF which include method, endpoint, and pdf. 30 | * @param {DistributionFunction} numCallsDist - Distribution function of number of calls server received over a 24 hour period. 31 | * @param {number} numDays - Number of days to run simulation for. 32 | * @param {number} granularity - Granularity, in minutes, of internal calculation period. Defaults to 30 minutes. Smaller values mean approximation is closer to continuous 33 | * pdf since the algorithm uses trapezoid rule numerical integration. Warning of using too small granularity: if the number of calls for a granularity interval sums to less 34 | * than one, no calls will be made for the period. 35 | * 36 | * @returns Array of Endpoints that simulate how a real-life server would have responded given the endpoint bias in pdf. 37 | * 38 | * @public 39 | */ 40 | export function simulateServerResponses(endpoints: EndpointPDF[], numCallsDist: DistributionFunction, numDays: number, granularity = 30): Endpoint[] { 41 | if ((24 * 60) % granularity !== 0) throw new RangeError('Granularity does not divide a day.'); 42 | 43 | const dayStartTime: Date = new Date(new Date().toLocaleDateString()); 44 | 45 | const simulatedCalls: Endpoint[] = []; 46 | for (let i = 0; i < numDays; i++) { 47 | simulatedCalls.push(...simulateSingleDayResponses(endpoints, numCallsDist, granularity, dayStartTime)); 48 | 49 | // Increment the day start time by one day 50 | dayStartTime.setDate(dayStartTime.getDate() + 1); 51 | } 52 | 53 | return simulatedCalls; 54 | } 55 | 56 | /** 57 | * Simulate the server responding to client calls for a period of 24 hours. 58 | * 59 | * @param {EndpointPDF} endpoints - Array of EndpointPDF which include method, endpoint, and pdf. 60 | * @param {DistributionFunction} numCallsDist - Distribution function of number of calls server received over a 24 hour period. 61 | * @param {number} granularity - Granularity, in minutes, of internal calculation period. Defaults to 30 minutes. Smaller values means approximation is closer to continuos pdf 62 | * since algorithm uses trapezoid rule numerical integration. Warning of using too small granularity: if number of calls for a granularity interval sums to less than one, no 63 | * calls will be made for period. 64 | * @param {Date} dayStart - Date indicating start of 24 hour period to be used as a full day. 65 | * 66 | * @returns Array of Endpoints that simulate how a real life server would have responded given the endpoint bias in pdf. 67 | * 68 | * @private 69 | */ 70 | function simulateSingleDayResponses(endpoints: EndpointPDF[], numCallsDist: DistributionFunction, granularity: number, dayStart: Date): Endpoint[] { 71 | const numIntervals = (24 * 60) / granularity; 72 | const intervalStartTime = new Date(dayStart); 73 | 74 | const dayResponses: Endpoint[] = []; 75 | for (let i = 0; i < numIntervals; i++) { 76 | const intervalStartHour: number = intervalStartTime.getHours() + (intervalStartTime.getMinutes() / 60); 77 | const intervalEndHour: number = intervalStartTime.getHours() + ((intervalStartTime.getMinutes() + granularity) / 60); 78 | 79 | // determine number of server calls in interval 80 | const numServerCalls: number = Math.floor(integrate(numCallsDist, intervalStartHour, intervalEndHour)); 81 | 82 | // build random endpoint selection array 83 | // each endpoint will be the value for 100 * # endpoint * probability indices in the array 84 | const endpointSelectionArray: EndpointPDF[] = endpoints.reduce( 85 | (selectionArray: EndpointPDF[], endpoint: EndpointPDF): EndpointPDF[] => { 86 | // endpoints probability as the interval's midpoint probability 87 | const probability = endpoint.pdf((intervalStartHour + intervalEndHour) / 2); 88 | 89 | const numIndices = Math.round(100 * endpoints.length * probability); 90 | for (let i = 0; i < numIndices; i++) selectionArray.push(endpoint); 91 | 92 | return selectionArray; 93 | }, []); 94 | 95 | const intervalResponses: Endpoint[] = Array.from({ length: numServerCalls }, (): Endpoint => { 96 | const endpoint: EndpointPDF = endpointSelectionArray[Math.floor(Math.random() * endpointSelectionArray.length)]; 97 | 98 | // pick time from random uniform distribution in interval 99 | const callTime = new Date(intervalStartTime); 100 | callTime.setMinutes(callTime.getMinutes() + Math.round(Math.random() * granularity)); 101 | 102 | return { 103 | method: endpoint.method, 104 | endpoint: endpoint.endpoint, 105 | callTime: callTime.getTime(), 106 | }; 107 | }); 108 | 109 | dayResponses.push(...intervalResponses); 110 | 111 | // Increment interval start time by the granularity 112 | intervalStartTime.setMinutes(intervalStartTime.getMinutes() + granularity); 113 | } 114 | 115 | return dayResponses; 116 | } 117 | 118 | /** 119 | * Numerically calculated approximation to the definite integral by trapezoid rule. 120 | * 121 | * @param {DistributionFunction} fn - Function to be integrated between intervalStartTime and intervalEndTime. 122 | * @param {number} intervalStartTime - Position on the x-axis at which to begin integration. 123 | * @param {number} intervalEndTime - Position on the x-aix at which to end integration. 124 | * 125 | * @returns {number} - area under curve between start and stop. 126 | * 127 | * @private 128 | */ 129 | function integrate(fn: DistributionFunction, intervalStartTime: number, intervalEndTime: number): number { 130 | return 0.5 * (intervalEndTime - intervalStartTime) * (fn(intervalStartTime) + fn(intervalEndTime)); 131 | } 132 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-require-imports */ 2 | 3 | const path = require('path'); 4 | 5 | const HtmlWebpackPlugin = require('html-webpack-plugin'); 6 | const MiniCssExtractPlugin = require('mini-css-extract-plugin'); 7 | const CssMinimizerPlugin = require('css-minimizer-webpack-plugin'); 8 | 9 | module.exports = { 10 | entry: { 11 | index: './src/client/index.tsx', 12 | }, 13 | output: { 14 | filename: 'scripts/[name].bundle.js', 15 | path: path.resolve(__dirname, 'build/client'), 16 | }, 17 | mode: process.env.NODE_ENV || 'production', 18 | plugins: [ 19 | new HtmlWebpackPlugin({ 20 | filename: 'index.html', 21 | template: './src/client/assets/index.html', 22 | //favicon: "./client/assets/icons/favicon.png" 23 | }), 24 | new HtmlWebpackPlugin({ 25 | filename: '404.html', 26 | template: './src/client/assets/404.html', 27 | inject: false, 28 | //favicon: "./client/assets/icons/favicon.png" 29 | }), 30 | new MiniCssExtractPlugin({ 31 | filename: 'styles/[name].css' 32 | }), 33 | ], 34 | module: { 35 | rules: [ 36 | { 37 | test: /\.tsx?$/, 38 | exclude: /node_modules/, 39 | use: { 40 | loader: 'babel-loader', 41 | options: { 42 | presets: [ 43 | '@babel/preset-env', 44 | '@babel/preset-react', 45 | '@babel/preset-typescript' 46 | ], 47 | plugins: [ 48 | [ 49 | '@babel/plugin-transform-runtime', 50 | { 51 | 'regenerator': true 52 | } 53 | ] 54 | ] 55 | } 56 | } 57 | }, 58 | { 59 | test: /\.s[ac]ss$/i, 60 | use: [ 61 | process.env.NODE_ENV === 'production' ? MiniCssExtractPlugin.loader : 'style-loader', 62 | 'css-loader', 63 | 'sass-loader' 64 | ], 65 | }, 66 | { 67 | test: /\.css$/i, 68 | use: [ 69 | process.env.NODE_ENV === 'production' ? MiniCssExtractPlugin.loader : 'style-loader', 70 | 'css-loader' 71 | ], 72 | }, 73 | { 74 | test: /\.(gif|png|jpe?g|svg)$/i, 75 | use: [ 76 | { 77 | loader: 'file-loader', 78 | options: { 79 | name: 'images/[hash].[ext]', 80 | }, 81 | }, 82 | { 83 | loader: 'image-webpack-loader', 84 | options: { 85 | // Options to lower image quality for faster network exchanges 86 | //webp: { 87 | // quality: 30, 88 | //} 89 | name: 'images/[hash].[ext]', 90 | }, 91 | }, 92 | ], 93 | }, 94 | ] 95 | }, 96 | optimization: { 97 | minimizer: [ 98 | '...', 99 | new CssMinimizerPlugin(), 100 | ], 101 | }, 102 | resolve: { 103 | extensions: ['.ts', '.tsx', '.js', '.svg', '.png'] 104 | }, 105 | devServer: { 106 | publicPath: '/', 107 | proxy: { 108 | '/api': { target: 'http://localhost:3000' }, 109 | }, 110 | hot: true, 111 | }, 112 | }; 113 | -------------------------------------------------------------------------------- /whitebg.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oslabs-beta/gimbap/6a46ff5ccea5e2b6ae4b0d78e701f8fec2c011c6/whitebg.jpg --------------------------------------------------------------------------------