├── .env.example ├── .gitignore ├── .htmlnanorc.js ├── .nvmrc ├── .prettierrc ├── LICENSE ├── README.md ├── apollo.config.example.js ├── common ├── package.json └── types │ ├── DoesStoryExist.ts │ ├── GetAllStoryIds.ts │ ├── GetFirstHnStoryId.ts │ ├── GetGithubUser.ts │ ├── GetHnUser.ts │ ├── GetLastHnStoryId.ts │ ├── GetLatestStoryIds.ts │ ├── GetNumberOfStories.ts │ ├── GetStories.ts │ ├── GetTwitterUser.ts │ ├── GetWebsite.ts │ ├── InsertGithubUser.ts │ ├── InsertHnUser.ts │ ├── InsertStory.ts │ ├── InsertTwitterUser.ts │ ├── InsertWebsite.ts │ ├── UpdateGithubUser.ts │ ├── UpdateHnUser.ts │ ├── UpdateStory.ts │ ├── UpdateStoryScore.ts │ ├── UpdateTwitterUser.ts │ ├── UpdateWebsite.ts │ └── globalTypes.ts ├── docker-compose.yaml ├── hasura_metadata.json ├── hasura_pg_dump.sql ├── netlify.toml ├── package.json ├── scraper ├── .eslintrc.js ├── .prettierrc ├── package.json ├── src │ ├── api.ts │ ├── cli │ │ ├── get-show-hn-stories.ts │ │ ├── newest-scraper.ts │ │ ├── past-scraper.ts │ │ ├── rescrap.ts │ │ ├── scrap-stories.ts │ │ ├── scrap-story.ts │ │ └── update-stories-scores.ts │ ├── functions │ │ ├── scraper.ts │ │ └── update-stories-scores.ts │ ├── graph.ts │ ├── helpers.ts │ └── scraper.ts └── tsconfig.json ├── web ├── .eslintrc.js ├── .postcssrc ├── package.json ├── public │ └── index.html ├── src │ ├── components │ │ ├── App.tsx │ │ ├── Filters.tsx │ │ ├── SocialLink.tsx │ │ ├── StoryList.tsx │ │ ├── StorySummary.tsx │ │ └── WebsiteIcon.tsx │ ├── index.tsx │ ├── nhost.ts │ ├── styles.css │ └── types.ts ├── tailwind.config.js └── tsconfig.json └── yarn.lock /.env.example: -------------------------------------------------------------------------------- 1 | # Twitter API 2 | TWITTER_API_KEY=your_twitter_api_key 3 | TWITTER_API_SECRET_KEY=your_twitter_secret_key 4 | TWITTER_API_BEARER_TOKEN=your_twitter_api_bearer_token 5 | 6 | # GitHub API 7 | GITHUB_USERNAME=your_github_username 8 | GITHUB_PERSONAL_ACCESS_TOKEN=your_github_access_token 9 | 10 | # Nhost 11 | HBP_BASE_URL=http://localhost:3000 12 | HASURA_GRAPHQL_URL=http://localhost:8080/v1/graphql 13 | HASURA_GRAPHQL_ADMIN_SECRET=supersecret 14 | S3_SECRET_ACCESS_KEY=supers3secret 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | yarn-error.log 3 | .env 4 | .parcel-cache 5 | .cache 6 | dist 7 | apollo.config.js 8 | hn_stories*.txt -------------------------------------------------------------------------------- /.htmlnanorc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | minifySvg: false, 3 | } 4 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v15.0.1 -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true, 4 | "proseWrap": "always" 5 | } 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Show HN projects 2 | 3 | A webapp to browse projects posted on 4 | [Show HN](https://news.ycombinator.com/show). The application is available at 5 | [showhn-dashboard.netlify.app](https://showhn-dashboard.netlify.app/), but you 6 | can also run it locally or on your own server if you want to. Note that you will 7 | need a Twitter and GitHub APIs access. 8 | 9 | The project is composed of: 10 | 11 | - A backend containing a PostgreSQL database and a GraphQL API. It is handled by 12 | [Nhost](https://nhost.io/), which uses [Hasura](https://hasura.io/). 13 | - A Node.js application to scrap the projects (using HN, Twitter and GitHub APIs 14 | and some web scraping). The application provides serverless functions and CLI 15 | tools. 16 | - A React web application that connects to the GraphQL API to display the 17 | projects. 18 | 19 | ## Running the backend 20 | 21 | ### Option 1: Nhost account 22 | 23 | Create a project on [Nhost](https://nhost.io/) (they offer a 30-days free 24 | trial), and note: 25 | 26 | - the GraphQL API and backend URLs (on the Dashboard tab) 27 | - the admin secret (on the Hasura tab) 28 | 29 | Copy the _.env.example_ file to create _.env_, and replace the values in there. 30 | 31 | ### Option 2: local Docker container 32 | 33 | Copy the _.env.example_ file to create _.env_, and replace the values in there. 34 | The default URLs (Nhost section) should be the right ones for your local 35 | installation. 36 | 37 | Run `yarn docker:start` (it will use Docker-compose) to run the PostgreSQL 38 | database and Hasura GraphQL server locally. 39 | 40 | The NHost console will be available at http://localhost:8080, the secret is the 41 | value of `HASURA_GRAPHQL_ADMIN_SECRET` in your _.env_ file. 42 | 43 | ### Final steps 44 | 45 | - Connect to the NHost console 46 | - Import the PostgreSQL dump in _hasura_pg_dump.sql_ (_Data_ tab, then _SQL_ in 47 | the sidebar, copy-paste the file content) 48 | - Import the Hasura metatata in _hasura_metadata.json_ (settings icon at 49 | top-right, then _Import metadata_) 50 | 51 | ## Running the frontend 52 | 53 | Run `yarn web:run:dev`. 54 | 55 | ## Running the scrapper 56 | 57 | In the _scraper_ directory (`cd scraper`): 58 | 59 | - `yarn run:dev src/cli/newest-scraper.ts`: scraps the latest Show HN stories, 60 | i.e. the ones you can see on HN in the _Show_ section. 61 | - `yarn run:dev src/cli/update-stories-score.ts`: updates all the stories score 62 | by refetching it from the HN API. 63 | - `yarn run:dev src/cli/rescrap.ts`: updates all stories existing in the 64 | database by rescraping them (useful in case the scraper was updated). 65 | 66 | ## Work in progress 🏗 67 | 68 | As you can see this README is very minimal. There is a lot of information to put 69 | in there. Don’t hesitate to ask me any information you need, while I’m working 70 | on improving this documentation. 71 | -------------------------------------------------------------------------------- /apollo.config.example.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | module.exports = { 3 | client: { 4 | service: { 5 | name: 'nhost', 6 | url: 'http://localhost:8080/v1/graphql', 7 | headers: { 8 | 'x-hasura-admin-secret': 'supersecret', 9 | }, 10 | }, 11 | includes: ['./scraper/src/**/*.ts', './web/src/**/*.{ts,tsx}'], 12 | excludes: [], 13 | }, 14 | } 15 | -------------------------------------------------------------------------------- /common/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sidep-common", 3 | "version": "1.0.0", 4 | "license": "MIT" 5 | } 6 | -------------------------------------------------------------------------------- /common/types/DoesStoryExist.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: DoesStoryExist 8 | // ==================================================== 9 | 10 | export interface DoesStoryExist_hn_stories_by_pk { 11 | __typename: "hn_stories"; 12 | id: number; 13 | } 14 | 15 | export interface DoesStoryExist { 16 | /** 17 | * fetch data from the table: "hn_stories" using primary key columns 18 | */ 19 | hn_stories_by_pk: DoesStoryExist_hn_stories_by_pk | null; 20 | } 21 | 22 | export interface DoesStoryExistVariables { 23 | id: number; 24 | } 25 | -------------------------------------------------------------------------------- /common/types/GetAllStoryIds.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: GetAllStoryIds 8 | // ==================================================== 9 | 10 | export interface GetAllStoryIds_hn_stories { 11 | __typename: "hn_stories"; 12 | id: number; 13 | } 14 | 15 | export interface GetAllStoryIds { 16 | /** 17 | * fetch data from the table: "hn_stories" 18 | */ 19 | hn_stories: GetAllStoryIds_hn_stories[]; 20 | } 21 | -------------------------------------------------------------------------------- /common/types/GetFirstHnStoryId.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: GetFirstHnStoryId 8 | // ==================================================== 9 | 10 | export interface GetFirstHnStoryId_hn_stories { 11 | __typename: "hn_stories"; 12 | id: number; 13 | } 14 | 15 | export interface GetFirstHnStoryId { 16 | /** 17 | * fetch data from the table: "hn_stories" 18 | */ 19 | hn_stories: GetFirstHnStoryId_hn_stories[]; 20 | } 21 | -------------------------------------------------------------------------------- /common/types/GetGithubUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: GetGithubUser 8 | // ==================================================== 9 | 10 | export interface GetGithubUser_github_users { 11 | __typename: "github_users"; 12 | username: string; 13 | name: string | null; 14 | nb_followers: any; 15 | website: string | null; 16 | twitter_username: string | null; 17 | location: string | null; 18 | email: string | null; 19 | profile_image_url: string | null; 20 | } 21 | 22 | export interface GetGithubUser { 23 | /** 24 | * fetch data from the table: "github_users" 25 | */ 26 | github_users: GetGithubUser_github_users[]; 27 | } 28 | 29 | export interface GetGithubUserVariables { 30 | username: string; 31 | } 32 | -------------------------------------------------------------------------------- /common/types/GetHnUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: GetHnUser 8 | // ==================================================== 9 | 10 | export interface GetHnUser_hn_users { 11 | __typename: "hn_users"; 12 | username: string; 13 | karma: any; 14 | } 15 | 16 | export interface GetHnUser { 17 | /** 18 | * fetch data from the table: "hn_users" 19 | */ 20 | hn_users: GetHnUser_hn_users[]; 21 | } 22 | 23 | export interface GetHnUserVariables { 24 | username: string; 25 | } 26 | -------------------------------------------------------------------------------- /common/types/GetLastHnStoryId.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: GetLastHnStoryId 8 | // ==================================================== 9 | 10 | export interface GetLastHnStoryId_hn_stories { 11 | __typename: "hn_stories"; 12 | id: number; 13 | } 14 | 15 | export interface GetLastHnStoryId { 16 | /** 17 | * fetch data from the table: "hn_stories" 18 | */ 19 | hn_stories: GetLastHnStoryId_hn_stories[]; 20 | } 21 | -------------------------------------------------------------------------------- /common/types/GetLatestStoryIds.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: GetLatestStoryIds 8 | // ==================================================== 9 | 10 | export interface GetLatestStoryIds_hn_stories { 11 | __typename: "hn_stories"; 12 | id: number; 13 | score: any; 14 | } 15 | 16 | export interface GetLatestStoryIds { 17 | /** 18 | * fetch data from the table: "hn_stories" 19 | */ 20 | hn_stories: GetLatestStoryIds_hn_stories[]; 21 | } 22 | 23 | export interface GetLatestStoryIdsVariables { 24 | minDate?: any | null; 25 | } 26 | -------------------------------------------------------------------------------- /common/types/GetNumberOfStories.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | import { hn_stories_bool_exp } from "./globalTypes"; 7 | 8 | // ==================================================== 9 | // GraphQL subscription operation: GetNumberOfStories 10 | // ==================================================== 11 | 12 | export interface GetNumberOfStories_hn_stories_aggregate_aggregate { 13 | __typename: "hn_stories_aggregate_fields"; 14 | count: number | null; 15 | } 16 | 17 | export interface GetNumberOfStories_hn_stories_aggregate { 18 | __typename: "hn_stories_aggregate"; 19 | aggregate: GetNumberOfStories_hn_stories_aggregate_aggregate | null; 20 | } 21 | 22 | export interface GetNumberOfStories { 23 | /** 24 | * fetch aggregated fields from the table: "hn_stories" 25 | */ 26 | hn_stories_aggregate: GetNumberOfStories_hn_stories_aggregate; 27 | } 28 | 29 | export interface GetNumberOfStoriesVariables { 30 | filters?: hn_stories_bool_exp | null; 31 | } 32 | -------------------------------------------------------------------------------- /common/types/GetStories.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | import { hn_stories_bool_exp } from "./globalTypes"; 7 | 8 | // ==================================================== 9 | // GraphQL subscription operation: GetStories 10 | // ==================================================== 11 | 12 | export interface GetStories_hn_stories_hn_user { 13 | __typename: "hn_users"; 14 | username: string; 15 | karma: any; 16 | } 17 | 18 | export interface GetStories_hn_stories_website_twitter_user { 19 | __typename: "twitter_users"; 20 | username: string; 21 | name: string | null; 22 | website: string | null; 23 | nb_followers: any; 24 | location: string | null; 25 | profile_image_url: string | null; 26 | } 27 | 28 | export interface GetStories_hn_stories_website_github_user { 29 | __typename: "github_users"; 30 | username: string; 31 | name: string | null; 32 | website: string | null; 33 | nb_followers: any; 34 | location: string | null; 35 | email: string | null; 36 | profile_image_url: string | null; 37 | } 38 | 39 | export interface GetStories_hn_stories_website { 40 | __typename: "websites"; 41 | url: string; 42 | title: string | null; 43 | description: string | null; 44 | /** 45 | * An object relationship 46 | */ 47 | twitter_user: GetStories_hn_stories_website_twitter_user | null; 48 | /** 49 | * An object relationship 50 | */ 51 | github_user: GetStories_hn_stories_website_github_user | null; 52 | } 53 | 54 | export interface GetStories_hn_stories { 55 | __typename: "hn_stories"; 56 | id: number; 57 | title: string; 58 | score: any; 59 | date: any; 60 | /** 61 | * An object relationship 62 | */ 63 | hn_user: GetStories_hn_stories_hn_user | null; 64 | /** 65 | * An object relationship 66 | */ 67 | website: GetStories_hn_stories_website | null; 68 | } 69 | 70 | export interface GetStories { 71 | /** 72 | * fetch data from the table: "hn_stories" 73 | */ 74 | hn_stories: GetStories_hn_stories[]; 75 | } 76 | 77 | export interface GetStoriesVariables { 78 | filters?: hn_stories_bool_exp | null; 79 | limit?: number | null; 80 | } 81 | -------------------------------------------------------------------------------- /common/types/GetTwitterUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: GetTwitterUser 8 | // ==================================================== 9 | 10 | export interface GetTwitterUser_twitter_users { 11 | __typename: "twitter_users"; 12 | username: string; 13 | name: string | null; 14 | nb_followers: any; 15 | website: string | null; 16 | location: string | null; 17 | profile_image_url: string | null; 18 | } 19 | 20 | export interface GetTwitterUser { 21 | /** 22 | * fetch data from the table: "twitter_users" 23 | */ 24 | twitter_users: GetTwitterUser_twitter_users[]; 25 | } 26 | 27 | export interface GetTwitterUserVariables { 28 | username: string; 29 | } 30 | -------------------------------------------------------------------------------- /common/types/GetWebsite.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL query operation: GetWebsite 8 | // ==================================================== 9 | 10 | export interface GetWebsite_websites { 11 | __typename: "websites"; 12 | url: string; 13 | title: string | null; 14 | description: string | null; 15 | twitter_username: string | null; 16 | github_username: string | null; 17 | } 18 | 19 | export interface GetWebsite { 20 | /** 21 | * fetch data from the table: "websites" 22 | */ 23 | websites: GetWebsite_websites[]; 24 | } 25 | 26 | export interface GetWebsiteVariables { 27 | url: string; 28 | } 29 | -------------------------------------------------------------------------------- /common/types/InsertGithubUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: InsertGithubUser 8 | // ==================================================== 9 | 10 | export interface InsertGithubUser_insert_github_users_one { 11 | __typename: "github_users"; 12 | username: string; 13 | name: string | null; 14 | website: string | null; 15 | nb_followers: any; 16 | twitter_username: string | null; 17 | location: string | null; 18 | email: string | null; 19 | profile_image_url: string | null; 20 | } 21 | 22 | export interface InsertGithubUser { 23 | /** 24 | * insert a single row into the table: "github_users" 25 | */ 26 | insert_github_users_one: InsertGithubUser_insert_github_users_one | null; 27 | } 28 | 29 | export interface InsertGithubUserVariables { 30 | username: string; 31 | name?: string | null; 32 | website?: string | null; 33 | nb_followers: any; 34 | twitter_username?: string | null; 35 | location?: string | null; 36 | email?: string | null; 37 | profile_image_url?: string | null; 38 | } 39 | -------------------------------------------------------------------------------- /common/types/InsertHnUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: InsertHnUser 8 | // ==================================================== 9 | 10 | export interface InsertHnUser_insert_hn_users_one { 11 | __typename: "hn_users"; 12 | username: string; 13 | karma: any; 14 | } 15 | 16 | export interface InsertHnUser { 17 | /** 18 | * insert a single row into the table: "hn_users" 19 | */ 20 | insert_hn_users_one: InsertHnUser_insert_hn_users_one | null; 21 | } 22 | 23 | export interface InsertHnUserVariables { 24 | username: string; 25 | karma: any; 26 | } 27 | -------------------------------------------------------------------------------- /common/types/InsertStory.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: InsertStory 8 | // ==================================================== 9 | 10 | export interface InsertStory_insert_hn_stories_one_hn_user { 11 | __typename: "hn_users"; 12 | username: string; 13 | karma: any; 14 | } 15 | 16 | export interface InsertStory_insert_hn_stories_one_website_twitter_user { 17 | __typename: "twitter_users"; 18 | username: string; 19 | name: string | null; 20 | website: string | null; 21 | nb_followers: any; 22 | } 23 | 24 | export interface InsertStory_insert_hn_stories_one_website_github_user { 25 | __typename: "github_users"; 26 | username: string; 27 | name: string | null; 28 | website: string | null; 29 | nb_followers: any; 30 | } 31 | 32 | export interface InsertStory_insert_hn_stories_one_website { 33 | __typename: "websites"; 34 | url: string; 35 | title: string | null; 36 | description: string | null; 37 | /** 38 | * An object relationship 39 | */ 40 | twitter_user: InsertStory_insert_hn_stories_one_website_twitter_user | null; 41 | /** 42 | * An object relationship 43 | */ 44 | github_user: InsertStory_insert_hn_stories_one_website_github_user | null; 45 | } 46 | 47 | export interface InsertStory_insert_hn_stories_one { 48 | __typename: "hn_stories"; 49 | id: number; 50 | title: string; 51 | score: any; 52 | date: any; 53 | /** 54 | * An object relationship 55 | */ 56 | hn_user: InsertStory_insert_hn_stories_one_hn_user | null; 57 | /** 58 | * An object relationship 59 | */ 60 | website: InsertStory_insert_hn_stories_one_website | null; 61 | } 62 | 63 | export interface InsertStory { 64 | /** 65 | * insert a single row into the table: "hn_stories" 66 | */ 67 | insert_hn_stories_one: InsertStory_insert_hn_stories_one | null; 68 | } 69 | 70 | export interface InsertStoryVariables { 71 | id: number; 72 | date: any; 73 | hn_username?: string | null; 74 | website_url?: string | null; 75 | score: any; 76 | title: string; 77 | } 78 | -------------------------------------------------------------------------------- /common/types/InsertTwitterUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: InsertTwitterUser 8 | // ==================================================== 9 | 10 | export interface InsertTwitterUser_insert_twitter_users_one { 11 | __typename: "twitter_users"; 12 | username: string; 13 | name: string | null; 14 | website: string | null; 15 | nb_followers: any; 16 | location: string | null; 17 | profile_image_url: string | null; 18 | } 19 | 20 | export interface InsertTwitterUser { 21 | /** 22 | * insert a single row into the table: "twitter_users" 23 | */ 24 | insert_twitter_users_one: InsertTwitterUser_insert_twitter_users_one | null; 25 | } 26 | 27 | export interface InsertTwitterUserVariables { 28 | username: string; 29 | name?: string | null; 30 | website?: string | null; 31 | nb_followers: any; 32 | location?: string | null; 33 | profile_image_url?: string | null; 34 | } 35 | -------------------------------------------------------------------------------- /common/types/InsertWebsite.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: InsertWebsite 8 | // ==================================================== 9 | 10 | export interface InsertWebsite_insert_websites_one { 11 | __typename: "websites"; 12 | url: string; 13 | title: string | null; 14 | description: string | null; 15 | twitter_username: string | null; 16 | github_username: string | null; 17 | } 18 | 19 | export interface InsertWebsite { 20 | /** 21 | * insert a single row into the table: "websites" 22 | */ 23 | insert_websites_one: InsertWebsite_insert_websites_one | null; 24 | } 25 | 26 | export interface InsertWebsiteVariables { 27 | url: string; 28 | title?: string | null; 29 | description?: string | null; 30 | twitter_username?: string | null; 31 | github_username?: string | null; 32 | } 33 | -------------------------------------------------------------------------------- /common/types/UpdateGithubUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: UpdateGithubUser 8 | // ==================================================== 9 | 10 | export interface UpdateGithubUser_update_github_users { 11 | __typename: "github_users_mutation_response"; 12 | /** 13 | * number of affected rows by the mutation 14 | */ 15 | affected_rows: number; 16 | } 17 | 18 | export interface UpdateGithubUser { 19 | /** 20 | * update data of the table: "github_users" 21 | */ 22 | update_github_users: UpdateGithubUser_update_github_users | null; 23 | } 24 | 25 | export interface UpdateGithubUserVariables { 26 | username: string; 27 | name?: string | null; 28 | website?: string | null; 29 | nb_followers: any; 30 | twitter_username?: string | null; 31 | location?: string | null; 32 | email?: string | null; 33 | profile_image_url?: string | null; 34 | } 35 | -------------------------------------------------------------------------------- /common/types/UpdateHnUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: UpdateHnUser 8 | // ==================================================== 9 | 10 | export interface UpdateHnUser_update_hn_users { 11 | __typename: "hn_users_mutation_response"; 12 | /** 13 | * number of affected rows by the mutation 14 | */ 15 | affected_rows: number; 16 | } 17 | 18 | export interface UpdateHnUser { 19 | /** 20 | * update data of the table: "hn_users" 21 | */ 22 | update_hn_users: UpdateHnUser_update_hn_users | null; 23 | } 24 | 25 | export interface UpdateHnUserVariables { 26 | username: string; 27 | karma: any; 28 | } 29 | -------------------------------------------------------------------------------- /common/types/UpdateStory.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: UpdateStory 8 | // ==================================================== 9 | 10 | export interface UpdateStory_update_hn_stories { 11 | __typename: "hn_stories_mutation_response"; 12 | /** 13 | * number of affected rows by the mutation 14 | */ 15 | affected_rows: number; 16 | } 17 | 18 | export interface UpdateStory { 19 | /** 20 | * update data of the table: "hn_stories" 21 | */ 22 | update_hn_stories: UpdateStory_update_hn_stories | null; 23 | } 24 | 25 | export interface UpdateStoryVariables { 26 | id: number; 27 | date: any; 28 | hn_username?: string | null; 29 | website_url?: string | null; 30 | score: any; 31 | title: string; 32 | } 33 | -------------------------------------------------------------------------------- /common/types/UpdateStoryScore.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: UpdateStoryScore 8 | // ==================================================== 9 | 10 | export interface UpdateStoryScore_update_hn_stories { 11 | __typename: "hn_stories_mutation_response"; 12 | /** 13 | * number of affected rows by the mutation 14 | */ 15 | affected_rows: number; 16 | } 17 | 18 | export interface UpdateStoryScore { 19 | /** 20 | * update data of the table: "hn_stories" 21 | */ 22 | update_hn_stories: UpdateStoryScore_update_hn_stories | null; 23 | } 24 | 25 | export interface UpdateStoryScoreVariables { 26 | storyId: number; 27 | score: any; 28 | } 29 | -------------------------------------------------------------------------------- /common/types/UpdateTwitterUser.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: UpdateTwitterUser 8 | // ==================================================== 9 | 10 | export interface UpdateTwitterUser_update_twitter_users { 11 | __typename: "twitter_users_mutation_response"; 12 | /** 13 | * number of affected rows by the mutation 14 | */ 15 | affected_rows: number; 16 | } 17 | 18 | export interface UpdateTwitterUser { 19 | /** 20 | * update data of the table: "twitter_users" 21 | */ 22 | update_twitter_users: UpdateTwitterUser_update_twitter_users | null; 23 | } 24 | 25 | export interface UpdateTwitterUserVariables { 26 | username: string; 27 | name?: string | null; 28 | website?: string | null; 29 | nb_followers: any; 30 | location?: string | null; 31 | profile_image_url?: string | null; 32 | } 33 | -------------------------------------------------------------------------------- /common/types/UpdateWebsite.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | // ==================================================== 7 | // GraphQL mutation operation: UpdateWebsite 8 | // ==================================================== 9 | 10 | export interface UpdateWebsite_update_websites { 11 | __typename: "websites_mutation_response"; 12 | /** 13 | * number of affected rows by the mutation 14 | */ 15 | affected_rows: number; 16 | } 17 | 18 | export interface UpdateWebsite { 19 | /** 20 | * update data of the table: "websites" 21 | */ 22 | update_websites: UpdateWebsite_update_websites | null; 23 | } 24 | 25 | export interface UpdateWebsiteVariables { 26 | url: string; 27 | title?: string | null; 28 | description?: string | null; 29 | twitter_username?: string | null; 30 | github_username?: string | null; 31 | } 32 | -------------------------------------------------------------------------------- /common/types/globalTypes.ts: -------------------------------------------------------------------------------- 1 | /* tslint:disable */ 2 | /* eslint-disable */ 3 | // @generated 4 | // This file was automatically generated and should not be edited. 5 | 6 | //============================================================== 7 | // START Enums and Input Objects 8 | //============================================================== 9 | 10 | /** 11 | * expression to compare columns of type Int. All fields are combined with logical 'AND'. 12 | */ 13 | export interface Int_comparison_exp { 14 | _eq?: number | null; 15 | _gt?: number | null; 16 | _gte?: number | null; 17 | _in?: number[] | null; 18 | _is_null?: boolean | null; 19 | _lt?: number | null; 20 | _lte?: number | null; 21 | _neq?: number | null; 22 | _nin?: number[] | null; 23 | } 24 | 25 | /** 26 | * expression to compare columns of type String. All fields are combined with logical 'AND'. 27 | */ 28 | export interface String_comparison_exp { 29 | _eq?: string | null; 30 | _gt?: string | null; 31 | _gte?: string | null; 32 | _ilike?: string | null; 33 | _in?: string[] | null; 34 | _is_null?: boolean | null; 35 | _like?: string | null; 36 | _lt?: string | null; 37 | _lte?: string | null; 38 | _neq?: string | null; 39 | _nilike?: string | null; 40 | _nin?: string[] | null; 41 | _nlike?: string | null; 42 | _nsimilar?: string | null; 43 | _similar?: string | null; 44 | } 45 | 46 | /** 47 | * Boolean expression to filter rows from the table "github_users". All fields are combined with a logical 'AND'. 48 | */ 49 | export interface github_users_bool_exp { 50 | _and?: (github_users_bool_exp | null)[] | null; 51 | _not?: github_users_bool_exp | null; 52 | _or?: (github_users_bool_exp | null)[] | null; 53 | email?: String_comparison_exp | null; 54 | location?: String_comparison_exp | null; 55 | name?: String_comparison_exp | null; 56 | nb_followers?: numeric_comparison_exp | null; 57 | profile_image_url?: String_comparison_exp | null; 58 | twitter_username?: String_comparison_exp | null; 59 | username?: String_comparison_exp | null; 60 | website?: String_comparison_exp | null; 61 | websites?: websites_bool_exp | null; 62 | } 63 | 64 | /** 65 | * Boolean expression to filter rows from the table "hn_stories". All fields are combined with a logical 'AND'. 66 | */ 67 | export interface hn_stories_bool_exp { 68 | _and?: (hn_stories_bool_exp | null)[] | null; 69 | _not?: hn_stories_bool_exp | null; 70 | _or?: (hn_stories_bool_exp | null)[] | null; 71 | date?: timestamptz_comparison_exp | null; 72 | hn_user?: hn_users_bool_exp | null; 73 | hn_username?: String_comparison_exp | null; 74 | id?: Int_comparison_exp | null; 75 | score?: numeric_comparison_exp | null; 76 | title?: String_comparison_exp | null; 77 | website?: websites_bool_exp | null; 78 | website_url?: String_comparison_exp | null; 79 | } 80 | 81 | /** 82 | * Boolean expression to filter rows from the table "hn_users". All fields are combined with a logical 'AND'. 83 | */ 84 | export interface hn_users_bool_exp { 85 | _and?: (hn_users_bool_exp | null)[] | null; 86 | _not?: hn_users_bool_exp | null; 87 | _or?: (hn_users_bool_exp | null)[] | null; 88 | hn_stories?: hn_stories_bool_exp | null; 89 | karma?: numeric_comparison_exp | null; 90 | username?: String_comparison_exp | null; 91 | } 92 | 93 | /** 94 | * expression to compare columns of type numeric. All fields are combined with logical 'AND'. 95 | */ 96 | export interface numeric_comparison_exp { 97 | _eq?: any | null; 98 | _gt?: any | null; 99 | _gte?: any | null; 100 | _in?: any[] | null; 101 | _is_null?: boolean | null; 102 | _lt?: any | null; 103 | _lte?: any | null; 104 | _neq?: any | null; 105 | _nin?: any[] | null; 106 | } 107 | 108 | /** 109 | * expression to compare columns of type timestamptz. All fields are combined with logical 'AND'. 110 | */ 111 | export interface timestamptz_comparison_exp { 112 | _eq?: any | null; 113 | _gt?: any | null; 114 | _gte?: any | null; 115 | _in?: any[] | null; 116 | _is_null?: boolean | null; 117 | _lt?: any | null; 118 | _lte?: any | null; 119 | _neq?: any | null; 120 | _nin?: any[] | null; 121 | } 122 | 123 | /** 124 | * Boolean expression to filter rows from the table "twitter_users". All fields are combined with a logical 'AND'. 125 | */ 126 | export interface twitter_users_bool_exp { 127 | _and?: (twitter_users_bool_exp | null)[] | null; 128 | _not?: twitter_users_bool_exp | null; 129 | _or?: (twitter_users_bool_exp | null)[] | null; 130 | location?: String_comparison_exp | null; 131 | name?: String_comparison_exp | null; 132 | nb_followers?: numeric_comparison_exp | null; 133 | profile_image_url?: String_comparison_exp | null; 134 | username?: String_comparison_exp | null; 135 | website?: String_comparison_exp | null; 136 | websites?: websites_bool_exp | null; 137 | } 138 | 139 | /** 140 | * Boolean expression to filter rows from the table "websites". All fields are combined with a logical 'AND'. 141 | */ 142 | export interface websites_bool_exp { 143 | _and?: (websites_bool_exp | null)[] | null; 144 | _not?: websites_bool_exp | null; 145 | _or?: (websites_bool_exp | null)[] | null; 146 | description?: String_comparison_exp | null; 147 | github_user?: github_users_bool_exp | null; 148 | github_username?: String_comparison_exp | null; 149 | hn_stories?: hn_stories_bool_exp | null; 150 | title?: String_comparison_exp | null; 151 | twitter_user?: twitter_users_bool_exp | null; 152 | twitter_username?: String_comparison_exp | null; 153 | url?: String_comparison_exp | null; 154 | } 155 | 156 | //============================================================== 157 | // END Enums and Input Objects 158 | //============================================================== 159 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.6' 2 | services: 3 | postgres: 4 | image: postgres:12 5 | ports: 6 | - '5432:5432' 7 | restart: always 8 | environment: 9 | POSTGRES_PASSWORD: postgrespassword 10 | volumes: 11 | - db_data:/var/lib/postgresql/data 12 | graphql-engine: 13 | image: hasura/graphql-engine:v1.2.1 14 | ports: 15 | - '8080:8080' 16 | depends_on: 17 | - 'postgres' 18 | restart: always 19 | environment: 20 | HASURA_GRAPHQL_DATABASE_URL: postgres://postgres:postgrespassword@postgres:5432/postgres 21 | HASURA_GRAPHQL_ADMIN_SECRET: '$HASURA_GRAPHQL_ADMIN_SECRET' 22 | HASURA_GRAPHQL_JWT_SECRET: '{"type": "RS256", "jwk_url": "http://hasura-backend-plus:3000/auth/jwks"}' 23 | HASURA_GRAPHQL_ENABLE_CONSOLE: 'true' 24 | HASURA_GRAPHQL_UNAUTHORIZED_ROLE: 'public' 25 | hasura-backend-plus: 26 | image: nhost/hasura-backend-plus:latest 27 | ports: 28 | - '3000:3000' 29 | environment: 30 | SERVER_URL: http://localhost:3000 31 | HASURA_ENDPOINT: http://graphql-engine:8080/v1/graphql 32 | HASURA_GRAPHQL_ADMIN_SECRET: '$HASURA_GRAPHQL_ADMIN_SECRET' 33 | JWT_ALGORITHM: RS256 34 | S3_ENDPOINT: http://minio:9000 35 | S3_BUCKET: hasura-backend-plus 36 | S3_ACCESS_KEY_ID: minio_access_key 37 | S3_SECRET_ACCESS_KEY: '${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY}' 38 | AUTO_MIGRATE: 'true' 39 | minio: 40 | image: minio/minio 41 | restart: always 42 | environment: 43 | S3_BUCKET: hasura-backend-plus 44 | MINIO_ACCESS_KEY: minio_access_key 45 | MINIO_SECRET_KEY: '${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY}' ## min 8 character 46 | entrypoint: sh 47 | command: "-c 'mkdir -p /export/hasura-backend-plus && /usr/bin/minio server /export'" 48 | volumes: 49 | - 'minio_data:/data' 50 | volumes: 51 | db_data: 52 | minio_data: 53 | -------------------------------------------------------------------------------- /hasura_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 2, 3 | "tables": [ 4 | { 5 | "table": { "schema": "auth", "name": "account_providers" }, 6 | "object_relationships": [ 7 | { 8 | "name": "account", 9 | "using": { "foreign_key_constraint_on": "account_id" } 10 | }, 11 | { 12 | "name": "provider", 13 | "using": { "foreign_key_constraint_on": "auth_provider" } 14 | } 15 | ] 16 | }, 17 | { 18 | "table": { "schema": "auth", "name": "account_roles" }, 19 | "object_relationships": [ 20 | { 21 | "name": "account", 22 | "using": { "foreign_key_constraint_on": "account_id" } 23 | }, 24 | { 25 | "name": "roleByRole", 26 | "using": { "foreign_key_constraint_on": "role" } 27 | } 28 | ] 29 | }, 30 | { 31 | "table": { "schema": "auth", "name": "accounts" }, 32 | "object_relationships": [ 33 | { 34 | "name": "role", 35 | "using": { "foreign_key_constraint_on": "default_role" } 36 | }, 37 | { "name": "user", "using": { "foreign_key_constraint_on": "user_id" } } 38 | ], 39 | "array_relationships": [ 40 | { 41 | "name": "account_providers", 42 | "using": { 43 | "foreign_key_constraint_on": { 44 | "column": "account_id", 45 | "table": { "schema": "auth", "name": "account_providers" } 46 | } 47 | } 48 | }, 49 | { 50 | "name": "account_roles", 51 | "using": { 52 | "foreign_key_constraint_on": { 53 | "column": "account_id", 54 | "table": { "schema": "auth", "name": "account_roles" } 55 | } 56 | } 57 | }, 58 | { 59 | "name": "refresh_tokens", 60 | "using": { 61 | "foreign_key_constraint_on": { 62 | "column": "account_id", 63 | "table": { "schema": "auth", "name": "refresh_tokens" } 64 | } 65 | } 66 | } 67 | ] 68 | }, 69 | { 70 | "table": { "schema": "auth", "name": "providers" }, 71 | "array_relationships": [ 72 | { 73 | "name": "account_providers", 74 | "using": { 75 | "foreign_key_constraint_on": { 76 | "column": "auth_provider", 77 | "table": { "schema": "auth", "name": "account_providers" } 78 | } 79 | } 80 | } 81 | ] 82 | }, 83 | { 84 | "table": { "schema": "auth", "name": "refresh_tokens" }, 85 | "object_relationships": [ 86 | { 87 | "name": "account", 88 | "using": { "foreign_key_constraint_on": "account_id" } 89 | } 90 | ] 91 | }, 92 | { 93 | "table": { "schema": "auth", "name": "roles" }, 94 | "array_relationships": [ 95 | { 96 | "name": "account_roles", 97 | "using": { 98 | "foreign_key_constraint_on": { 99 | "column": "role", 100 | "table": { "schema": "auth", "name": "account_roles" } 101 | } 102 | } 103 | }, 104 | { 105 | "name": "accounts", 106 | "using": { 107 | "foreign_key_constraint_on": { 108 | "column": "default_role", 109 | "table": { "schema": "auth", "name": "accounts" } 110 | } 111 | } 112 | } 113 | ] 114 | }, 115 | { 116 | "table": { "schema": "public", "name": "github_users" }, 117 | "array_relationships": [ 118 | { 119 | "name": "websites", 120 | "using": { 121 | "foreign_key_constraint_on": { 122 | "column": "github_username", 123 | "table": { "schema": "public", "name": "websites" } 124 | } 125 | } 126 | } 127 | ], 128 | "select_permissions": [ 129 | { 130 | "role": "public", 131 | "permission": { 132 | "columns": [ 133 | "email", 134 | "location", 135 | "name", 136 | "nb_followers", 137 | "profile_image_url", 138 | "twitter_username", 139 | "username", 140 | "website" 141 | ], 142 | "filter": {} 143 | } 144 | } 145 | ] 146 | }, 147 | { 148 | "table": { "schema": "public", "name": "hn_stories" }, 149 | "object_relationships": [ 150 | { 151 | "name": "hn_user", 152 | "using": { "foreign_key_constraint_on": "hn_username" } 153 | }, 154 | { 155 | "name": "website", 156 | "using": { "foreign_key_constraint_on": "website_url" } 157 | } 158 | ], 159 | "select_permissions": [ 160 | { 161 | "role": "public", 162 | "permission": { 163 | "columns": [ 164 | "date", 165 | "hn_username", 166 | "id", 167 | "score", 168 | "title", 169 | "website_url" 170 | ], 171 | "filter": {}, 172 | "allow_aggregations": true 173 | } 174 | } 175 | ] 176 | }, 177 | { 178 | "table": { "schema": "public", "name": "hn_users" }, 179 | "array_relationships": [ 180 | { 181 | "name": "hn_stories", 182 | "using": { 183 | "foreign_key_constraint_on": { 184 | "column": "hn_username", 185 | "table": { "schema": "public", "name": "hn_stories" } 186 | } 187 | } 188 | } 189 | ], 190 | "select_permissions": [ 191 | { 192 | "role": "public", 193 | "permission": { "columns": ["karma", "username"], "filter": {} } 194 | } 195 | ] 196 | }, 197 | { 198 | "table": { "schema": "public", "name": "twitter_users" }, 199 | "array_relationships": [ 200 | { 201 | "name": "websites", 202 | "using": { 203 | "foreign_key_constraint_on": { 204 | "column": "twitter_username", 205 | "table": { "schema": "public", "name": "websites" } 206 | } 207 | } 208 | } 209 | ], 210 | "select_permissions": [ 211 | { 212 | "role": "public", 213 | "permission": { 214 | "columns": [ 215 | "location", 216 | "name", 217 | "nb_followers", 218 | "profile_image_url", 219 | "username", 220 | "website" 221 | ], 222 | "filter": {} 223 | } 224 | } 225 | ] 226 | }, 227 | { 228 | "table": { "schema": "public", "name": "users" }, 229 | "object_relationships": [ 230 | { 231 | "name": "account", 232 | "using": { 233 | "manual_configuration": { 234 | "remote_table": { "schema": "auth", "name": "accounts" }, 235 | "column_mapping": { "id": "user_id" } 236 | } 237 | } 238 | } 239 | ] 240 | }, 241 | { 242 | "table": { "schema": "public", "name": "websites" }, 243 | "object_relationships": [ 244 | { 245 | "name": "github_user", 246 | "using": { "foreign_key_constraint_on": "github_username" } 247 | }, 248 | { 249 | "name": "twitter_user", 250 | "using": { "foreign_key_constraint_on": "twitter_username" } 251 | } 252 | ], 253 | "array_relationships": [ 254 | { 255 | "name": "hn_stories", 256 | "using": { 257 | "foreign_key_constraint_on": { 258 | "column": "website_url", 259 | "table": { "schema": "public", "name": "hn_stories" } 260 | } 261 | } 262 | } 263 | ], 264 | "select_permissions": [ 265 | { 266 | "role": "public", 267 | "permission": { 268 | "columns": [ 269 | "description", 270 | "github_username", 271 | "title", 272 | "twitter_username", 273 | "url" 274 | ], 275 | "filter": {} 276 | } 277 | } 278 | ] 279 | } 280 | ] 281 | } 282 | -------------------------------------------------------------------------------- /hasura_pg_dump.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE IF EXISTS ONLY public.websites DROP CONSTRAINT IF EXISTS websites_twitter_user_fkey; 2 | ALTER TABLE IF EXISTS ONLY public.websites DROP CONSTRAINT IF EXISTS websites_github_username_fkey; 3 | ALTER TABLE IF EXISTS ONLY public.hn_stories DROP CONSTRAINT IF EXISTS hn_stories_website_url_fkey; 4 | ALTER TABLE IF EXISTS ONLY public.hn_stories DROP CONSTRAINT IF EXISTS hn_stories_username_fkey; 5 | ALTER TABLE IF EXISTS ONLY public.websites DROP CONSTRAINT IF EXISTS websites_pkey; 6 | ALTER TABLE IF EXISTS ONLY public.twitter_users DROP CONSTRAINT IF EXISTS twitter_users_pkey; 7 | ALTER TABLE IF EXISTS ONLY public.hn_users DROP CONSTRAINT IF EXISTS hn_users_pkey; 8 | ALTER TABLE IF EXISTS ONLY public.hn_stories DROP CONSTRAINT IF EXISTS hn_stories_pkey; 9 | ALTER TABLE IF EXISTS ONLY public.github_users DROP CONSTRAINT IF EXISTS github_users_pkey; 10 | DROP TABLE IF EXISTS public.websites; 11 | DROP TABLE IF EXISTS public.twitter_users; 12 | DROP TABLE IF EXISTS public.hn_users; 13 | DROP TABLE IF EXISTS public.hn_stories; 14 | DROP TABLE IF EXISTS public.github_users; 15 | CREATE TABLE public.github_users ( 16 | username text NOT NULL, 17 | name text, 18 | website text, 19 | nb_followers numeric NOT NULL, 20 | twitter_username text, 21 | location text, 22 | email text, 23 | profile_image_url text 24 | ); 25 | CREATE TABLE public.hn_stories ( 26 | id integer NOT NULL, 27 | title text NOT NULL, 28 | date timestamp with time zone NOT NULL, 29 | score numeric NOT NULL, 30 | hn_username text, 31 | website_url text 32 | ); 33 | CREATE TABLE public.hn_users ( 34 | username text NOT NULL, 35 | karma numeric NOT NULL 36 | ); 37 | CREATE TABLE public.twitter_users ( 38 | username text NOT NULL, 39 | name text, 40 | website text, 41 | nb_followers numeric NOT NULL, 42 | location text, 43 | profile_image_url text 44 | ); 45 | CREATE TABLE public.websites ( 46 | url text NOT NULL, 47 | title text, 48 | description text, 49 | twitter_username text, 50 | github_username text 51 | ); 52 | ALTER TABLE ONLY public.github_users 53 | ADD CONSTRAINT github_users_pkey PRIMARY KEY (username); 54 | ALTER TABLE ONLY public.hn_stories 55 | ADD CONSTRAINT hn_stories_pkey PRIMARY KEY (id); 56 | ALTER TABLE ONLY public.hn_users 57 | ADD CONSTRAINT hn_users_pkey PRIMARY KEY (username); 58 | ALTER TABLE ONLY public.twitter_users 59 | ADD CONSTRAINT twitter_users_pkey PRIMARY KEY (username); 60 | ALTER TABLE ONLY public.websites 61 | ADD CONSTRAINT websites_pkey PRIMARY KEY (url); 62 | ALTER TABLE ONLY public.hn_stories 63 | ADD CONSTRAINT hn_stories_username_fkey FOREIGN KEY (hn_username) REFERENCES public.hn_users(username) ON UPDATE CASCADE ON DELETE CASCADE; 64 | ALTER TABLE ONLY public.hn_stories 65 | ADD CONSTRAINT hn_stories_website_url_fkey FOREIGN KEY (website_url) REFERENCES public.websites(url) ON UPDATE CASCADE ON DELETE SET DEFAULT; 66 | ALTER TABLE ONLY public.websites 67 | ADD CONSTRAINT websites_github_username_fkey FOREIGN KEY (github_username) REFERENCES public.github_users(username) ON UPDATE CASCADE ON DELETE SET DEFAULT; 68 | ALTER TABLE ONLY public.websites 69 | ADD CONSTRAINT websites_twitter_user_fkey FOREIGN KEY (twitter_username) REFERENCES public.twitter_users(username) ON UPDATE CASCADE ON DELETE SET DEFAULT; 70 | -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | command = "yarn web:build && yarn scraper:build" 3 | functions = "./scraper/dist/functions" 4 | publish = "./web/dist" -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sidep", 3 | "private": true, 4 | "workspaces": [ 5 | "common", 6 | "scraper", 7 | "web" 8 | ], 9 | "scripts": { 10 | "docker:start": "docker-compose up -d", 11 | "web:run:dev": "yarn workspace sidep-web run:dev", 12 | "web:build": "yarn workspace sidep-web build", 13 | "scraper:run:dev": "yarn workspace sidep-scraper run:dev", 14 | "scraper:build": "yarn workspace sidep-scraper build", 15 | "apollo:gentypes": "apollo codegen:generate --target typescript --tagName=gql --outputFlat common/types", 16 | "start": "netlify dev", 17 | "hasura:pg_dump": "curl -X POST http://localhost:8080/v1alpha1/pg_dump -H 'x-hasura-admin-secret: supersecret' -d '{\"opts\":[\"-t\", \"hn_stories\", \"-t\", \"websites\", \"-t\", \"hn_users\", \"-t\", \"github_users\", \"-t\", \"twitter_users\", \"-c\", \"--if-exists\", \"-O\", \"-x\", \"--schema-only\", \"--schema\", \"public\"],\"clean_output\":true}' -H 'Content-Type: application/json'", 18 | "hasura:export_metadata": "curl -X POST http://localhost:8080/v1/query -H 'x-hasura-admin-secret: supersecret' -d '{\"type\":\"export_metadata\",\"args\":{}}' -H 'Content-Type: application/json'" 19 | }, 20 | "devDependencies": { 21 | "apollo": "^2.31.1", 22 | "netlify-cli": "^2.68.5" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /scraper/.eslintrc.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | module.exports = { 3 | ignorePatterns: ['dist/**'], 4 | env: { 5 | browser: true, 6 | es2021: true, 7 | }, 8 | extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'], 9 | parser: '@typescript-eslint/parser', 10 | parserOptions: { 11 | ecmaVersion: 12, 12 | sourceType: 'module', 13 | }, 14 | plugins: ['@typescript-eslint'], 15 | rules: { 16 | '@typescript-eslint/explicit-module-boundary-types': 0, 17 | '@typescript-eslint/no-non-null-assertion': 0, 18 | }, 19 | } 20 | -------------------------------------------------------------------------------- /scraper/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "singleQuote": true 4 | } -------------------------------------------------------------------------------- /scraper/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sidep-scraper", 3 | "version": "1.0.0", 4 | "license": "MIT", 5 | "scripts": { 6 | "build": "tsc", 7 | "run:dev": "DOTENV_CONFIG_PATH=../.env ts-node -r dotenv/config" 8 | }, 9 | "devDependencies": { 10 | "@types/aws-lambda": "^8.10.64", 11 | "@types/node-fetch": "^2.5.7", 12 | "@typescript-eslint/eslint-plugin": "^4.7.0", 13 | "@typescript-eslint/parser": "^4.7.0", 14 | "aws-lambda": "^1.0.6", 15 | "dotenv": "^8.2.0", 16 | "eslint": "^7.13.0", 17 | "ts-node": "^9.0.0", 18 | "typescript": "^4.0.5" 19 | }, 20 | "dependencies": { 21 | "@types/cheerio": "^0.22.22", 22 | "apollo-boost": "^0.4.9", 23 | "cheerio": "^1.0.0-rc.3", 24 | "firebase": "^8.0.2", 25 | "graphql": "^15.4.0", 26 | "graphql-tag": "^2.11.0", 27 | "node-fetch": "^2.6.1", 28 | "sidep-common": "^1.0.0" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /scraper/src/api.ts: -------------------------------------------------------------------------------- 1 | import firebase from 'firebase/app' 2 | import 'firebase/database' 3 | import fetch from 'node-fetch' 4 | import cheerio from 'cheerio' 5 | import { GetTwitterUser_twitter_users } from 'sidep-common/types/GetTwitterUser' 6 | import { GetGithubUser_github_users } from 'sidep-common/types/GetGithubUser' 7 | import { InsertWebsiteVariables } from 'sidep-common/types/InsertWebsite' 8 | import { InsertStoryVariables } from 'sidep-common/types/InsertStory' 9 | import { GetHnUser_hn_users } from 'sidep-common/types/GetHnUser' 10 | import { decodeEntities, URL_REGEXP } from './helpers' 11 | 12 | export type HnUser = Omit 13 | export type GithubUser = Omit 14 | export type TwitterUser = Omit 15 | export type Website = InsertWebsiteVariables 16 | export type HnStory = InsertStoryVariables 17 | 18 | firebase.initializeApp({ 19 | databaseURL: 'https://hacker-news.firebaseio.com', 20 | }) 21 | 22 | const getSnapshotValue = async (path: string) => { 23 | const snapshot = await firebase.database().ref(path).once('value') 24 | return snapshot.val() as T 25 | } 26 | 27 | export const getHnUserInfo = async (username: string): Promise => { 28 | const hnUserInfo = await getSnapshotValue<{ 29 | id: string 30 | karma: number 31 | created: number 32 | submitted: number[] 33 | about: string 34 | }>(`v0/user/${username}`) 35 | if (!hnUserInfo) { 36 | throw new Error(`Error fetching info for HN user ${username}`) 37 | } 38 | return { username: hnUserInfo.id, karma: hnUserInfo.karma } 39 | } 40 | 41 | export const getHnStoryIds = async () => [ 42 | // FIXME eliminate duplicates 43 | ...(await getSnapshotValue('v0/newstories')), 44 | ...(await getSnapshotValue('v0/showstories')), 45 | ] 46 | 47 | export const getMaxHnItemId = async () => getSnapshotValue('v0/maxitem') 48 | 49 | const cleanText = (s: string | undefined) => s?.trim().replace(/\s+/g, ' ') 50 | 51 | export const getGithubUserInfo = async ( 52 | username: string 53 | ): Promise => { 54 | const userInfo = await ( 55 | await fetch(`https://api.github.com/users/${username}`, { 56 | headers: { 57 | Authorization: `Basic ${Buffer.from( 58 | `${process.env.GITHUB_USERNAME}:${process.env.GITHUB_PERSONAL_ACCESS_TOKEN}` 59 | ).toString('base64')}`, 60 | }, 61 | }) 62 | ).json() 63 | if (!userInfo) { 64 | throw new Error(`Error getting the Github info for user ${username}`) 65 | } 66 | 67 | let twitterUsername = userInfo.twitter_username 68 | 69 | if (!twitterUsername && userInfo?.blog?.match(/twitter.com/i)) { 70 | const [, username] = 71 | userInfo?.blog.match(/twitter.com\/([a-z0-9_]+)/i) || [] 72 | if (username) { 73 | twitterUsername = username 74 | } 75 | } 76 | 77 | return { 78 | username: username, 79 | name: userInfo.name, 80 | website: userInfo.blog, 81 | nb_followers: userInfo.followers, 82 | twitter_username: twitterUsername, 83 | location: userInfo.location, 84 | email: userInfo.email, 85 | profile_image_url: userInfo.avatar_url, 86 | } 87 | } 88 | 89 | export const getTwitterUserInfo = async ( 90 | username: string 91 | ): Promise => { 92 | const res = await ( 93 | await fetch( 94 | `https://api.twitter.com/2/users/by/username/${username}?user.fields=url,public_metrics,description,location,profile_image_url,entities`, 95 | { 96 | headers: { 97 | Authorization: `Bearer ${process.env.TWITTER_API_BEARER_TOKEN}`, 98 | }, 99 | } 100 | ) 101 | ).json() 102 | if (!res.data) { 103 | throw new Error(`Error getting Twitter info for user ${username}`) 104 | } 105 | const userInfo = res.data 106 | 107 | return { 108 | username, 109 | name: userInfo.name, 110 | website: userInfo.entities?.url?.urls?.[0]?.expanded_url || userInfo.url, 111 | nb_followers: userInfo.public_metrics?.followers_count || 0, 112 | location: userInfo.location, 113 | profile_image_url: userInfo.profile_image_url, 114 | } 115 | } 116 | 117 | export const getWebsiteInfo = async (url: string): Promise => { 118 | const res = await fetch(url) 119 | const $ = cheerio.load(await res.text()) 120 | 121 | const title = cleanText($('title').text()) || null 122 | const description = 123 | cleanText($('meta[name="description"]').attr('content')) || null 124 | 125 | let twitterUsername = 126 | cleanText($('meta[name="twitter:creator"]').attr('content')) 127 | ?.replace(/^@/, '') 128 | .replace(/^https?:\/\/(.*)?twitter.com\//, '') || null 129 | 130 | const urlObj = new URL(url) 131 | let githubUsername = 132 | urlObj.hostname === 'github.com' 133 | ? urlObj.pathname.replace(/^\//, '').replace(/\/.*/, '') 134 | : null 135 | 136 | if (!twitterUsername) { 137 | const twitterUrl = $('a[href*="twitter.com"]').attr('href') 138 | if (twitterUrl) { 139 | const [, username] = twitterUrl.match(/twitter.com\/([a-z0-9_]+)/i) || [] 140 | if (username) twitterUsername = username 141 | } 142 | } 143 | 144 | if (!githubUsername) { 145 | const githubUrl = $('a[href*="github.com"]').attr('href') 146 | if (githubUrl) { 147 | const [, username] = githubUrl.match(/github.com\/([a-z0-9_.]+)/i) || [] 148 | if (username) githubUsername = username 149 | } 150 | } 151 | 152 | return { 153 | url, 154 | title, 155 | description, 156 | twitter_username: twitterUsername, 157 | github_username: githubUsername, 158 | } 159 | } 160 | 161 | export const getStoryInfo = async ( 162 | storyId: number 163 | ): Promise<(InsertStoryVariables & { website_url?: string }) | undefined> => { 164 | const storyInfo = await getSnapshotValue<{ 165 | by: string 166 | descendants: number 167 | id: number 168 | kids: number[] 169 | score: number 170 | time: number 171 | title: string 172 | type: 'story' 173 | url?: string 174 | text?: string 175 | }>(`v0/item/${storyId}`) 176 | if (storyInfo?.type !== 'story' || !storyInfo?.title?.match(/^Show HN:/i)) 177 | return undefined 178 | 179 | let url = storyInfo.url 180 | if (!url && storyInfo.text) { 181 | const [foundUrl] = decodeEntities(storyInfo.text).match(URL_REGEXP) || [] 182 | if (foundUrl) url = foundUrl 183 | } 184 | 185 | return { 186 | id: storyInfo.id, 187 | date: new Date(storyInfo.time * 1000), 188 | title: storyInfo.title, 189 | score: storyInfo.score, 190 | hn_username: storyInfo.by, 191 | website_url: url, 192 | } 193 | } 194 | 195 | export const getNewestStoryIds = async () => { 196 | let url: string | undefined = 'https://news.ycombinator.com/shownew' 197 | const storyIds: number[] = [] 198 | while (url) { 199 | const res = await fetch(url) 200 | const html: string = await res.text() 201 | const $ = cheerio.load(html) 202 | storyIds.push( 203 | ...$('.itemlist > tbody > tr[id]') 204 | .toArray() 205 | .map((el) => Number($(el).attr('id'))) 206 | ) 207 | const href = $('.morelink').attr('href') 208 | url = href && `https://news.ycombinator.com/${href}` 209 | } 210 | return storyIds 211 | } 212 | -------------------------------------------------------------------------------- /scraper/src/cli/get-show-hn-stories.ts: -------------------------------------------------------------------------------- 1 | import firebase from 'firebase/app' 2 | import 'firebase/database' 3 | 4 | firebase.initializeApp({ 5 | databaseURL: 'https://hacker-news.firebaseio.com', 6 | }) 7 | 8 | const getSnapshotValue = async (path: string) => { 9 | const snapshot = await firebase.database().ref(path).once('value') 10 | return snapshot.val() as T 11 | } 12 | 13 | const main = async () => { 14 | try { 15 | const storyId = process.argv[2] 16 | if (!storyId) { 17 | console.log('No story ID') 18 | } else { 19 | for (let id = Number(storyId) - 1; id > 0; id--) { 20 | const storyInfo = await getSnapshotValue<{ 21 | descendants: number 22 | id: number 23 | kids: number[] 24 | score: number 25 | time: number 26 | title: string 27 | type: 'story' 28 | url?: string 29 | text?: string 30 | }>(`v0/item/${id}`) 31 | if (storyInfo.type === 'story' && storyInfo.title?.match(/^Show HN/i)) { 32 | console.log(storyInfo.id, new Date(storyInfo.time * 1000)) 33 | } 34 | } 35 | 36 | process.exit(0) 37 | } 38 | } catch (err) { 39 | console.error(err) 40 | process.exit(1) 41 | } 42 | } 43 | 44 | main() 45 | -------------------------------------------------------------------------------- /scraper/src/cli/newest-scraper.ts: -------------------------------------------------------------------------------- 1 | import { runNewest } from '../scraper' 2 | 3 | const main = async () => { 4 | try { 5 | await runNewest({ dryRun: false, force: false }) 6 | process.exit(0) 7 | } catch (err) { 8 | console.error(err) 9 | process.exit(1) 10 | } 11 | } 12 | 13 | main() 14 | -------------------------------------------------------------------------------- /scraper/src/cli/past-scraper.ts: -------------------------------------------------------------------------------- 1 | import { runPast } from '../scraper' 2 | 3 | const main = async () => { 4 | try { 5 | await runPast({ dryRun: false, force: false }) 6 | process.exit(0) 7 | } catch (err) { 8 | console.error(err) 9 | process.exit(1) 10 | } 11 | } 12 | 13 | main() 14 | -------------------------------------------------------------------------------- /scraper/src/cli/rescrap.ts: -------------------------------------------------------------------------------- 1 | import { runRescrap } from '../scraper' 2 | 3 | const main = async () => { 4 | try { 5 | await runRescrap({ dryRun: false }) 6 | process.exit(0) 7 | } catch (err) { 8 | console.error(err) 9 | process.exit(1) 10 | } 11 | } 12 | 13 | main() 14 | -------------------------------------------------------------------------------- /scraper/src/cli/scrap-stories.ts: -------------------------------------------------------------------------------- 1 | import { runStories } from '../scraper' 2 | 3 | const main = async () => { 4 | try { 5 | const storyIds = process.argv.slice(2).map(Number) 6 | await runStories(storyIds) 7 | process.exit(0) 8 | } catch (err) { 9 | console.error(err) 10 | process.exit(1) 11 | } 12 | } 13 | 14 | main() 15 | -------------------------------------------------------------------------------- /scraper/src/cli/scrap-story.ts: -------------------------------------------------------------------------------- 1 | import { getStoryInfo } from '../scraper' 2 | 3 | const main = async () => { 4 | try { 5 | const storyId = process.argv[2] 6 | if (!storyId) { 7 | console.log('No story ID') 8 | } else { 9 | console.log(`Scaping HN story ${storyId}...`) 10 | const info = await getStoryInfo(Number(storyId), { 11 | force: true, 12 | dryRun: true, 13 | }) 14 | console.log(info) 15 | } 16 | process.exit(0) 17 | } catch (err) { 18 | console.error(err) 19 | process.exit(1) 20 | } 21 | } 22 | 23 | main() 24 | -------------------------------------------------------------------------------- /scraper/src/cli/update-stories-scores.ts: -------------------------------------------------------------------------------- 1 | import { updateStoriesScore } from '../scraper' 2 | 3 | const main = async () => { 4 | try { 5 | await updateStoriesScore({}) 6 | process.exit(0) 7 | } catch (err) { 8 | console.error(err) 9 | process.exit(1) 10 | } 11 | } 12 | 13 | main() 14 | -------------------------------------------------------------------------------- /scraper/src/functions/scraper.ts: -------------------------------------------------------------------------------- 1 | import { APIGatewayEvent } from 'aws-lambda' 2 | import { runNewest } from '../scraper' 3 | 4 | export const handler = async (event: APIGatewayEvent) => { 5 | if ( 6 | event.queryStringParameters?.secret !== 7 | process.env.HASURA_GRAPHQL_ADMIN_SECRET 8 | ) { 9 | return { statusCode: 403 } 10 | } 11 | 12 | try { 13 | await runNewest({ dryRun: false, force: false }) 14 | return { statusCode: 200 } 15 | } catch (err) { 16 | console.error(err) 17 | return { statusCode: 500 } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /scraper/src/functions/update-stories-scores.ts: -------------------------------------------------------------------------------- 1 | import { APIGatewayEvent } from 'aws-lambda' 2 | import { updateStoriesScore } from '../scraper' 3 | 4 | export const handler = async (event: APIGatewayEvent) => { 5 | if ( 6 | event.queryStringParameters?.secret !== 7 | process.env.HASURA_GRAPHQL_ADMIN_SECRET 8 | ) { 9 | return { statusCode: 403 } 10 | } 11 | 12 | try { 13 | await updateStoriesScore({}) 14 | return { statusCode: 200 } 15 | } catch (err) { 16 | console.error(err) 17 | return { statusCode: 500 } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /scraper/src/graph.ts: -------------------------------------------------------------------------------- 1 | import ApolloClient from 'apollo-boost' 2 | import gql from 'graphql-tag' 3 | import fetch from 'node-fetch' 4 | import { GetHnUser, GetHnUserVariables } from 'sidep-common/types/GetHnUser' 5 | import { 6 | InsertHnUser, 7 | InsertHnUserVariables, 8 | } from 'sidep-common/types/InsertHnUser' 9 | import { 10 | GetTwitterUser, 11 | GetTwitterUserVariables, 12 | } from 'sidep-common/types/GetTwitterUser' 13 | import { 14 | InsertTwitterUser, 15 | InsertTwitterUserVariables, 16 | } from 'sidep-common/types/InsertTwitterUser' 17 | import { 18 | GetGithubUser, 19 | GetGithubUserVariables, 20 | } from 'sidep-common/types/GetGithubUser' 21 | import { 22 | InsertGithubUser, 23 | InsertGithubUserVariables, 24 | } from 'sidep-common/types/InsertGithubUser' 25 | import { 26 | InsertWebsite, 27 | InsertWebsiteVariables, 28 | } from 'sidep-common/types/InsertWebsite' 29 | import { 30 | InsertStory, 31 | InsertStoryVariables, 32 | } from 'sidep-common/types/InsertStory' 33 | import { 34 | DoesStoryExist, 35 | DoesStoryExistVariables, 36 | } from 'sidep-common/types/DoesStoryExist' 37 | import { GetLastHnStoryId } from 'sidep-common/types/GetLastHnStoryId' 38 | import { GetFirstHnStoryId } from 'sidep-common/types/GetFirstHnStoryId' 39 | import { GetLatestStoryIds } from 'sidep-common/types/GetLatestStoryIds' 40 | import { 41 | UpdateStoryScore, 42 | UpdateStoryScoreVariables, 43 | } from 'sidep-common/types/UpdateStoryScore' 44 | import { 45 | UpdateStory, 46 | UpdateStoryVariables, 47 | } from 'sidep-common/types/UpdateStory' 48 | import { 49 | UpdateHnUser, 50 | UpdateHnUserVariables, 51 | } from 'sidep-common/types/UpdateHnUser' 52 | import { 53 | UpdateTwitterUser, 54 | UpdateTwitterUserVariables, 55 | } from 'sidep-common/types/UpdateTwitterUser' 56 | import { 57 | UpdateGithubUser, 58 | UpdateGithubUserVariables, 59 | } from 'sidep-common/types/UpdateGithubUser' 60 | import { 61 | UpdateWebsite, 62 | UpdateWebsiteVariables, 63 | } from 'sidep-common/types/UpdateWebsite' 64 | import { GetWebsite, GetWebsiteVariables } from 'sidep-common/types/GetWebsite' 65 | import { GetAllStoryIds } from 'sidep-common/types/GetAllStoryIds' 66 | import { omit } from './helpers' 67 | 68 | const client = new ApolloClient({ 69 | uri: process.env.HASURA_GRAPHQL_URL, 70 | fetch: fetch as never, 71 | headers: { 72 | 'x-hasura-admin-secret': process.env.HASURA_GRAPHQL_ADMIN_SECRET, 73 | }, 74 | }) 75 | 76 | export const getHnUser = async (username: string) => { 77 | const { data } = await client.query({ 78 | query: gql` 79 | query GetHnUser($username: String!) { 80 | hn_users(where: { username: { _ilike: $username } }) { 81 | username 82 | karma 83 | } 84 | } 85 | `, 86 | variables: { username }, 87 | fetchPolicy: 'network-only', 88 | }) 89 | return omit(data.hn_users[0], '__typename') 90 | } 91 | 92 | export const updateHnUser = async (variables: UpdateHnUserVariables) => { 93 | const { data } = await client.mutate({ 94 | mutation: gql` 95 | mutation UpdateHnUser($username: String!, $karma: numeric!) { 96 | update_hn_users( 97 | _set: { karma: $karma } 98 | where: { username: { _eq: $username } } 99 | ) { 100 | affected_rows 101 | } 102 | } 103 | `, 104 | variables, 105 | }) 106 | return data?.update_hn_users?.affected_rows 107 | } 108 | 109 | export const insertHnUser = async (variables: InsertHnUserVariables) => { 110 | const { data } = await client.mutate({ 111 | mutation: gql` 112 | mutation InsertHnUser($username: String!, $karma: numeric!) { 113 | insert_hn_users_one(object: { username: $username, karma: $karma }) { 114 | username 115 | karma 116 | } 117 | } 118 | `, 119 | variables, 120 | }) 121 | return omit(data?.insert_hn_users_one, '__typename') 122 | } 123 | 124 | export const updateStory = async (variables: UpdateStoryVariables) => { 125 | const { data } = await client.mutate({ 126 | mutation: gql` 127 | mutation UpdateStory( 128 | $id: Int! 129 | $date: timestamptz! 130 | $hn_username: String 131 | $website_url: String 132 | $score: numeric! 133 | $title: String! 134 | ) { 135 | update_hn_stories( 136 | _set: { 137 | date: $date 138 | hn_username: $hn_username 139 | website_url: $website_url 140 | score: $score 141 | title: $title 142 | } 143 | where: { id: { _eq: $id } } 144 | ) { 145 | affected_rows 146 | } 147 | } 148 | `, 149 | variables, 150 | }) 151 | return data?.update_hn_stories?.affected_rows 152 | } 153 | 154 | export const insertStory = async (variables: InsertStoryVariables) => { 155 | const { data } = await client.mutate({ 156 | mutation: gql` 157 | mutation InsertStory( 158 | $id: Int! 159 | $date: timestamptz! 160 | $hn_username: String 161 | $website_url: String 162 | $score: numeric! 163 | $title: String! 164 | ) { 165 | insert_hn_stories_one( 166 | object: { 167 | id: $id 168 | date: $date 169 | hn_username: $hn_username 170 | website_url: $website_url 171 | score: $score 172 | title: $title 173 | } 174 | ) { 175 | id 176 | title 177 | score 178 | date 179 | hn_user { 180 | username 181 | karma 182 | } 183 | website { 184 | url 185 | title 186 | description 187 | twitter_user { 188 | username 189 | name 190 | website 191 | nb_followers 192 | } 193 | github_user { 194 | username 195 | name 196 | website 197 | nb_followers 198 | } 199 | } 200 | } 201 | } 202 | `, 203 | variables, 204 | }) 205 | return omit(data?.insert_hn_stories_one, '__typename') 206 | } 207 | 208 | export const doesStoryExist = async (id: number) => { 209 | const { data } = await client.query({ 210 | query: gql` 211 | query DoesStoryExist($id: Int!) { 212 | hn_stories_by_pk(id: $id) { 213 | id 214 | } 215 | } 216 | `, 217 | variables: { id }, 218 | fetchPolicy: 'network-only', 219 | }) 220 | return Boolean(data.hn_stories_by_pk) 221 | } 222 | 223 | export const getTwitterUser = async (username: string) => { 224 | const { data } = await client.query({ 225 | query: gql` 226 | query GetTwitterUser($username: String!) { 227 | twitter_users(where: { username: { _ilike: $username } }) { 228 | username 229 | name 230 | nb_followers 231 | website 232 | location 233 | profile_image_url 234 | } 235 | } 236 | `, 237 | variables: { username }, 238 | fetchPolicy: 'network-only', 239 | }) 240 | return omit(data.twitter_users[0], '__typename') 241 | } 242 | 243 | export const updateTwitterUser = async ( 244 | variables: UpdateTwitterUserVariables 245 | ) => { 246 | const { data } = await client.mutate< 247 | UpdateTwitterUser, 248 | UpdateTwitterUserVariables 249 | >({ 250 | mutation: gql` 251 | mutation UpdateTwitterUser( 252 | $username: String! 253 | $name: String 254 | $website: String 255 | $nb_followers: numeric! 256 | $location: String 257 | $profile_image_url: String 258 | ) { 259 | update_twitter_users( 260 | where: { username: { _ilike: $username } } 261 | _set: { 262 | name: $name 263 | website: $website 264 | nb_followers: $nb_followers 265 | location: $location 266 | profile_image_url: $profile_image_url 267 | } 268 | ) { 269 | affected_rows 270 | } 271 | } 272 | `, 273 | variables, 274 | }) 275 | return data?.update_twitter_users?.affected_rows 276 | } 277 | 278 | export const insertTwitterUser = async ( 279 | variables: InsertTwitterUserVariables 280 | ) => { 281 | const { data } = await client.mutate< 282 | InsertTwitterUser, 283 | InsertTwitterUserVariables 284 | >({ 285 | mutation: gql` 286 | mutation InsertTwitterUser( 287 | $username: String! 288 | $name: String 289 | $website: String 290 | $nb_followers: numeric! 291 | $location: String 292 | $profile_image_url: String 293 | ) { 294 | insert_twitter_users_one( 295 | object: { 296 | username: $username 297 | name: $name 298 | website: $website 299 | nb_followers: $nb_followers 300 | location: $location 301 | profile_image_url: $profile_image_url 302 | } 303 | ) { 304 | username 305 | name 306 | website 307 | nb_followers 308 | location 309 | profile_image_url 310 | } 311 | } 312 | `, 313 | variables, 314 | }) 315 | return omit(data?.insert_twitter_users_one, '__typename') 316 | } 317 | 318 | export const getGithubUser = async (username: string) => { 319 | const { data } = await client.query({ 320 | query: gql` 321 | query GetGithubUser($username: String!) { 322 | github_users(where: { username: { _ilike: $username } }) { 323 | username 324 | name 325 | nb_followers 326 | website 327 | twitter_username 328 | location 329 | email 330 | profile_image_url 331 | } 332 | } 333 | `, 334 | variables: { username }, 335 | fetchPolicy: 'network-only', 336 | }) 337 | return omit(data.github_users[0], '__typename') 338 | } 339 | 340 | export const insertGithubUser = async ( 341 | variables: InsertGithubUserVariables 342 | ) => { 343 | const { data } = await client.mutate< 344 | InsertGithubUser, 345 | InsertGithubUserVariables 346 | >({ 347 | mutation: gql` 348 | mutation InsertGithubUser( 349 | $username: String! 350 | $name: String 351 | $website: String 352 | $nb_followers: numeric! 353 | $twitter_username: String 354 | $location: String 355 | $email: String 356 | $profile_image_url: String 357 | ) { 358 | insert_github_users_one( 359 | object: { 360 | username: $username 361 | name: $name 362 | website: $website 363 | nb_followers: $nb_followers 364 | twitter_username: $twitter_username 365 | location: $location 366 | email: $email 367 | profile_image_url: $profile_image_url 368 | } 369 | ) { 370 | username 371 | name 372 | website 373 | nb_followers 374 | twitter_username 375 | location 376 | email 377 | profile_image_url 378 | } 379 | } 380 | `, 381 | variables, 382 | }) 383 | return omit(data?.insert_github_users_one, '__typename') 384 | } 385 | 386 | export const updateGithubUser = async ( 387 | variables: UpdateGithubUserVariables 388 | ) => { 389 | const { data } = await client.mutate< 390 | UpdateGithubUser, 391 | UpdateGithubUserVariables 392 | >({ 393 | mutation: gql` 394 | mutation UpdateGithubUser( 395 | $username: String! 396 | $name: String 397 | $website: String 398 | $nb_followers: numeric! 399 | $twitter_username: String 400 | $location: String 401 | $email: String 402 | $profile_image_url: String 403 | ) { 404 | update_github_users( 405 | where: { username: { _ilike: $username } } 406 | _set: { 407 | name: $name 408 | website: $website 409 | nb_followers: $nb_followers 410 | twitter_username: $twitter_username 411 | location: $location 412 | email: $email 413 | profile_image_url: $profile_image_url 414 | } 415 | ) { 416 | affected_rows 417 | } 418 | } 419 | `, 420 | variables, 421 | }) 422 | return data?.update_github_users?.affected_rows 423 | } 424 | 425 | export const getWebsite = async (url: string) => { 426 | const { data } = await client.query({ 427 | query: gql` 428 | query GetWebsite($url: String!) { 429 | websites(where: { url: { _ilike: $url } }) { 430 | url 431 | title 432 | description 433 | twitter_username 434 | github_username 435 | } 436 | } 437 | `, 438 | variables: { url }, 439 | fetchPolicy: 'network-only', 440 | }) 441 | return omit(data.websites[0], '__typename') 442 | } 443 | 444 | export const insertWebsite = async (variables: InsertWebsiteVariables) => { 445 | const { data } = await client.mutate({ 446 | mutation: gql` 447 | mutation InsertWebsite( 448 | $url: String! 449 | $title: String 450 | $description: String 451 | $twitter_username: String 452 | $github_username: String 453 | ) { 454 | insert_websites_one( 455 | object: { 456 | url: $url 457 | title: $title 458 | description: $description 459 | twitter_username: $twitter_username 460 | github_username: $github_username 461 | } 462 | ) { 463 | url 464 | title 465 | description 466 | twitter_username 467 | github_username 468 | } 469 | } 470 | `, 471 | variables, 472 | }) 473 | return omit(data?.insert_websites_one, '__typename') 474 | } 475 | 476 | export const updateWebsite = async (variables: UpdateWebsiteVariables) => { 477 | const { data } = await client.mutate({ 478 | mutation: gql` 479 | mutation UpdateWebsite( 480 | $url: String! 481 | $title: String 482 | $description: String 483 | $twitter_username: String 484 | $github_username: String 485 | ) { 486 | update_websites( 487 | where: { url: { _ilike: $url } } 488 | _set: { 489 | title: $title 490 | description: $description 491 | twitter_username: $twitter_username 492 | github_username: $github_username 493 | } 494 | ) { 495 | affected_rows 496 | } 497 | } 498 | `, 499 | variables, 500 | }) 501 | return data?.update_websites?.affected_rows 502 | } 503 | 504 | export const getLastStoryId = async (): Promise => { 505 | const { data } = await client.query({ 506 | query: gql` 507 | query GetLastHnStoryId { 508 | hn_stories(limit: 1, order_by: { id: desc }) { 509 | id 510 | } 511 | } 512 | `, 513 | fetchPolicy: 'network-only', 514 | }) 515 | return data.hn_stories[0]?.id 516 | } 517 | 518 | export const getAllStoryIds = async () => { 519 | const { data } = await client.query({ 520 | query: gql` 521 | query GetAllStoryIds { 522 | hn_stories { 523 | id 524 | } 525 | } 526 | `, 527 | }) 528 | return data.hn_stories.map((s) => s.id) 529 | } 530 | 531 | export const getFirstStoryId = async (): Promise => { 532 | const { data } = await client.query({ 533 | query: gql` 534 | query GetFirstHnStoryId { 535 | hn_stories(limit: 1, order_by: { id: asc }) { 536 | id 537 | } 538 | } 539 | `, 540 | fetchPolicy: 'network-only', 541 | }) 542 | return data.hn_stories[0]?.id 543 | } 544 | 545 | export const getLatestStoryIds = async () => { 546 | const { data } = await client.query({ 547 | query: gql` 548 | query GetLatestStoryIds($minDate: timestamptz) { 549 | hn_stories(where: { date: { _gte: $minDate } }) { 550 | id 551 | score 552 | } 553 | } 554 | `, 555 | variables: { minDate: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000) }, 556 | }) 557 | return data.hn_stories 558 | } 559 | 560 | export const updateStoryScore = async (storyId: number, score: number) => { 561 | await client.mutate({ 562 | mutation: gql` 563 | mutation UpdateStoryScore($storyId: Int!, $score: numeric!) { 564 | update_hn_stories( 565 | _set: { score: $score } 566 | where: { id: { _eq: $storyId } } 567 | ) { 568 | affected_rows 569 | } 570 | } 571 | `, 572 | variables: { storyId, score }, 573 | }) 574 | } 575 | -------------------------------------------------------------------------------- /scraper/src/helpers.ts: -------------------------------------------------------------------------------- 1 | export const decodeEntities = (encodedString: string) => { 2 | const translate: Record = { 3 | nbsp: ' ', 4 | amp: '&', 5 | quot: '"', 6 | lt: '<', 7 | gt: '>', 8 | } 9 | const translate_re = new RegExp( 10 | `&(${Object.keys(translate).join('|')});`, 11 | 'g' 12 | ) 13 | return encodedString 14 | .replace(translate_re, (match, entity) => translate[entity]) 15 | .replace(/&#(\d+);/gi, (match, numStr) => 16 | String.fromCharCode(parseInt(numStr, 10)) 17 | ) 18 | .replace(/&#x([0-9a-f]+);/gi, (match, numStr) => 19 | String.fromCharCode(parseInt(numStr, 16)) 20 | ) 21 | } 22 | 23 | export const URL_REGEXP = /https?:\/\/[-A-Z0-9+&@#/%?=~_|!:,.;]*[-A-Z0-9+&@#/%=~_|]/gi 24 | 25 | export const omit = ( 26 | obj: T | undefined | null, 27 | key: K 28 | ): Omit | undefined | null => { 29 | if (!obj) return obj 30 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 31 | const { [key]: _, ...rest } = obj 32 | return rest 33 | } 34 | 35 | export const omitAll = ( 36 | arr: T[] | undefined | null, 37 | key: K 38 | ): (Omit | null | undefined)[] | null | undefined => 39 | arr && arr.map((obj) => omit(obj, key)) 40 | -------------------------------------------------------------------------------- /scraper/src/scraper.ts: -------------------------------------------------------------------------------- 1 | import * as graph from './graph' 2 | import * as api from './api' 3 | 4 | interface Info { 5 | source: 'db' | 'api' 6 | existsInDb: boolean 7 | value: T 8 | } 9 | 10 | interface Options { 11 | dryRun: boolean 12 | force: boolean 13 | } 14 | 15 | const getHnUserInfo = async ( 16 | username: string, 17 | options: Options 18 | ): Promise> => { 19 | const hnUser = await graph.getHnUser(username) 20 | const existsInDb = Boolean(hnUser) 21 | if (hnUser && !options.force) { 22 | return { source: 'db', existsInDb, value: hnUser } 23 | } 24 | return { 25 | source: 'api', 26 | existsInDb, 27 | value: await api.getHnUserInfo(username), 28 | } 29 | } 30 | 31 | const getGithubUserInfo = async ( 32 | username: string, 33 | options: Options 34 | ): Promise> => { 35 | const githubUser = await graph.getGithubUser(username) 36 | const existsInDb = Boolean(githubUser) 37 | if (!options.force && githubUser) { 38 | return { source: 'db', existsInDb, value: githubUser } 39 | } 40 | return { 41 | source: 'api', 42 | existsInDb, 43 | value: await api.getGithubUserInfo(username), 44 | } 45 | } 46 | 47 | const getTwitterUserInfo = async ( 48 | username: string, 49 | options: Options 50 | ): Promise> => { 51 | const twitterUser = await graph.getTwitterUser(username) 52 | const existsInDb = Boolean(twitterUser) 53 | if (!options.force && twitterUser) { 54 | return { source: 'db', existsInDb, value: twitterUser } 55 | } 56 | return { 57 | source: 'api', 58 | existsInDb, 59 | value: await api.getTwitterUserInfo(username), 60 | } 61 | } 62 | 63 | const getWebsiteInfo = async ( 64 | url: string, 65 | options: Options 66 | ): Promise> => { 67 | const website = await graph.getWebsite(url) 68 | const existsInDb = Boolean(website) 69 | if (!options.force && website) { 70 | return { source: 'db', existsInDb, value: website } 71 | } 72 | return { 73 | source: 'api', 74 | existsInDb, 75 | value: await api.getWebsiteInfo(url), 76 | } 77 | } 78 | 79 | export const getStoryInfo = async ( 80 | storyId: number, 81 | options: Options 82 | ): Promise<{ 83 | story: 84 | | { source: 'db' | 'no_show_hn'; existsInDb: boolean; value: undefined } 85 | | { source: 'api'; existsInDb: boolean; value: api.HnStory } 86 | hnUser?: Info 87 | website?: Info 88 | twitterUser?: Info 89 | githubUser?: Info 90 | }> => { 91 | const existsInDb = await graph.doesStoryExist(storyId) 92 | if (!options.force && existsInDb) { 93 | return { story: { source: 'db', existsInDb, value: undefined } } 94 | } 95 | 96 | const storyInfo = await api.getStoryInfo(storyId) 97 | if (!storyInfo) { 98 | return { story: { source: 'no_show_hn', existsInDb, value: undefined } } 99 | } 100 | 101 | const hnUser = storyInfo.hn_username 102 | ? await getHnUserInfo(storyInfo.hn_username, { 103 | ...options, 104 | force: false, 105 | }) 106 | : undefined 107 | 108 | let website, twitterUser, githubUser 109 | if (storyInfo.website_url) { 110 | try { 111 | website = await getWebsiteInfo(storyInfo.website_url, options) 112 | } catch (err) { 113 | console.error(err) 114 | storyInfo.website_url = undefined 115 | } 116 | if (website?.value.twitter_username) { 117 | try { 118 | twitterUser = await getTwitterUserInfo( 119 | website.value.twitter_username, 120 | options 121 | ) 122 | } catch (err) { 123 | console.error(err) 124 | website.value.twitter_username = null 125 | } 126 | } 127 | if (website?.value.github_username) { 128 | try { 129 | githubUser = await getGithubUserInfo( 130 | website.value.github_username, 131 | options 132 | ) 133 | } catch (err) { 134 | console.error(err) 135 | website.value.github_username = null 136 | } 137 | if (!twitterUser && githubUser?.value.twitter_username) { 138 | website.value.twitter_username = githubUser.value.twitter_username 139 | try { 140 | twitterUser = await getTwitterUserInfo( 141 | githubUser.value.twitter_username, 142 | options 143 | ) 144 | website.value.twitter_username = githubUser.value.twitter_username 145 | } catch (err) { 146 | console.error(err) 147 | } 148 | } 149 | } 150 | } 151 | 152 | return { 153 | story: { source: 'api', existsInDb, value: storyInfo }, 154 | hnUser, 155 | website, 156 | twitterUser, 157 | githubUser, 158 | } 159 | } 160 | 161 | const saveHnUser = async ( 162 | hnUser: Info | undefined, 163 | options: Options 164 | ) => { 165 | if (!options.dryRun && hnUser) { 166 | if (hnUser.existsInDb) { 167 | if (options.force) { 168 | console.log(`Updating HN user ${hnUser.value.username}`) 169 | const affectedRows = await graph.updateHnUser(hnUser.value) 170 | if (!affectedRows) throw new Error('No updated HN user') 171 | } 172 | } else { 173 | console.log(`Inserting HN user ${hnUser.value.username}`) 174 | const insertedUser = await graph.insertHnUser(hnUser.value) 175 | if (!insertedUser) throw new Error('No inserted HN user.') 176 | } 177 | } 178 | } 179 | 180 | const saveTwitterUser = async ( 181 | twitterUser: Info | undefined, 182 | options: Options 183 | ) => { 184 | if (!options.dryRun && twitterUser) { 185 | if (twitterUser.existsInDb) { 186 | if (options.force) { 187 | console.log(`Updating Twitter user ${twitterUser.value.username}`) 188 | const affectedRows = await graph.updateTwitterUser(twitterUser.value) 189 | if (!affectedRows) throw new Error('No updated Twitter user') 190 | } 191 | } else { 192 | console.log(`Inserting Twitter user ${twitterUser.value.username}`) 193 | const insertedTwitterUser = await graph.insertTwitterUser( 194 | twitterUser.value 195 | ) 196 | if (!insertedTwitterUser) throw new Error('No inserted Twitter user.') 197 | } 198 | } 199 | } 200 | 201 | const saveGithubUser = async ( 202 | githubUser: Info | undefined, 203 | options: Options 204 | ) => { 205 | if (!options.dryRun && githubUser) { 206 | if (githubUser.existsInDb) { 207 | if (options.force) { 208 | console.log(`Updating GitHub user ${githubUser.value.username}`) 209 | const affectedRows = await graph.updateGithubUser(githubUser.value) 210 | if (!affectedRows) throw new Error('No updated GitHub user') 211 | } 212 | } else { 213 | console.log(`Inserting GitHub user ${githubUser.value.username}`) 214 | const insertedGithubUser = await graph.insertGithubUser(githubUser.value) 215 | if (!insertedGithubUser) throw new Error('No inserted GitHub user.') 216 | } 217 | } 218 | } 219 | 220 | const saveWebsite = async ( 221 | website: Info | undefined, 222 | options: Options 223 | ) => { 224 | if (!options.dryRun && website) { 225 | if (website.existsInDb) { 226 | if (options.force) { 227 | console.log(`Updating website ${website.value.url}`) 228 | const affectedRows = await graph.updateWebsite(website.value) 229 | if (!affectedRows) throw new Error('No updated website') 230 | } 231 | } else { 232 | console.log(`Inserting website ${website.value.url}`) 233 | const insertedwebsite = await graph.insertWebsite(website.value) 234 | if (!insertedwebsite) throw new Error('No inserted website.') 235 | } 236 | } 237 | } 238 | 239 | const saveHnStory = async ( 240 | story: 241 | | { 242 | source: 'db' | 'no_show_hn' 243 | existsInDb: boolean 244 | value: undefined 245 | } 246 | | { 247 | source: 'api' 248 | existsInDb: boolean 249 | value: api.HnStory 250 | }, 251 | options: Options 252 | ) => { 253 | if (!options.dryRun && story.value) { 254 | if (story.existsInDb) { 255 | if (options.force) { 256 | console.log(`Updating story ${story.value.id}`) 257 | const affectedRows = await graph.updateStory(story.value) 258 | if (!affectedRows) throw new Error('No updated story') 259 | } 260 | } else { 261 | console.log(`Inserting story ${story.value.id}`) 262 | const insertedStory = await graph.insertStory(story.value) 263 | if (!insertedStory) throw new Error('No inserted story') 264 | } 265 | } 266 | } 267 | 268 | const scrapStory = async (storyId: number, options: Options) => { 269 | try { 270 | const { 271 | story, 272 | hnUser, 273 | website, 274 | twitterUser, 275 | githubUser, 276 | } = await getStoryInfo(storyId, options) 277 | 278 | if (story.source === 'no_show_hn') { 279 | return 280 | } 281 | 282 | if (!options.force && story.source === 'db') { 283 | console.log(`HN story ${storyId}... exists`) 284 | return 285 | } 286 | 287 | console.log(`HN story ${storyId} (${story.value?.date})...`) 288 | 289 | try { 290 | await saveTwitterUser(twitterUser, options) 291 | } catch (err) { 292 | if (website) website.value.twitter_username = null 293 | } 294 | 295 | try { 296 | await saveGithubUser(githubUser, options) 297 | } catch (err) { 298 | if (website) website.value.github_username = null 299 | } 300 | 301 | try { 302 | await saveWebsite(website, options) 303 | } catch (err) { 304 | if (story.value) story.value.website_url = null 305 | } 306 | 307 | try { 308 | await saveHnUser(hnUser, options) 309 | } catch (err) { 310 | if (story.value) story.value.hn_username = null 311 | } 312 | 313 | await saveHnStory(story, options) 314 | } catch (err) { 315 | console.error(`Error scraping HN story ${storyId}`, err) 316 | } 317 | } 318 | 319 | export const updateStoriesScore = async ({ dryRun = false }) => { 320 | const storyIds = await graph.getLatestStoryIds() 321 | for (const { id: storyId, score: currentScore } of storyIds) { 322 | const storyInfo = await api.getStoryInfo(storyId) 323 | if (storyInfo?.score && storyInfo.score !== currentScore) { 324 | console.log( 325 | `Updating score for ${storyId}: ${currentScore} → ${storyInfo.score}` 326 | ) 327 | if (!dryRun) { 328 | await graph.updateStoryScore(storyId, storyInfo.score) 329 | } 330 | } 331 | } 332 | } 333 | 334 | const run = async ( 335 | scrapStories: ( 336 | scrapStory: (storyId: number) => Promise 337 | ) => Promise, 338 | options: Options 339 | ) => { 340 | console.log('New scraping...') 341 | await scrapStories(async (storyId) => await scrapStory(storyId, options)) 342 | console.log('Done!') 343 | } 344 | 345 | export const runPast = async ( 346 | { dryRun, force }: Options = { dryRun: false, force: false } 347 | ) => { 348 | await run( 349 | async (scrapStory) => { 350 | const maxHnItemId = await api.getMaxHnItemId() 351 | const firstHnStoryId = (await graph.getFirstStoryId()) || maxHnItemId 352 | console.log({ firstHnStoryId, maxHnItemId }) 353 | 354 | for (let storyId = firstHnStoryId - 1; storyId > 0; storyId--) { 355 | await scrapStory(storyId) 356 | } 357 | }, 358 | { dryRun, force } 359 | ) 360 | } 361 | 362 | export const runStories = async ( 363 | storyIds: number[], 364 | { dryRun, force }: Options = { dryRun: false, force: false } 365 | ) => { 366 | await run( 367 | async (scrapStory) => { 368 | for (const storyId of storyIds) { 369 | await scrapStory(storyId) 370 | } 371 | }, 372 | { dryRun, force } 373 | ) 374 | } 375 | 376 | export const runNewest = async ( 377 | { dryRun, force }: Options = { dryRun: false, force: false } 378 | ) => { 379 | await run( 380 | async (scrapStory) => { 381 | const lastHnStoryId = await graph.getLastStoryId() 382 | console.log({ lastHnStoryId }) 383 | const storyIds = (await api.getNewestStoryIds()) 384 | .filter((storyId) => force || !lastHnStoryId || storyId > lastHnStoryId) 385 | .reverse() 386 | console.log(`${storyIds.length} Show HN stories`) 387 | 388 | for (const storyId of storyIds) { 389 | await scrapStory(storyId) 390 | } 391 | }, 392 | { dryRun, force } 393 | ) 394 | } 395 | 396 | export const runRescrap = async ({ dryRun = false }) => { 397 | await run( 398 | async (scrapStory) => { 399 | const storyIds = await graph.getAllStoryIds() 400 | for (const storyId of storyIds) { 401 | await scrapStory(storyId) 402 | } 403 | }, 404 | { dryRun, force: true } 405 | ) 406 | } 407 | -------------------------------------------------------------------------------- /scraper/tsconfig.json: -------------------------------------------------------------------------------- 1 | // prettier-ignore 2 | { 3 | "include": ["src/**/*.ts"], 4 | "compilerOptions": { 5 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 6 | 7 | /* Basic Options */ 8 | // "incremental": true, /* Enable incremental compilation */ 9 | "target": "es3", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ 10 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ 11 | // "lib": [], /* Specify library files to be included in the compilation. */ 12 | // "allowJs": true, /* Allow javascript files to be compiled. */ 13 | // "checkJs": true, /* Report errors in .js files. */ 14 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 15 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 16 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 17 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 18 | // "outFile": "./", /* Concatenate and emit output to single file. */ 19 | "outDir": "./dist", /* Redirect output structure to the directory. */ 20 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 21 | // "composite": true, /* Enable project compilation */ 22 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 23 | // "removeComments": true, /* Do not emit comments to output. */ 24 | // "noEmit": true, /* Do not emit outputs. */ 25 | "importHelpers": true, /* Import emit helpers from 'tslib'. */ 26 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 27 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 28 | 29 | /* Strict Type-Checking Options */ 30 | "strict": true, /* Enable all strict type-checking options. */ 31 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 32 | // "strictNullChecks": true, /* Enable strict null checks. */ 33 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 34 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 35 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 36 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 37 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 38 | 39 | /* Additional Checks */ 40 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 41 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 42 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 43 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 44 | 45 | /* Module Resolution Options */ 46 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 47 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 48 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 49 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 50 | // "typeRoots": [], /* List of folders to include type definitions from. */ 51 | // "types": [], /* Type declaration files to be included in compilation. */ 52 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 53 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 54 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 55 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 56 | 57 | /* Source Map Options */ 58 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 59 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 60 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 61 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 62 | 63 | /* Experimental Options */ 64 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 65 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 66 | 67 | /* Advanced Options */ 68 | "skipLibCheck": true, /* Skip type checking of declaration files. */ 69 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /web/.eslintrc.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | module.exports = { 3 | env: { 4 | browser: true, 5 | es2021: true, 6 | }, 7 | extends: ['eslint:recommended', 'plugin:react/recommended'], 8 | parserOptions: { 9 | ecmaFeatures: { 10 | jsx: true, 11 | }, 12 | ecmaVersion: 12, 13 | sourceType: 'module', 14 | }, 15 | plugins: ['react'], 16 | rules: {}, 17 | } 18 | -------------------------------------------------------------------------------- /web/.postcssrc: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": { 3 | "postcss-nested": {}, 4 | "tailwindcss": {}, 5 | "autoprefixer": {} 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sidep-web", 3 | "version": "1.0.0", 4 | "license": "MIT", 5 | "scripts": { 6 | "run:dev": "parcel public/index.html --no-autoinstall", 7 | "build": "parcel build public/index.html" 8 | }, 9 | "dependencies": { 10 | "@apollo/client": "^3.2.5", 11 | "@material-ui/core": "^4.11.0", 12 | "@material-ui/icons": "^4.9.1", 13 | "@nhost/react-apollo": "^1.0.5", 14 | "@nhost/react-auth": "^1.0.5", 15 | "@react-hook/debounce": "^3.0.0", 16 | "autoprefixer": "^10.0.2", 17 | "date-fns": "^2.16.1", 18 | "graphql": "^15.4.0", 19 | "graphql-tag": "^2.11.0", 20 | "nhost-js-sdk": "2.2.3", 21 | "postcss": "^8.1.7", 22 | "react": "^17.0.1", 23 | "react-dom": "^17.0.1", 24 | "react-popover": "^0.5.10", 25 | "sidep-common": "^1.0.0", 26 | "tailwindcss": "^1.9.6" 27 | }, 28 | "devDependencies": { 29 | "@types/react": "^16.9.56", 30 | "@types/react-dom": "^16.9.9", 31 | "@types/react-popover": "^0.5.3", 32 | "apollo": "^2.31.1", 33 | "eslint": "^7.13.0", 34 | "eslint-plugin-react": "^7.21.5", 35 | "parcel": "^2.0.0-nightly.446", 36 | "postcss-nested": "^5.0.1", 37 | "typescript": "^4.0.5" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /web/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Show HN projects — Browse the best projects posted on Show HN 8 | 9 | 13 | 14 | 18 | 19 | 20 | 24 | 25 | 26 | 27 | 28 | 29 | 33 | 37 | 38 | 39 |
40 |
41 |

42 | Show HN projects 43 |

44 |
45 | Browse the best projects posted on 46 | Show HN 53 |
54 | 55 | 62 | 68 | 69 | 74 | 78 | 79 | 80 |
81 | 82 |
Loading…
83 | 84 |
87 |
Made in Montréal, Québec 🇨🇦
88 |
89 | Built by 90 | Sébastien Castiel 97 | (@scastiel) 104 |
105 |
106 | Source code available on 107 | GitHub 114 |
115 |
116 |
117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /web/src/components/App.tsx: -------------------------------------------------------------------------------- 1 | import { useQuery, useSubscription } from '@apollo/client' 2 | import gql from 'graphql-tag' 3 | import React, { useEffect, useState } from 'react' 4 | import { Story } from '../types' 5 | import { 6 | GetStories, 7 | GetStoriesVariables, 8 | GetStories_hn_stories, 9 | } from 'sidep-common/types/GetStories' 10 | import { 11 | GetNumberOfStories, 12 | GetNumberOfStoriesVariables, 13 | } from 'sidep-common/types/GetNumberOfStories' 14 | import { StoryList } from './StoryList' 15 | import { Filters, FiltersSet } from './Filters' 16 | import { hn_stories_bool_exp } from 'sidep-common/types/globalTypes' 17 | import { useDebounce } from '@react-hook/debounce' 18 | import Popover from 'react-popover' 19 | 20 | export const ITEMS_PER_PAGE = 20 21 | 22 | export interface Props { 23 | stories: Story[] 24 | } 25 | 26 | const GET_NUMBER_OF_RESULTS = gql` 27 | subscription GetNumberOfStories($filters: hn_stories_bool_exp) { 28 | hn_stories_aggregate(where: $filters) { 29 | aggregate { 30 | count 31 | } 32 | } 33 | } 34 | ` 35 | 36 | const GET_STORIES = gql` 37 | subscription GetStories($filters: hn_stories_bool_exp, $limit: Int) { 38 | hn_stories(order_by: { date: desc }, where: $filters, limit: $limit) { 39 | id 40 | title 41 | score 42 | date 43 | hn_user { 44 | username 45 | karma 46 | } 47 | website { 48 | url 49 | title 50 | description 51 | twitter_user { 52 | username 53 | name 54 | website 55 | nb_followers 56 | location 57 | profile_image_url 58 | } 59 | github_user { 60 | username 61 | name 62 | website 63 | nb_followers 64 | location 65 | email 66 | profile_image_url 67 | } 68 | } 69 | } 70 | } 71 | ` 72 | 73 | const uniqBy = (array: T[], key: K): T[] => { 74 | const keys = new Set() 75 | const res = [] 76 | for (const el of array) { 77 | if (!keys.has(el[key])) { 78 | keys.add(el[key]) 79 | res.push(el) 80 | } 81 | } 82 | return res 83 | } 84 | 85 | const createFilters = ( 86 | search: string, 87 | filtersSet: FiltersSet 88 | ): { nbFilters: number; filters: hn_stories_bool_exp; query: string } => { 89 | let query: Record = {} 90 | let nbFilters = 0 91 | const criteria = { _ilike: `%${search}%` } 92 | 93 | const filters: hn_stories_bool_exp = {} 94 | if ( 95 | filtersSet.scoreFilter && 96 | filtersSet.minScore && 97 | filtersSet.minScore > 0 98 | ) { 99 | nbFilters++ 100 | filters.score = { _gte: filtersSet.minScore } 101 | query.min_score = String(filtersSet.minScore) 102 | } 103 | if ( 104 | filtersSet.karmaFilter && 105 | filtersSet.minKarma && 106 | filtersSet.minKarma > 0 107 | ) { 108 | nbFilters++ 109 | filters.hn_user = { karma: { _gte: filtersSet.minKarma } } 110 | query.min_karma = String(filtersSet.minKarma) 111 | } 112 | if (filtersSet.twitterFilter) { 113 | nbFilters++ 114 | filters.website = { 115 | ...filters.website, 116 | twitter_username: { _is_null: false }, 117 | } 118 | query.has_twitter = 'true' 119 | if (filtersSet.minTwitterFollowers && filtersSet.minTwitterFollowers > 0) { 120 | filters.website.twitter_user = { 121 | nb_followers: { _gte: filtersSet.minTwitterFollowers }, 122 | } 123 | query.min_twitter_followers = String(filtersSet.minTwitterFollowers) 124 | } 125 | } 126 | if (filtersSet.githubFilter) { 127 | nbFilters++ 128 | filters.website = { 129 | ...filters.website, 130 | github_username: { _is_null: false }, 131 | } 132 | query.has_github = 'true' 133 | if (filtersSet.minGithubFollowers && filtersSet.minGithubFollowers > 0) { 134 | filters.website.github_user = { 135 | nb_followers: { _gte: filtersSet.minGithubFollowers }, 136 | } 137 | query.min_github_followers = String(filtersSet.minGithubFollowers) 138 | } 139 | } 140 | if (search) { 141 | filters._or = [ 142 | { title: criteria }, 143 | { hn_username: criteria }, 144 | { 145 | website: { 146 | _or: [ 147 | { github_username: criteria }, 148 | { twitter_username: criteria }, 149 | { 150 | github_user: { 151 | _or: [{ location: criteria }, { name: criteria }], 152 | }, 153 | }, 154 | { 155 | twitter_user: { 156 | _or: [{ location: criteria }, { name: criteria }], 157 | }, 158 | }, 159 | ], 160 | }, 161 | }, 162 | ] 163 | query.search = search 164 | } 165 | 166 | return { 167 | nbFilters, 168 | filters, 169 | query: Object.entries(query) 170 | .map(([k, v]) => `${k}=${v}`) 171 | .join('&'), 172 | } 173 | } 174 | 175 | export const App = () => { 176 | const [nbFilters, setNbFilters] = useState(0) 177 | const [filters, setFilters] = useDebounce({}, 300) 178 | const [filtersSet, setFiltersSet] = useState({ 179 | scoreFilter: false, 180 | karmaFilter: false, 181 | twitterFilter: false, 182 | githubFilter: false, 183 | }) 184 | const [search, setSearch] = useState('') 185 | const [expanded, setExpanded] = useState(false) 186 | const [limit, setLimit] = useState(ITEMS_PER_PAGE) 187 | const { loading, data, error } = useSubscription< 188 | GetStories, 189 | GetStoriesVariables 190 | >(GET_STORIES, { 191 | variables: { filters, limit }, 192 | }) 193 | const { data: nbStoriesData } = useSubscription< 194 | GetNumberOfStories, 195 | GetNumberOfStoriesVariables 196 | >(GET_NUMBER_OF_RESULTS, { 197 | variables: { filters }, 198 | }) 199 | const nbStories = nbStoriesData?.hn_stories_aggregate.aggregate?.count 200 | const [stories, setStories] = useState([]) 201 | const [invalidate, setInvalidate] = useState(true) 202 | const [filtersDisplayed, setFiltersDisplayed] = useState(false) 203 | 204 | useEffect(() => { 205 | if (data && stories !== data.hn_stories) { 206 | if (limit === ITEMS_PER_PAGE) setStories(data.hn_stories) 207 | else 208 | setStories( 209 | uniqBy( 210 | [...(invalidate ? [] : stories || []), ...data.hn_stories], 211 | 'id' 212 | ) 213 | ) 214 | setInvalidate(false) 215 | } 216 | }, [data, limit]) 217 | 218 | const [title, setTitle] = useState(

) 219 | useEffect(() => { 220 | let timeout: NodeJS.Timeout 221 | 222 | if (error) setTitle(

An error occured

) 223 | else if (loading || !stories || nbStories == null) { 224 | timeout = setTimeout(() => setTitle(

Loading…

), 500) 225 | } else if (nbStories === 0) setTitle(

No project matching filters

) 226 | else if (nbStories === 1) setTitle(

One project found

) 227 | else setTitle(

{nbStories} projects found

) 228 | 229 | return () => { 230 | if (timeout) clearTimeout(timeout) 231 | } 232 | }, [error, loading, stories, nbStories]) 233 | 234 | useEffect(() => { 235 | if (document.location.hash) { 236 | const values: Record = Object.fromEntries( 237 | document.location.hash 238 | .replace(/[?#]/, '') 239 | .split('&') 240 | .map((s) => s.split('=')) 241 | ) 242 | const filters: FiltersSet = { 243 | scoreFilter: Number(values.min_score) > 0, 244 | minScore: Number(values.min_score) || undefined, 245 | karmaFilter: Number(values.min_karma) > 0, 246 | minKarma: Number(values.min_karma) || undefined, 247 | twitterFilter: values.has_twitter === 'true', 248 | minTwitterFollowers: Number(values.min_twitter_followers) || undefined, 249 | githubFilter: values.has_github === 'true', 250 | minGithubFollowers: Number(values.min_github_followers) || undefined, 251 | } 252 | setFiltersSet(filters) 253 | if (values.search) { 254 | setSearch(values.search) 255 | } 256 | } 257 | }, []) 258 | 259 | useEffect(() => { 260 | setLimit(ITEMS_PER_PAGE) 261 | setInvalidate(true) 262 | const { filters, nbFilters, query } = createFilters(search, filtersSet) 263 | setFilters(filters) 264 | setNbFilters(nbFilters) 265 | window.history.replaceState({}, '', query ? `#${query}` : '.') 266 | }, [search, filtersSet]) 267 | 268 | return ( 269 |
270 |
271 |
272 |
273 | {title} 274 |
275 | setFiltersDisplayed(false)} 278 | place="below" 279 | tipSize={0.001} 280 | body={ 281 |
282 | 286 |
287 | } 288 | > 289 | 295 | 296 | 302 | setSearch(e.target.value)} 308 | /> 309 |
310 |
311 | {stories && stories.length > 0 && nbStories && ( 312 | <> 313 | 314 | {(nbStories && stories.length < nbStories && ( 315 | 324 | )) || 325 | null} 326 | 327 | )} 328 |
329 |
330 |
331 | ) 332 | } 333 | -------------------------------------------------------------------------------- /web/src/components/Filters.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from 'react' 2 | import { hn_stories_bool_exp } from 'sidep-common/types/globalTypes' 3 | 4 | export interface Props { 5 | filtersSet: FiltersSet 6 | onFiltersChange: (filtersSet: FiltersSet) => void 7 | } 8 | 9 | export interface FiltersSet { 10 | scoreFilter: boolean 11 | minScore?: number 12 | karmaFilter: boolean 13 | minKarma?: number 14 | twitterFilter: boolean 15 | minTwitterFollowers?: number 16 | githubFilter: boolean 17 | minGithubFollowers?: number 18 | } 19 | 20 | export const Filters = ({ filtersSet, onFiltersChange }: Props) => { 21 | return ( 22 |
23 |
    24 |
  • 25 | 40 | 56 |
    • 57 |
  • 58 | 73 | 89 |
    • 90 |
  • 91 | 106 | 125 |
    • 126 |
  • 127 | 142 | 161 |
    • 162 |
163 |
164 | ) 165 | } 166 | -------------------------------------------------------------------------------- /web/src/components/SocialLink.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | 3 | export interface Props { 4 | icon: any 5 | url: string 6 | username: string 7 | number: number 8 | } 9 | 10 | export const SocialLink = ({ icon, url, username, number }: Props) => { 11 | return ( 12 | 13 |
14 | {icon} 15 | {username} 16 |
17 | {number} 18 | 19 | ) 20 | } 21 | -------------------------------------------------------------------------------- /web/src/components/StoryList.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import { GetStories_hn_stories } from 'sidep-common/types/GetStories' 3 | import { StorySummary } from './StorySummary' 4 | 5 | export interface Props { 6 | stories: GetStories_hn_stories[] 7 | expanded: boolean 8 | } 9 | 10 | export const StoryList = ({ stories, expanded }: Props) => { 11 | return ( 12 |
    13 | {stories.map((story) => ( 14 |
  • 15 | 16 |
    • 17 | ))} 18 |
19 | ) 20 | } 21 | -------------------------------------------------------------------------------- /web/src/components/StorySummary.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from 'react' 2 | import { GetStories_hn_stories } from 'sidep-common/types/GetStories' 3 | import { SocialLink } from './SocialLink' 4 | import { WebsiteIcon } from './WebsiteIcon' 5 | import { isToday, isYesterday } from 'date-fns' 6 | import TwitterIcon from '@material-ui/icons/Twitter' 7 | import PersonIcon from '@material-ui/icons/Person' 8 | import GitHubIcon from '@material-ui/icons/GitHub' 9 | import DescriptionOutlinedIcon from '@material-ui/icons/DescriptionOutlined' 10 | 11 | export interface Props { 12 | story: GetStories_hn_stories 13 | expanded: boolean 14 | } 15 | 16 | export const StorySummary = ({ story, expanded }: Props) => { 17 | const [thisOneExpanded, setThisOneExpanded] = useState(expanded) 18 | 19 | useEffect(() => { 20 | setThisOneExpanded(expanded) 21 | }, [expanded]) 22 | 23 | return ( 24 |
25 |
26 | 29 | {story.score} 30 |
31 |
32 |

33 | {externalLink( 34 | story.website?.url || 35 | `https://news.ycombinator.com/item?id=${story.id}`, 36 | story.title, 37 | 'no-style' 38 | )} 39 |

40 | 41 |
    42 |
  • 43 | 46 | {formatDate(story.date)}{' '} 47 |
    • 48 |
  • 49 | {story.hn_user && ( 50 | } 52 | url={`https://news.ycombinator.com/user?id=${story.hn_user.username}`} 53 | username={story.hn_user.username} 54 | number={story.hn_user.karma} 55 | /> 56 | )} 57 | {story.website?.twitter_user && ( 58 | } 60 | url={`https://twitter.com/${story.website?.twitter_user.username}`} 61 | username={story.website?.twitter_user.username} 62 | number={story.website?.twitter_user.nb_followers} 63 | /> 64 | )} 65 | {story.website?.github_user && ( 66 | } 68 | url={`https://github.com/${story.website?.github_user.username}`} 69 | username={story.website?.github_user.username} 70 | number={story.website?.github_user.nb_followers} 71 | /> 72 | )} 73 |
    • 74 |
75 | 76 | {thisOneExpanded && ( 77 |
78 |
79 |

80 | Story info 81 |

82 |
    83 |
  • 84 | HN Story:{' '} 85 | {externalLink( 86 | `https://news.ycombinator.com/item?id=${story.id}`, 87 | String(story.id) 88 | )} 89 |
    • 90 |
  • Score: {story.score}
    • 91 | {story.website && ( 92 |
  • 93 | Posted URL: 94 | {externalLink( 95 | story.website.url, 96 | story.website.url, 97 | 'block w-48 truncate' 98 | )} 99 |
    • 100 | )} 101 |
102 |
103 | 104 | {story.hn_user && ( 105 |
106 |

107 | HackerNews user 108 |

109 |
    110 |
  • 111 | Username:{' '} 112 | {externalLink( 113 | `https://news.ycombinator.com/user?id=${story.hn_user.username}`, 114 | story.hn_user.username 115 | )} 116 |
    • 117 |
  • Karma: {story.hn_user.karma}
    • 118 |
119 |
120 | )} 121 | 122 | {story.website?.github_user && ( 123 |
124 |

125 | GitHub user 126 | {story.website.github_user.profile_image_url && ( 127 | 131 | )} 132 |

133 |
    134 |
  • 135 | Username:{' '} 136 | {externalLink( 137 | `https://github.com/${story.website.github_user.username}`, 138 | story.website.github_user.username 139 | )} 140 |
    • 141 | {story.website.github_user.name && ( 142 |
  • Name: {story.website.github_user.name}
    • 143 | )} 144 |
  • 145 | Number of followers:{' '} 146 | {story.website.github_user.nb_followers} 147 |
    • 148 | {story.website.github_user.location && ( 149 |
  • 150 | Location: 151 | {story.website.github_user.location} 152 |
    • 153 | )} 154 | {story.website.github_user.email && ( 155 |
  • 156 | E-mail address: 157 | {story.website.github_user.email} 158 |
    • 159 | )} 160 | {story.website.github_user.website && ( 161 |
  • 162 | Website: 163 | {externalLink( 164 | story.website.github_user.website, 165 | story.website.github_user.website, 166 | 'block w-48 truncate' 167 | )} 168 |
    • 169 | )} 170 |
171 |
172 | )} 173 | 174 | {story.website?.twitter_user && ( 175 |
176 |

177 | {' '} 178 | Twitter user 179 | {story.website.twitter_user.profile_image_url && ( 180 | 184 | )} 185 |

186 |
    187 |
  • 188 | Username:{' '} 189 | {externalLink( 190 | `https://twitter.com/${story.website.twitter_user.username}`, 191 | story.website.twitter_user.username 192 | )} 193 |
    • 194 | {story.website.twitter_user.name && ( 195 |
  • Name: {story.website.twitter_user.name}
    • 196 | )} 197 |
  • 198 | Number of followers:{' '} 199 | {story.website.twitter_user.nb_followers} 200 |
    • 201 | {story.website.twitter_user.location && ( 202 |
  • 203 | Location: 204 | {story.website.twitter_user.location} 205 |
    • 206 | )} 207 | {story.website.twitter_user.website && ( 208 |
  • 209 | Website: 210 | {externalLink( 211 | story.website.twitter_user.website, 212 | story.website.twitter_user.website, 213 | 'block w-48 truncate' 214 | )} 215 |
    • 216 | )} 217 |
218 |
219 | )} 220 |
221 | )} 222 |
223 |
224 | ) 225 | } 226 | 227 | const externalLink = (href: string, label: string, className = '') => ( 228 | 234 | {label} 235 | 236 | ) 237 | 238 | const formatDate = (date: any) => { 239 | date = new Date(date) 240 | if (isToday(date)) { 241 | return 'Today at ' + date.toLocaleTimeString() 242 | } 243 | if (isYesterday(date)) { 244 | return 'Yesterday at ' + date.toLocaleTimeString() 245 | } 246 | return date.toDateString() 247 | } 248 | -------------------------------------------------------------------------------- /web/src/components/WebsiteIcon.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | 3 | export interface Props { 4 | url: string 5 | } 6 | 7 | export const WebsiteIcon = ({ url }: Props) => { 8 | let domain 9 | try { 10 | const urlObj = new URL(url) 11 | domain = urlObj.hostname 12 | } catch (err) { 13 | domain = '' 14 | } 15 | return ( 16 | 21 | ) 22 | } 23 | -------------------------------------------------------------------------------- /web/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom' 3 | import { App } from './components/App' 4 | import './styles.css' 5 | import nhost from 'nhost-js-sdk' 6 | import { NhostApolloProvider } from '@nhost/react-apollo' 7 | 8 | nhost.initializeApp({ 9 | base_url: process.env.HBP_BASE_URL || '', 10 | }) 11 | 12 | ReactDOM.render( 13 | 14 | 15 | , 16 | document.getElementById('app') 17 | ) 18 | -------------------------------------------------------------------------------- /web/src/nhost.ts: -------------------------------------------------------------------------------- 1 | import nhost from 'nhost-js-sdk' 2 | 3 | nhost.initializeApp({ 4 | base_url: process.env.HBP_URL || '', 5 | }) 6 | 7 | export const auth = nhost.auth() 8 | -------------------------------------------------------------------------------- /web/src/styles.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | input[type='text'], 6 | input[type='search'], 7 | input[type='number'] { 8 | @apply border rounded p-1; 9 | 10 | &:focus { 11 | @apply outline-none shadow-outline; 12 | } 13 | } 14 | 15 | input[type='checkbox'], 16 | input[type='number'] { 17 | &:focus { 18 | @apply outline-none shadow-outline; 19 | } 20 | } 21 | 22 | body .MuiSvgIcon-root { 23 | font-size: unset; 24 | } 25 | 26 | input[type='checkbox'].switch { 27 | @apply invisible relative cursor-pointer; 28 | width: 25px; 29 | height: 18px; 30 | 31 | &::before { 32 | @apply bg-gray-300 visible block rounded absolute transition-all duration-300 ease-linear; 33 | content: ' '; 34 | height: 6px; 35 | top: 6px; 36 | left: 0; 37 | right: 0; 38 | } 39 | &::after { 40 | @apply bg-gray-500 absolute shadow visible block w-3 h-3 rounded-full transition-all duration-300 ease-linear; 41 | content: ' '; 42 | top: 3px; 43 | left: 0; 44 | } 45 | 46 | &:checked::before { 47 | @apply bg-blue-500; 48 | } 49 | &:checked::after { 50 | @apply bg-blue-700; 51 | left: 13px; 52 | } 53 | } 54 | 55 | a:not(.no-style) { 56 | @apply text-blue-800 underline; 57 | } 58 | 59 | .btn { 60 | @apply px-2 py-1 rounded; 61 | } 62 | .btn.btn-link { 63 | @apply text-blue-800 underline; 64 | } 65 | 66 | a:focus, 67 | button:focus { 68 | @apply rounded shadow-outline outline-none; 69 | } 70 | -------------------------------------------------------------------------------- /web/src/types.ts: -------------------------------------------------------------------------------- 1 | export interface StoryInfo { 2 | title: string 3 | date: Date 4 | score: number 5 | url: string 6 | } 7 | 8 | export interface HnUserInfo { 9 | username: string 10 | karma: number 11 | } 12 | 13 | export interface WebsiteInfo { 14 | url: string 15 | title: string 16 | description?: string 17 | } 18 | 19 | export interface GithubUserInfo { 20 | username: string 21 | url: string 22 | name: string | null 23 | blog?: string 24 | nbFollowers: number 25 | twitterUsername: string | null 26 | } 27 | 28 | export interface TwitterUserInfo { 29 | username: string 30 | url: string 31 | name: string 32 | blog?: string 33 | nbFollowers: number 34 | } 35 | 36 | export interface Story { 37 | id: number 38 | storyInfo: StoryInfo 39 | hnUserInfo: HnUserInfo 40 | websiteInfo?: WebsiteInfo 41 | githubInfo?: GithubUserInfo 42 | twitterInfo?: TwitterUserInfo 43 | } 44 | -------------------------------------------------------------------------------- /web/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | module.exports = { 3 | future: { 4 | // removeDeprecatedGapUtilities: true, 5 | // purgeLayersByDefault: true, 6 | }, 7 | purge: false, 8 | theme: { 9 | extend: {}, 10 | }, 11 | variants: { 12 | opacity: ['responsive', 'hover', 'focus', 'group-hover'], 13 | }, 14 | plugins: [], 15 | } 16 | -------------------------------------------------------------------------------- /web/tsconfig.json: -------------------------------------------------------------------------------- 1 | // prettier-ignore 2 | { 3 | "compilerOptions": { 4 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 5 | 6 | /* Basic Options */ 7 | // "incremental": true, /* Enable incremental compilation */ 8 | "target": "esnext", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */ 9 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */ 10 | // "lib": [], /* Specify library files to be included in the compilation. */ 11 | // "allowJs": true, /* Allow javascript files to be compiled. */ 12 | // "checkJs": true, /* Report errors in .js files. */ 13 | "jsx": "react", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 14 | // "declaration": true, /* Generates corresponding '.d.ts' file. */ 15 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 16 | // "sourceMap": true, /* Generates corresponding '.map' file. */ 17 | // "outFile": "./", /* Concatenate and emit output to single file. */ 18 | // "outDir": "./", /* Redirect output structure to the directory. */ 19 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 20 | // "composite": true, /* Enable project compilation */ 21 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 22 | // "removeComments": true, /* Do not emit comments to output. */ 23 | // "noEmit": true, /* Do not emit outputs. */ 24 | "importHelpers": true, /* Import emit helpers from 'tslib'. */ 25 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 26 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 27 | 28 | /* Strict Type-Checking Options */ 29 | "strict": true, /* Enable all strict type-checking options. */ 30 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 31 | // "strictNullChecks": true, /* Enable strict null checks. */ 32 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 33 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 34 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 35 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 36 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 37 | 38 | /* Additional Checks */ 39 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 40 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 41 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 42 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 43 | 44 | /* Module Resolution Options */ 45 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 46 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 47 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 48 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 49 | // "typeRoots": [], /* List of folders to include type definitions from. */ 50 | // "types": [], /* Type declaration files to be included in compilation. */ 51 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 52 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 53 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 54 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 55 | 56 | /* Source Map Options */ 57 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 58 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 59 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 60 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 61 | 62 | /* Experimental Options */ 63 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 64 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 65 | 66 | /* Advanced Options */ 67 | "skipLibCheck": true, /* Skip type checking of declaration files. */ 68 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ 69 | } 70 | } 71 | --------------------------------------------------------------------------------