├── .env.example ├── .github └── workflows │ └── writeup-finder-runner.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── command ├── action.go ├── command.go ├── completion.go └── flags.go ├── data ├── Youtube_channel.md ├── keywords.json └── url.txt ├── db ├── db.go └── db_test.go ├── global └── global.go ├── go.mod ├── go.sum ├── handler ├── handler.go ├── medium.go ├── utils.go └── youtube.go ├── main.go ├── run_writeUp-finder.sh ├── telegram ├── message.go ├── proxy.go ├── request.go └── telegram.go └── utils ├── env.go ├── filters.go ├── http.go ├── rss.go └── utils.go /.env.example: -------------------------------------------------------------------------------- 1 | TELEGRAM_BOT_TOKEN= 2 | TELEGRAM_CHANNEL_ID= 3 | CHAT_ID= 4 | 5 | 6 | MAIN_THREAD_ID= 7 | TRYHACKME_THREAD_ID= 8 | HACKTHEBOX_THREAD_ID= 9 | MOBILE_THREAD_ID= 10 | MONEY_THREAD_ID= 11 | RECON_THREAD_ID= 12 | BYPASS_THREAD_ID= 13 | PORTSWIGGER_THREAD_ID= 14 | BURPSUITE_THREAD_ID= 15 | OS_THREAD_ID= 16 | VULNERABILITIES_THREAD_ID= 17 | TOOLS_THREAD_ID= 18 | PROGRAMMINGLANGS_THREAD_ID= 19 | CVE_THREAD_ID= 20 | OSINT_THREAD_ID= 21 | CRYPTOGRAPHIC_THREAD_ID= 22 | STEGANOGRAPHY_THREAD_ID= 23 | WEBSCRAPING_THREAD_ID= 24 | YOUTUBE_THREAD_ID= 25 | 26 | DB_HOST= 27 | DB_PORT= 28 | DB_NAME= 29 | DB_USER= 30 | DB_PASSWORD= -------------------------------------------------------------------------------- /.github/workflows/writeup-finder-runner.yml: -------------------------------------------------------------------------------- 1 | name: Run Writeup Finder with Supabase 2 | 3 | on: 4 | schedule: 5 | - cron: "0 */3 * * *" # Runs every 3 hours 6 | workflow_dispatch: # Allows manual triggering 7 | 8 | jobs: 9 | writeup-finder: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v4 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v4 18 | with: 19 | go-version: 1.23.5 20 | 21 | - name: Install dependencies 22 | run: go mod tidy 23 | 24 | - name: Create .env file 25 | run: | 26 | echo "TELEGRAM_BOT_TOKEN=${{ secrets.TELEGRAM_BOT_TOKEN }}" >> $GITHUB_WORKSPACE/.env 27 | echo "TELEGRAM_CHANNEL_ID=${{ secrets.TELEGRAM_CHANNEL_ID }}" >> $GITHUB_WORKSPACE/.env 28 | echo "CHAT_ID=${{ secrets.CHAT_ID }}" >> $GITHUB_WORKSPACE/.env 29 | echo "MAIN_THREAD_ID=2" >> $GITHUB_WORKSPACE/.env 30 | echo "PLATFORMS_THREAD_ID=3" >> $GITHUB_WORKSPACE/.env 31 | echo "TRYHACKME_THREAD_ID=4326" >> $GITHUB_WORKSPACE/.env 32 | echo "HACKTHEBOX_THREAD_ID=4327" >> $GITHUB_WORKSPACE/.env 33 | echo "MOBILE_THREAD_ID=4328" >> $GITHUB_WORKSPACE/.env 34 | echo "MONEY_THREAD_ID=4329" >> $GITHUB_WORKSPACE/.env 35 | echo "RECON_THREAD_ID=4330" >> $GITHUB_WORKSPACE/.env 36 | echo "BYPASS_THREAD_ID=4331" >> $GITHUB_WORKSPACE/.env 37 | echo "PORTSWIGGER_THREAD_ID=4332" >> $GITHUB_WORKSPACE/.env 38 | echo "BURPSUITE_THREAD_ID=4333" >> $GITHUB_WORKSPACE/.env 39 | echo "CTF_THREAD_ID=4488" >> $GITHUB_WORKSPACE/.env 40 | echo "OS_THREAD_ID=4863" >> $GITHUB_WORKSPACE/.env 41 | echo "VULNERABILITIES_THREAD_ID=4861" >> $GITHUB_WORKSPACE/.env 42 | echo "TOOLS_THREAD_ID=4865" >> $GITHUB_WORKSPACE/.env 43 | echo "PROGRAMMINGLANGS_THREAD_ID=4867" >> $GITHUB_WORKSPACE/.env 44 | echo "CVE_THREAD_ID=5189" >> $GITHUB_WORKSPACE/.env 45 | echo "OSINT_THREAD_ID=5782" >> $GITHUB_WORKSPACE/.env 46 | echo "STEGANOGRAPHY_THREAD_ID=9239" >> $GITHUB_WORKSPACE/.env 47 | echo "WEBSCRAPING_THREAD_ID=9242" >> $GITHUB_WORKSPACE/.env 48 | echo "CRYPTOGRAPHIC_THREAD_ID=8668" >> $GITHUB_WORKSPACE/.env 49 | echo "YOUTUBE_THREAD_ID=9400" >> $GITHUB_WORKSPACE/.env 50 | echo "DB_HOST=${{ secrets.DB_HOST }}" >> $GITHUB_WORKSPACE/.env 51 | echo "DB_PORT=5432" >> $GITHUB_WORKSPACE/.env 52 | echo "DB_NAME=${{ secrets.DB_NAME }}" >> $GITHUB_WORKSPACE/.env 53 | echo "DB_USER=${{ secrets.DB_USER }}" >> $GITHUB_WORKSPACE/.env 54 | echo "DB_PASSWORD=${{ secrets.DB_PASSWORD }}" >> $GITHUB_WORKSPACE/.env 55 | 56 | - name: Run Writeup Finder 57 | id: run-writeup-finder 58 | run: | 59 | set +e 60 | go run main.go --database --telegram 61 | if [ $? -ne 0 ]; then 62 | echo "::set-output name=error_message::Writeup Finder failed with error" 63 | exit 1 64 | fi 65 | 66 | - name: Notify Error Alarm Channel 67 | if: steps.run-writeup-finder.outputs.error_message 68 | run: | 69 | curl -s -X POST "https://api.telegram.org/bot${{ secrets.TELEGRAM_BOT_TOKEN }}/sendMessage" \ 70 | -d chat_id=${{ secrets.ERROR_ALARM_CHANNEL_ID }} \ 71 | -d text="🚨 Error in Writeup Finder: ${{ steps.run-writeup-finder.outputs.error_message }}" 72 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | writeup-finder -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## [Unreleased] 6 | 7 | ### Added 8 | 9 | - Detect the premium medium article. 10 | - Update the version of Go to 1.23.5 11 | 12 | ### Added 13 | 14 | - Chunked large files into smaller, more manageable files for better maintainability. 15 | - Added comments to improve code readability and documentation. 16 | 17 | ### Added 18 | 19 | - Support for YouTube video RSS feeds. 20 | - New keywords and topics for filtering, including "web scraping" and "steganography". 21 | - Hashnode support for fetching writeups. 22 | - Improved regex patterns for better matching. 23 | - New topic "cryptographic" added to Telegram and script. 24 | 25 | ### Fixed 26 | 27 | - Resolved errors in GitHub Actions workflow. 28 | - Fixed typos in `keywords.json`. 29 | - Improved handling of writeups for today and yesterday. 30 | 31 | ### Changed 32 | 33 | - Updated workflow configurations. 34 | - Enhanced keyword filtering logic. 35 | 36 | ### Added 37 | 38 | - Initial project setup with core functionality for finding and processing writeups. 39 | - Telegram integration for notifications. 40 | - Database support for storing articles. 41 | - GitHub Actions workflow for automated testing and deployment. 42 | 43 | ### Fixed 44 | 45 | - Initial bug fixes and improvements. 46 | 47 | --- 48 | 49 | ## [Older Versions] 50 | 51 | ### Added 52 | 53 | - Early development commits and foundational features. 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Writeup Finder 2 | 3 | [![Go Version](https://img.shields.io/badge/go-1.17%20%7C%201.18%20%7C%201.19%20%7C%201.20-blue)](https://golang.org/dl/) 4 | [![GitHub Issues](https://img.shields.io/github/issues/blackvoidx/writeup-finder)](https://github.com/blackvoidx/writeup-finder/issues) 5 | [![GitHub Stars](https://img.shields.io/github/stars/blackvoidx/writeup-finder)](https://github.com/blackvoidx/writeup-finder/stargazers) 6 | [![GitHub License](https://img.shields.io/github/license/blackvoidx/writeup-finder)](https://github.com/blackvoidx/writeup-finder/blob/master/LICENSE) 7 | 8 |

9 | 10 | 11 | 12 | 13 |

14 | 15 | Join our Writeup Hacking supergroup for curated hacking writeups and resources! 16 | 17 | 📜🔍 https://t.me/writeup_hacking 18 | 19 | Writeup Finder is a tool designed to automatically find and save recent writeups from specified URLs. It supports saving the found writeups in a PostgreSQL database, and sending them directly to a Telegram. 20 | 21 | ``` 22 | Writeup-finder is a tool to search for writeups and manage article data, including sending notifications. 23 | 24 | Usage: 25 | writeup-finder [flags] 26 | writeup-finder [command] 27 | 28 | Available Commands: 29 | completion Generate autocompletion script 30 | help Help about any command 31 | 32 | Flags: 33 | --database Save new articles in the database 34 | --help Show help 35 | --proxy string Proxy URL to use for sending Telegram messages 36 | --telegram Send new articles to Telegram 37 | 38 | Use "writeup-finder [command] --help" for more information about a command. 39 | 40 | ``` 41 | 42 | ## Features 43 | 44 | - Fetch recent writeups from multiple URLs. 45 | - Save writeups to a PostgreSQL database. 46 | - Optionally send notifications of new writeups to a Telegram. 47 | - It filters topics based on the title and sends them to the corresponding topic in the Telegram group. 48 | 49 | ``` 50 | ── .env 51 | ├── .env.example 52 | ├── .github/ 53 | │ └── workflows/ 54 | │ └── writeup-finder-runner.yml 55 | ├── .gitignore 56 | ├── CHANGELOG.md 57 | ├── README.md 58 | ├── command/ 59 | │ ├── action.go 60 | │ ├── command.go 61 | │ ├── completion.go 62 | │ └── flags.go 63 | ├── data/ 64 | │ ├── Youtube_channel.md 65 | │ ├── keywords.json 66 | │ └── url.txt 67 | ├── db/ 68 | │ └── db.go 69 | ├── global/ 70 | │ └── global.go 71 | ├── go.mod 72 | ├── go.sum 73 | ├── handler/ 74 | │ ├── handler.go 75 | │ ├── medium.go 76 | │ ├── utils.go 77 | │ └── youtube.go 78 | ├── main.go 79 | ├── run_writeUp-finder.sh 80 | ├── telegram/ 81 | │ ├── message.go 82 | │ ├── proxy.go 83 | │ ├── request.go 84 | │ └── telegram.go 85 | ├── utils/ 86 | │ ├── env.go 87 | │ ├── filters.go 88 | │ ├── http.go 89 | │ ├── rss.go 90 | │ └── utils.go 91 | └── writeup-finder 92 | ``` 93 | 94 | ## Requirements 95 | 96 | - Go 1.16+ 97 | - PostgreSQL 98 | 99 | ## Setup 100 | 101 | 1. Clone the repository. 102 | 2. Install dependencies using `go mod tidy`. 103 | 3. Create a `.env` file with the `.env.example` file. 104 | 4. Update the `url.txt` file with the URLs you want to monitor. 105 | 5. Run the tool with the desired flags. 106 | 6. Run `go build -o writeup-finder` 107 | 108 | ## Usage 109 | 110 | | Command | Description | 111 | | ------------------------------------------------------------------------- | ------------------------------------------------ | 112 | | `writeup-finder --database` | Save new articles to PostgreSQL database | 113 | | `writeup-finder [--database] --telegram` | Send new writeups to Telegram | 114 | | `writeup-finder [--database] --telegram --proxy=PROTOCOL://HOSTNAME:PORT` | Send new writeups to Telegram with proxy support | 115 | 116 | ## Flags: 117 | - `--database` Save new articles in the database 118 | - `--help` Show help 119 | - `--proxy string` Proxy URL to use for sending Telegram messages 120 | - `--telegram` Send new articles to Telegram 121 | 122 | Use `writeup-finder [command] --help` for more information about a command. 123 | 124 | You can use `CRON` to run script every *hours, *days, or etc. 125 | 126 | #### Example for run script every 3 hour 127 | 128 | More read: [How to Automate Tasks with cron Jobs in Linux](https://www.freecodecamp.org/news/cron-jobs-in-linux/) 129 | 130 | ```bash 131 | 0 */3 * * * cd /path/to/your/script && /usr/local/go/bin/writeup-finder -d -t 132 | ``` 133 | -------------------------------------------------------------------------------- /command/action.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/fatih/color" 8 | "writeup-finder.go/global" 9 | "writeup-finder.go/handler" 10 | "writeup-finder.go/utils" 11 | ) 12 | 13 | // ManageAction processes the list of URLs, finds new articles, and logs the results. 14 | func ManageAction() { 15 | urlList := utils.ReadUrls(global.UrlFile) 16 | today := time.Now() 17 | 18 | // Process the URLs and store new articles in the database if enabled 19 | articlesFound := handler.ProcessUrls(urlList, today, global.DB) 20 | 21 | utils.PrintPretty(fmt.Sprintf("Total new articles found: %d", articlesFound), color.FgYellow, false) 22 | utils.PrintPretty("Writeup Finder Script Completed", color.FgHiYellow, true) 23 | } 24 | -------------------------------------------------------------------------------- /command/command.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/fatih/color" 8 | "github.com/spf13/cobra" 9 | "writeup-finder.go/db" 10 | "writeup-finder.go/global" 11 | "writeup-finder.go/utils" 12 | ) 13 | 14 | // rootCmd is the main command for the writeup-finder CLI tool. 15 | var rootCmd = &cobra.Command{ 16 | Use: "writeup-finder", 17 | Short: "A tool to find writeups and manage articles", 18 | Long: `Writeup-finder is a tool to search for writeups and manage article data, including sending notifications.`, 19 | Run: func(cmd *cobra.Command, args []string) { 20 | if cmd.CalledAs() != "completion" { 21 | // Load environment variables and flags 22 | utils.LoadEnv() 23 | ManageFlags() 24 | utils.PrintPretty("Starting Writeup Finder Script", color.FgHiYellow, true) 25 | 26 | // Connect to the database if enabled 27 | if global.UseDatabase { 28 | global.DB = db.ConnectDB() 29 | db.CreateArticlesTable(global.DB) 30 | defer global.DB.Close() 31 | } 32 | 33 | // Execute main logic of the script 34 | ManageAction() 35 | } 36 | }, 37 | } 38 | 39 | // Execute runs the root command, to be called in main. 40 | func Execute() { 41 | if err := rootCmd.Execute(); err != nil { 42 | fmt.Println(err) 43 | os.Exit(1) 44 | } 45 | } 46 | 47 | // init initializes global flags and subcommands. 48 | func init() { 49 | rootCmd.PersistentFlags().BoolVar(&global.UseDatabase, "database", false, "Save new articles in the database") 50 | rootCmd.PersistentFlags().BoolVar(&global.SendToTelegramFlag, "telegram", false, "Send new articles to Telegram") 51 | rootCmd.PersistentFlags().StringVar(&global.ProxyURL, "proxy", "", "Proxy URL to use for sending Telegram messages") 52 | rootCmd.PersistentFlags().BoolVar(&global.Help, "help", false, "Show help") 53 | 54 | rootCmd.AddCommand(completionCmd) 55 | } 56 | -------------------------------------------------------------------------------- /command/completion.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | // completionCmd generates shell autocompletion scripts. 11 | var completionCmd = &cobra.Command{ 12 | Use: "completion [bash|zsh]", 13 | Short: "Generate autocompletion script", 14 | Long: `To load completions: 15 | 16 | Bash: 17 | 18 | $ source <(writeup-finder completion bash) 19 | 20 | Zsh: 21 | 22 | $ source <(writeup-finder completion zsh) 23 | 24 | # To load completions for each session, execute once: 25 | # Linux: 26 | $ writeup-finder completion zsh > "${fpath[1]}/_writeup-finder" 27 | # macOS: 28 | $ writeup-finder completion zsh > /usr/local/share/zsh/site-functions/_writeup-finder 29 | `, 30 | Args: cobra.ExactArgs(1), 31 | Run: func(cmd *cobra.Command, args []string) { 32 | switch args[0] { 33 | case "bash": 34 | rootCmd.GenBashCompletion(os.Stdout) 35 | case "zsh": 36 | rootCmd.GenZshCompletion(os.Stdout) 37 | default: 38 | fmt.Println("Unsupported shell type. Please specify bash or zsh.") 39 | } 40 | }, 41 | } 42 | -------------------------------------------------------------------------------- /command/flags.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "writeup-finder.go/global" 6 | ) 7 | 8 | // ManageFlags validates and logs the parsed flags. 9 | func ManageFlags() { 10 | ValidateFlags() 11 | 12 | log.Infof("[+] Use Database: %v", global.UseDatabase) 13 | log.Infof("[+] Send to Telegram: %v", global.SendToTelegramFlag) 14 | 15 | if global.ProxyURL != "" { 16 | log.Infof("[+] Proxy URL: %v", global.ProxyURL) 17 | } else { 18 | log.Info("[+] No Proxy URL set.") 19 | } 20 | } 21 | 22 | // ValidateFlags ensures that flag combinations are valid and throws errors for invalid input. 23 | func ValidateFlags() { 24 | if !global.UseDatabase { 25 | log.Fatal("You must specify --database to save articles in the database.") 26 | } 27 | 28 | if global.ProxyURL != "" && !global.SendToTelegramFlag { 29 | log.Fatal("Error: --proxy option is only valid when used with --telegram.") 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /data/Youtube_channel.md: -------------------------------------------------------------------------------- 1 | These Youtube channel is used in script: 2 | 3 | NahamSec: https://www.youtube.com/feeds/videos.xml?channel_id=UCCZDt7MuC3Hzs6IH4xODLBw 4 | PentesterLand: https://www.youtube.com/feeds/videos.xml?channel_id=UCO9Qw6grdoSJhVy8NHjuuiQ 5 | ippsec: https://www.youtube.com/feeds/videos.xml?channel_id=UCa6eh7gCkpPo5XXUDfygQQA 6 | Voorivex: https://www.youtube.com/feeds/videos.xml?channel_id=UCz4A6ALhUVHuiXzoJrIGc1Q 7 | HuntLearnCo: https://www.youtube.com/feeds/videos.xml?channel_id=UCSGYQv26CXeE8q9w2X4NttA 8 | \_JohnHammond: https://www.youtube.com/feeds/videos.xml?channel_id=UCVeW9qkBjo3zosnqUbG7CFw 9 | The_Helpful_Hacker: https://www.youtube.com/feeds/videos.xml?channel_id=UCTnOFmMwZIi7jJYS_nAfeyQ 10 | \_CryptoCat: https://www.youtube.com/feeds/videos.xml?channel_id=UCEeuul0q7C8Zs5C8rc4REFQ 11 | CyberFlow10: https://www.youtube.com/feeds/videos.xml?channel_id=UCUvkPEm38w6pq8w5QYgtPPw 12 | z3nsh3ll: https://www.youtube.com/feeds/videos.xml?channel_id=UCnWAPVvXxivIo40QpjEMU1w 13 | -------------------------------------------------------------------------------- /data/keywords.json: -------------------------------------------------------------------------------- 1 | { 2 | "groups": [ 3 | { 4 | "name": "general", 5 | "keywords": [ 6 | { 7 | "pattern": "\\$[0-9]+|[0-9]+\\$|¥[0-9]+|[0-9]+¥|£[0-9]+|[0-9]+£|€[0-9]+|[0-9]+€|\\bMoney\\b|\\bMy\\sFirst\\sBug\\sBounty\\b|\\bMy\\sFirst\\sBug\\b|\\bMy\\sFirst\\sBounty\\b|\\bVDP\\b", 8 | "threadID": "MONEY_THREAD_ID", 9 | "priority": 2 10 | }, 11 | { 12 | "pattern": "\\bBypass\\b|\\bWAF\\b|\\bfirewall(?:-bypass)?\\b|\\bwaf-bypass\\b", 13 | "threadID": "BYPASS_THREAD_ID", 14 | "priority": 6 15 | }, 16 | { 17 | "pattern": "\\bRecon\\b|\\bReconnaissance\\b", 18 | "threadID": "RECON_THREAD_ID", 19 | "priority": 4 20 | } 21 | ] 22 | }, 23 | { 24 | "name": "platforms", 25 | "keywords": [ 26 | { 27 | "pattern": "\\bHackerOne\\b|\\bBugcrowd\\b|\\bYesWeHack\\b|\\bIntigriti\\b", 28 | "threadID": "PLATFORMS_THREAD_ID", 29 | "priority": 6 30 | } 31 | ] 32 | }, 33 | { 34 | "name": "training", 35 | "keywords": [ 36 | { 37 | "pattern": "\\bTHM\\b|Try\\s?Hack\\s?Me|\\bTRYHACKME\\b|\\bTryHackMe\\b|\\bTryHackme’s\\b", 38 | "threadID": "TRYHACKME_THREAD_ID", 39 | "priority": 5 40 | }, 41 | { 42 | "pattern": "\\bHTB\\b|Hack\\s?The\\s?Box|\\bHACKTHEBOX\\b|\\bHackTheBox\\b", 43 | "threadID": "HACKTHEBOX_THREAD_ID", 44 | "priority": 5 45 | } 46 | ] 47 | }, 48 | { 49 | "name": "technology", 50 | "keywords": [ 51 | { 52 | "pattern": "\\bLinux\\b|\\bUnix\\b|\\bWindows\\b|\\bMac\\s?OS\\b|\\bGNU\\sLinux\\b|\\bUbuntu\\sServer\\b", 53 | "threadID": "OS_THREAD_ID", 54 | "priority": 6 55 | }, 56 | { 57 | "pattern": "\\bMobile\\b|\\bAndroid\\b|\\biOS\\b|\\biPhone\\b|\\biPad\\b|\\bPhone\\b|\\bTablet\\b|\\bSamsung\\b", 58 | "threadID": "MOBILE_THREAD_ID", 59 | "priority": 5 60 | } 61 | ] 62 | }, 63 | { 64 | "name": "tools", 65 | "keywords": [ 66 | { 67 | "pattern": "\\bNessus\\b|\\bMetasploit\\b|\\bZAP\\b|\\bWireshark\\b|\\bNikto\\b|\\bHydra\\b|\\bdig\\b|\\bCurl\\b|\\bSQLMap\\b|\\btplmap\\b|\\bAcunetix\\b|\\bFFUF\\b|\\bWpscan\\b|\\bdirsearch\\b|\\bNmap\\b|\\bX8\\b|\\bKiterunner\\b|\\bArjun\\b|\\bGobuster\\b|\\bBrute\\s?Force\\b|\\bJohn\\s?The\\s?Ripper\\b|\\bHashcat\\b|\\bNetcat\\b|\\bBrowser\\s?Extensions?\\b|\\bWappalyzer\\b|\\bGospider\\b|\\bcrawler\\b|\\bgau\\b|\\bNuclei\\b|\\bhash-identifier\\b|\\bKatana\\b|\\bLinkfinder\\b|\\bUnfurl\\b|\\bwaybackurls\\b|\\bsearchsploit\\b|\\bjwt_tool\\b|\\bcookiemonster\\b|\\bShell\\b|\\bBash\\b|\\bZsh\\b|\\bBrowser\\b|\\bShodan\\b|\\bHandbook\\b|\\bGoogle\\sDorking\\b|\\bCyberchef\\b|\\bChatGPT\\b|\\bSubdomain\\sEnumeration\\b|\\bNetwork\\sScaning|\\bNetwork\\sScan\\b|\\btools\\b|\\btool\\b|\\bVirustotal\\b|\\bHTTPX\\b|\\bDorks?\\b|\\bGoogle\\sDorks?\\b|\\bScanner\\stools?\\b|\\bScanner\\b|\\bPowershell\\b|\\bPort\\sScanning\\b", 68 | "threadID": "TOOLS_THREAD_ID", 69 | "priority": 8 70 | } 71 | ] 72 | }, 73 | { 74 | "name": "vulnerabilities", 75 | "keywords": [ 76 | { 77 | "pattern": "\\bSQL\\sInjection\\b|\\bSQLI\\b|\\bCross(-|\\s)Site\\sScripting\\b|\\bXSS\\b|\\bCommand\\sInjection\\b|\\bRemote\\sCode\\sExecution\\b|\\bRCE\\b|\\bBuffer\\sOverflow\\b|\\bDenial\\sof\\sService\\b|\\bDoS\\b|\\bPath\\sTraversal\\b|\\bLocal\\sFile\\sInclusion\\b|\\bLFI\\b|\\bRemote\\sFile\\sInclusion\\b|\\bRFI\\b|\\bInsecure\\sDirect\\sObject\\sReferences\\b|\\bIDOR\\b|\\bSecurity\\sMisconfiguration\\b|\\bSensitive\\sData\\sExposure\\b|\\bBroken\\sAuthentication\\b|\\bBroken\\sAccess\\sControl\\b|\\bCross(-|\\s)Site\\sRequest\\sForgery\\b|\\bCSRF\\b|\\bClickjacking\\b|\\bXML\\sExternal\\sEntity\\b|\\bXXE\\b|\\bInsecure\\sDeserialization\\b|\\bRace\\sCondition\\b|\\bPrivilege\\sEscalation\\b|\\bInsufficient\\sLogging\\sand\\sMonitoring\\b|\\bServer-Side\\sRequest\\sForgery\\b|\\bSSRF\\b|\\bMass\\sAssignment\\b|\\bOpen\\sRedirect\\b|\\bDirectory\\sTraversal\\b|\\bCross-Origin\\sResource\\sSharing\\sMisconfiguration\\b|\\bCORS\\b|\\bHTTP\\sResponse\\sSplitting\\b|\\bHTTP\\sHeader\\sInjection\\b|\\bSession\\sFixation\\b|\\bCredential\\sStuffing\\b|\\bXML\\sInjection\\b|\\bJSON\\sInjection\\b|\\bCode\\sInjection\\b|\\bCommand\\sExecution\\b|\\bInformation\\sDisclosure\\b|\\bUnvalidated\\sRedirects\\sand\\sForwards\\b|\\bBusiness\\sLogic\\sVulnerabilit(y|ies)\\b|\\bPassword\\sReset\\b|\\bBasic\\sAuthentication\\b|\\bPassword\\sVulnerabilit(y|ies)\\b|\\bOauth\\sMisconfiguration\\b|\\b(web)?\\s?cache(\\s|-)(deception|Poisoning)\\b|\\b(HTTP\\s)?(Request\\s)?Smuggling\\b|\\b(weak\\s)?default(\\s|-)credentials\\b|\\bOpen\\sRedirect\\b|\\bNOSQL\\sinjection\\b|\\bATO\\b|\\bAccount\\sTake\\sOver\\b|\\bAPI\\sSecurity\\b|\\bAPI\\sVulnerabilit(y|ies)\\b|\\bSupply\\sChain\\sAttack\\b|\\bPayment\\sProcess\\sVulnerabilit(y|ies)\\b|\\bcryptographic\\sFailures\\b|\\bCookies\\sSteal\\b|\\bSSO\\b|\\bActive\\sDirectory\\b|\\bFile\\sUpload\\sVulnerabilit(y|ies)\\b|\\bWeb\\sSecurity\\b|\\bOAuth\\b|\\bOAuth2\\b|\\bUnauthorized\\sAccess\\b|\\bPII\\sInformation\\b|\\bP2\\sInformation\\b|\\bData\\sExfiltration\\b|\\bSSTI\\b|\\bRXSS\\b|\\bZero-click\\b|\\bHTML\\sInjection\\b|\\bWifi\\sVulnerabilit(y|ies)\\b|\\bCRLF\\b|\\bDDOS\\b|\\bHTTP\\sParameter\\sPollution\\b|\\bMan(-\\s)in(-\\s)the(-\\s)Middle\\sAttack\\b|\\bprompt\\sinjection\\b|\\bAccount\\sTackover\\b", 78 | "threadID": "VULNERABILITIES_THREAD_ID", 79 | "priority": 7 80 | }, 81 | { 82 | "pattern": "\\bCVE\\b|\\bexploit-db\\b", 83 | "threadID": "CVE_THREAD_ID", 84 | "priority": 3 85 | } 86 | ] 87 | }, 88 | { 89 | "name": "programming_languages", 90 | "keywords": [ 91 | { 92 | "pattern": "\\bGolang\\b|\\bPython\\b|\\bRuby\\b|\\bJavascript\\b|\\bPHP\\b|\\bRust\\b|\\bTypescript\\b|\\bBash\\s?script\\b|\\bShell\\s?Script\\b|\\bLua\\b|\\bDart\\b|\\bJava\\b|\\bNodeJs\\b|\\bWordpress\\b|\\bAngular\\b|\\bDocker\\b|\\bReact\\b|\\bVue\\b|\\bWeb-Server\\b|\\bWeb\\sServer\\b|\\bNginx\\b|\\bApache\\b|\\bCMS\\b|\\bFlask\\b|\\bFastapi\\b|\\bdjango\\b|\\bLaravel\\b|\\bC#\\b", 93 | "threadID": "PROGRAMMINGLANGS_THREAD_ID", 94 | "priority": 9 95 | } 96 | ] 97 | }, 98 | { 99 | "name": "other", 100 | "keywords": [ 101 | { 102 | "pattern": "\\bPort\\s?Swigger\\b", 103 | "threadID": "PORTSWIGGER_THREAD_ID", 104 | "priority": 5 105 | }, 106 | { 107 | "pattern": "\\bBurp\\b|\\bBurp\\s?Suite\\b|\\bBurpsuite-Pro\\b", 108 | "threadID": "BURPSUITE_THREAD_ID", 109 | "priority": 5 110 | }, 111 | { 112 | "pattern": "\\bCTFs\\b|\\bCapture\\s?The\\s?Flag\\b|\\bCTF\\b|\\bpicoctf\\b|\\bvulnhub\\b|\\bUOFTCTF\\b|\\bUOFTCTF-CTF\\b|\\bVulnyx\\b|\\beJPT\\b|\\bHackMyVM\\b|\\bTSCCTF\\b|\\bHacker101\\sCTF\\b", 113 | "threadID": "CTF_THREAD_ID", 114 | "priority": 6 115 | }, 116 | { 117 | "pattern": "\\bOSINT\\b|\\bOpen-Source\\s?Intelligence\\b|\\bOpen\\s?Source\\s?Intelligence\\b|\\bOsint4fun\\b|\\bImage\\sGeolocation\\b|\\bGeolocation\\b|\\bReverse\\sImage\\b", 118 | "threadID": "OSINT_THREAD_ID", 119 | "priority": 3 120 | }, 121 | { 122 | "pattern": "\\bSteganography\\b|\\bStego\\b|\\bHidden\\sData\\b|\\bData\\sHiding\\b|\\bLSB\\b|\\bLeast\\sSignificant\\sBit\\b|\\bImage\\sSteganography\\b|\\bAudio\\sSteganography\\b|\\bVideo\\sSteganography\\b", 123 | "threadID": "STEGANOGRAPHY_THREAD_ID", 124 | "priority": 3 125 | }, 126 | { 127 | "pattern": "\\bWeb\\sScraping\\b|\\bScraping\\b|\\bScraper\\b|\\bData\\sExtraction\\b|\\bBeautifulSoup\\b|\\bSelenium\\b|\\bPuppeteer\\b|\\bWeb\\sCrawler\\b|\\bHTTP\\sRequests\\b|\\bParse\\b", 128 | "threadID": "WEBSCRAPING_THREAD_ID", 129 | "priority": 3 130 | }, 131 | { 132 | "pattern": "\\bCryptography\\b|\\bEncryption\\b|\\bDecryption\\b|\\bHashes\\b|\\bEncoding\\b|\\bDecoding\\b|\\bCipher\\b|\\bSymmetric\\s?Encryption\\b|\\bAsymmetric\\s?Encryption\\b|\\bRSA\\b|\\bAES\\b|\\bElliptic\\s?Curve\\b|\\bKey\\s?Management\\b|\\bDigital\\s?Signature\\b|\\bCryptanalysis\\b|\\bCryptographic\\s?Protocol\\b|\\bHMAC\\b|\\bSHA\\s?1\\b|\\bSHA\\s?2\\b|\\bSHA\\s?3\\b|\\bSHA-256\\b|\\bSHA-512\\b|\\bMD5\\b|\\bPBKDF2\\b|\\bSalt\\b|\\bBcrypt\\b|\\bArgon2\\b|\\bBlowfish\\b|\\bTwofish\\b|\\bSerpent\\b|\\bDES\\b|\\bTriple\\s?DES\\b|\\bGPG\\b|\\bOpenSSL\\b|\\bHashcat\\b|\\bJohn\\s?the\\s?Ripper\\b|\\bCain\\s?and\\s?Abel\\b|\\bBase64\\b|\\bHex\\b|\\bROT13\\b|\\bHashing\\b", 133 | "threadID": "CRYPTOGRAPHIC_THREAD_ID", 134 | "priority": 8 135 | } 136 | ] 137 | } 138 | ] 139 | } 140 | -------------------------------------------------------------------------------- /data/url.txt: -------------------------------------------------------------------------------- 1 | https://medium.com/feed/tag/Steganography 2 | https://medium.com/feed/tag/Image-Steganography 3 | https://medium.com/feed/tag/Digital-Steganography 4 | https://medium.com/feed/tag/Audio-Steganography 5 | https://medium.com/feed/tag/Video-Steganography 6 | https://medium.com/feed/tag/Text-Steganography 7 | https://medium.com/feed/tag/Types-Of-Steganography 8 | 9 | https://www.youtube.com/feeds/videos.xml?channel_id=UCCZDt7MuC3Hzs6IH4xODLBw 10 | https://www.youtube.com/feeds/videos.xml?channel_id=UCO9Qw6grdoSJhVy8NHjuuiQ 11 | https://www.youtube.com/feeds/videos.xml?channel_id=UCa6eh7gCkpPo5XXUDfygQQA 12 | https://www.youtube.com/feeds/videos.xml?channel_id=UCz4A6ALhUVHuiXzoJrIGc1Q 13 | https://www.youtube.com/feeds/videos.xml?channel_id=UCSGYQv26CXeE8q9w2X4NttA 14 | https://www.youtube.com/feeds/videos.xml?channel_id=UCVeW9qkBjo3zosnqUbG7CFw 15 | https://www.youtube.com/feeds/videos.xml?channel_id=UCTnOFmMwZIi7jJYS_nAfeyQ 16 | https://www.youtube.com/feeds/videos.xml?channel_id=UCEeuul0q7C8Zs5C8rc4REFQ 17 | https://www.youtube.com/feeds/videos.xml?channel_id=UCUvkPEm38w6pq8w5QYgtPPw 18 | https://www.youtube.com/feeds/videos.xml?channel_id=UCnWAPVvXxivIo40QpjEMU1w 19 | 20 | https://medium.com/feed/tag/Web-Scrapping 21 | https://medium.com/feed/tag/Web-Scraping 22 | https://medium.com/feed/tag/Web-Scraping-Tools 23 | https://medium.com/feed/tag/Web-Scraping-Tips 24 | 25 | https://medium.com/feed/tag/bug-bounty-program 26 | https://medium.com/feed/tag/bug-hunting 27 | https://medium.com/feed/tag/web-security 28 | https://medium.com/feed/tag/application-security 29 | https://medium.com/feed/tag/penetration-testing 30 | https://medium.com/feed/tag/security-tools 31 | https://medium.com/feed/tag/vulnerability 32 | https://medium.com/feed/tag/web-security-tools 33 | https://medium.com/feed/tag/recon 34 | https://medium.com/feed/tag/tryhackme 35 | https://medium.com/feed/tag/picoctf 36 | https://medium.com/feed/tag/hackthebox-writeup 37 | https://medium.com/feed/tag/hackerone 38 | https://medium.com/feed/tag/intigriti 39 | https://medium.com/feed/tag/osint 40 | https://medium.com/feed/tag/open-source-intelligence 41 | https://medium.com/feed/tag/osint-tools 42 | https://medium.com/feed/tag/intelligence 43 | https://medium.com/feed/tag/cve 44 | https://medium.com/feed/tag/exploit 45 | https://medium.com/feed/tag/hashing 46 | https://medium.com/feed/tag/cryptography 47 | https://medium.com/feed/tag/encryption 48 | https://medium.com/feed/tag/owasp-top-10 49 | https://medium.com/feed/tag/ssrf 50 | https://medium.com/feed/tag/sqli 51 | https://medium.com/feed/tag/cross-site-scripting 52 | https://medium.com/feed/tag/rootme 53 | https://medium.com/feed/tag/portswigger 54 | https://medium.com/feed/tag/burp 55 | https://medium.com/feed/tag/burp-suite 56 | https://medium.com/feed/tag/broken-authentication 57 | https://medium.com/feed/tag/broken-access-control 58 | https://medium.com/feed/tag/mass-hunting 59 | https://medium.com/feed/tag/fuzz 60 | https://medium.com/feed/tag/sqlmap 61 | https://medium.com/feed/tag/path-traversal 62 | https://medium.com/feed/tag/csrf 63 | https://medium.com/feed/tag/cors-misconfiguration 64 | https://medium.com/feed/tag/weak-password 65 | https://medium.com/feed/tag/verb-tampering 66 | https://medium.com/feed/tag/default-credential 67 | https://medium.com/feed/tag/basic-authentication 68 | https://medium.com/feed/tag/reset-password 69 | https://medium.com/feed/tag/open-redirect 70 | https://medium.com/feed/tag/idor 71 | https://medium.com/feed/tag/command-injection 72 | https://medium.com/feed/tag/ssti 73 | https://medium.com/feed/tag/account-takeover 74 | https://medium.com/feed/tag/rce 75 | https://medium.com/feed/tag/insecure-deserialization 76 | https://medium.com/feed/tag/security-misconfiguration 77 | https://medium.com/feed/tag/subdomain-takeover 78 | https://medium.com/feed/tag/web-application-firewalls 79 | https://medium.com/feed/tag/bugcrowd 80 | https://medium.com/feed/tag/yeswehack 81 | https://medium.com/feed/tag/api-security 82 | https://medium.com/feed/tag/appsec 83 | https://medium.com/feed/tag/web-application-security 84 | https://medium.com/feed/tag/ctf 85 | https://medium.com/feed/tag/writeup 86 | https://medium.com/feed/tag/vulnhub 87 | https://medium.com/feed/tag/bug-hunter 88 | https://medium.com/feed/tag/bug-bounty 89 | https://medium.com/feed/tag/hackthebox-writeup 90 | https://medium.com/feed/tag/xss-vulnerability 91 | https://medium.com/feed/tag/rce-vulnerability 92 | https://medium.com/feed/tag/sql-injection 93 | https://medium.com/feed/tag/bug-bounty-writeup 94 | https://medium.com/feed/tag/bugbounty-writeup 95 | https://medium.com/feed/tag/bug-bounty-tips 96 | https://medium.com/feed/tag/mobile-hacking 97 | https://medium.com/feed/tag/android-hacking 98 | 99 | 100 | https://hashnode.com/n/bug-hunting/rss 101 | https://hashnode.com/n/web-security/rss 102 | https://hashnode.com/n/application-security/rss 103 | https://hashnode.com/n/penetration-testing/rss 104 | https://hashnode.com/n/security-tools/rss 105 | https://hashnode.com/n/vulnerability/rss 106 | https://hashnode.com/n/recon/rss 107 | https://hashnode.com/n/tryhackme/rss 108 | https://hashnode.com/n/picoctf/rss 109 | https://hashnode.com/n/hackerone/rss 110 | https://hashnode.com/n/intigriti/rss 111 | https://hashnode.com/n/osint/rss 112 | https://hashnode.com/n/open-source-intelligence/rss 113 | https://hashnode.com/n/intelligence/rss 114 | https://hashnode.com/n/cve/rss 115 | https://hashnode.com/n/exploit/rss 116 | https://hashnode.com/n/hashing/rss 117 | https://hashnode.com/n/cryptography/rss 118 | https://hashnode.com/n/encryption/rss 119 | https://hashnode.com/n/owasp-top-10/rss 120 | https://hashnode.com/n/ssrf/rss 121 | https://hashnode.com/n/sqli/rss 122 | https://hashnode.com/n/cross-site-scripting/rss 123 | https://hashnode.com/n/rootme/rss 124 | https://hashnode.com/n/portswigger/rss 125 | https://hashnode.com/n/burp/rss 126 | https://hashnode.com/n/burp-suite/rss 127 | https://hashnode.com/n/broken-authentication/rss 128 | https://hashnode.com/n/broken-access-control/rss 129 | https://hashnode.com/n/fuzz/rss 130 | https://hashnode.com/n/sqlmap/rss 131 | https://hashnode.com/n/csrf/rss 132 | https://hashnode.com/n/default-credential/rss 133 | https://hashnode.com/n/basic-authentication/rss 134 | https://hashnode.com/n/open-redirect/rss 135 | https://hashnode.com/n/idor/rss 136 | https://hashnode.com/n/command-injection/rss 137 | https://hashnode.com/n/ssti/rss 138 | https://hashnode.com/n/account-takeover/rss 139 | https://hashnode.com/n/rce/rss 140 | https://hashnode.com/n/subdomain-takeover/rss 141 | https://hashnode.com/n/web-application-firewalls/rss 142 | https://hashnode.com/n/bugcrowd/rss 143 | https://hashnode.com/n/api-security/rss 144 | https://hashnode.com/n/appsec/rss 145 | https://hashnode.com/n/web-application-security/rss 146 | https://hashnode.com/n/ctf/rss 147 | https://hashnode.com/n/writeup/rss 148 | https://hashnode.com/n/vulnhub/rss 149 | https://hashnode.com/n/bug-bounty/rss 150 | https://hashnode.com/n/sql-injection/rss 151 | https://hashnode.com/n/bug-bounty-tips/rss 152 | https://hashnode.com/n/android-hacking/rss 153 | 154 | 155 | https://medium.com/feed/@NahamSec 156 | https://medium.com/feed/@jhaddix 157 | https://medium.com/feed/@TomNomNom 158 | https://medium.com/feed/@rAmpancist 159 | https://medium.com/feed/@zseano 160 | https://medium.com/feed/@projectdiscovery 161 | https://medium.com/feed/@infosecwriteups 162 | https://medium.com/feed/@securitylit 163 | https://medium.com/feed/@cappriciosec 164 | https://medium.com/feed/@projectdiscovery 165 | https://medium.com/feed/@newp_th 166 | https://medium.com/feed/@pdelteil 167 | https://ruvlol.medium.com/feed 168 | https://medium.com/@know.0nix/feed 169 | https://medium.com/@bugh4nter/feed 170 | https://seqrity.medium.com/feed 171 | https://vickieli.medium.com/feed 172 | https://medium.com/feed/intigriti 173 | https://medium.com/@intideceukelaire/feed 174 | https://medium.com/@Hacker0x01/feed 175 | https://medium.com/feed/pentesternepal 176 | https://0xjin.medium.com/feed 177 | https://medium.com/@infosecwriteups/feed 178 | https://orwaatyat.medium.com/feed 179 | https://d0nut.medium.com/feed 180 | https://medium.com/feed/towards-aws 181 | https://medium.com/@stackzero/feed 182 | https://surya-dev.medium.com/feed 183 | 184 | https://infosecwriteups.com/feed 185 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "os" 7 | 8 | _ "github.com/lib/pq" // Postgres driver 9 | "github.com/sirupsen/logrus" 10 | "writeup-finder.go/utils" 11 | ) 12 | 13 | // ConnectDB establishes a connection to the PostgreSQL database using environment variables. 14 | // It returns a pointer to the sql.DB object or logs a fatal error if the connection fails. 15 | func ConnectDB() *sql.DB { 16 | connStr := fmt.Sprintf("host=%s port=%s dbname=%s user=%s password=%s sslmode=require", 17 | os.Getenv("DB_HOST"), os.Getenv("DB_PORT"), os.Getenv("DB_NAME"), 18 | os.Getenv("DB_USER"), os.Getenv("DB_PASSWORD")) 19 | 20 | db, err := sql.Open("postgres", connStr) 21 | utils.HandleError(err, "Error in Connection to DB", true) 22 | 23 | logrus.Info("[+] Database connection established.") 24 | return db 25 | } 26 | 27 | // SaveUrlToDB inserts a URL and its corresponding title into the articles table. 28 | // It logs an error if the operation fails but does not stop the program execution. 29 | func SaveUrlToDB(db *sql.DB, url, title string) { 30 | _, err := db.Exec("INSERT INTO articles (url, title) VALUES ($1, $2)", url, title) 31 | utils.HandleError(err, "Error saving URL and title to database", false) 32 | } 33 | 34 | // CreateArticlesTable creates the articles table if it does not already exist. 35 | // The table includes columns for id (primary key), url, and title. 36 | // It logs a fatal error if the table creation fails. 37 | func CreateArticlesTable(db *sql.DB) { 38 | _, err := db.Exec(` 39 | CREATE TABLE IF NOT EXISTS articles ( 40 | id SERIAL PRIMARY KEY, 41 | url VARCHAR(1000), 42 | title VARCHAR(1000) 43 | ); 44 | `) 45 | 46 | utils.HandleError(err, "Error creating articles table", true) 47 | logrus.Info("[+] Articles table created successfully.") 48 | } 49 | -------------------------------------------------------------------------------- /db/db_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/DATA-DOG/go-sqlmock" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | // TestConnectDB tests the ConnectDB function using sqlmock. 11 | func TestConnectDB(t *testing.T) { 12 | // Create a mock database 13 | db, mock, err := sqlmock.New() 14 | assert.NoError(t, err) 15 | defer db.Close() 16 | 17 | // Mock the Ping method to simulate a successful connection 18 | mock.ExpectPing() 19 | 20 | // Call the ConnectDB function (using the mock database) 21 | // Note: In a real test, you would replace sql.Open with a function that returns the mock DB. 22 | // For simplicity, we'll directly use the mock DB here. 23 | err = db.Ping() 24 | assert.NoError(t, err, "Failed to connect to the database") 25 | 26 | // Ensure all expectations were met 27 | assert.NoError(t, mock.ExpectationsWereMet()) 28 | } 29 | 30 | // TestCreateArticlesTable tests the CreateArticlesTable function using sqlmock. 31 | func TestCreateArticlesTable(t *testing.T) { 32 | // Create a mock database 33 | db, mock, err := sqlmock.New() 34 | assert.NoError(t, err) 35 | defer db.Close() 36 | 37 | // Mock the Exec method to simulate table creation 38 | mock.ExpectExec("CREATE TABLE IF NOT EXISTS articles"). 39 | WillReturnResult(sqlmock.NewResult(0, 0)) 40 | 41 | // Call the CreateArticlesTable function 42 | CreateArticlesTable(db) 43 | 44 | // Ensure all expectations were met 45 | assert.NoError(t, mock.ExpectationsWereMet()) 46 | } 47 | 48 | // TestSaveUrlToDB tests the SaveUrlToDB function using sqlmock. 49 | func TestSaveUrlToDB(t *testing.T) { 50 | // Create a mock database 51 | db, mock, err := sqlmock.New() 52 | assert.NoError(t, err) 53 | defer db.Close() 54 | 55 | // Mock the Exec method to simulate inserting a URL and title 56 | mock.ExpectExec("INSERT INTO articles"). 57 | WithArgs("https://example.com", "Example Title"). 58 | WillReturnResult(sqlmock.NewResult(1, 1)) 59 | 60 | // Call the SaveUrlToDB function 61 | SaveUrlToDB(db, "https://example.com", "Example Title") 62 | 63 | // Ensure all expectations were met 64 | assert.NoError(t, mock.ExpectationsWereMet()) 65 | } 66 | -------------------------------------------------------------------------------- /global/global.go: -------------------------------------------------------------------------------- 1 | package global 2 | 3 | import "database/sql" 4 | 5 | const ( 6 | DataFolder = "data/" 7 | UrlFile = DataFolder + "url.txt" 8 | DateFormat = "2006-01-02" 9 | ) 10 | 11 | var ( 12 | DB *sql.DB 13 | UseDatabase bool 14 | SendToTelegramFlag bool 15 | ProxyURL string 16 | Help bool 17 | ) 18 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module writeup-finder.go 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.5 6 | 7 | require ( 8 | github.com/DATA-DOG/go-sqlmock v1.5.2 9 | github.com/fatih/color v1.17.0 10 | github.com/joho/godotenv v1.5.1 11 | github.com/lib/pq v1.10.9 12 | github.com/mmcdole/gofeed v1.3.0 13 | github.com/sirupsen/logrus v1.9.3 14 | github.com/spf13/cobra v1.8.1 15 | github.com/stretchr/testify v1.10.0 16 | ) 17 | 18 | require ( 19 | github.com/PuerkitoBio/goquery v1.8.0 // indirect 20 | github.com/andybalholm/cascadia v1.3.1 // indirect 21 | github.com/chromedp/cdproto v0.0.0-20250126231910-1730200a0f74 // indirect 22 | github.com/chromedp/chromedp v0.12.1 // indirect 23 | github.com/chromedp/sysutil v1.1.0 // indirect 24 | github.com/davecgh/go-spew v1.1.1 // indirect 25 | github.com/gobwas/httphead v0.1.0 // indirect 26 | github.com/gobwas/pool v0.2.1 // indirect 27 | github.com/gobwas/ws v1.4.0 // indirect 28 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 29 | github.com/josharian/intern v1.0.0 // indirect 30 | github.com/json-iterator/go v1.1.12 // indirect 31 | github.com/mailru/easyjson v0.9.0 // indirect 32 | github.com/mattn/go-colorable v0.1.13 // indirect 33 | github.com/mattn/go-isatty v0.0.20 // indirect 34 | github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect 35 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 36 | github.com/modern-go/reflect2 v1.0.2 // indirect 37 | github.com/pmezard/go-difflib v1.0.0 // indirect 38 | github.com/spf13/pflag v1.0.5 // indirect 39 | golang.org/x/net v0.4.0 // indirect 40 | golang.org/x/sys v0.29.0 // indirect 41 | golang.org/x/text v0.5.0 // indirect 42 | gopkg.in/yaml.v3 v3.0.1 // indirect 43 | ) 44 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= 2 | github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= 3 | github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= 4 | github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= 5 | github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= 6 | github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= 7 | github.com/chromedp/cdproto v0.0.0-20250126231910-1730200a0f74 h1:oul0R+ZyorVxVNr7bALbPolKvbXlpIefB4lDv5sjt00= 8 | github.com/chromedp/cdproto v0.0.0-20250126231910-1730200a0f74/go.mod h1:RTGuBeCeabAJGi3OZf71a6cGa7oYBfBP75VJZFLv6SU= 9 | github.com/chromedp/chromedp v0.12.1 h1:kBMblXk7xH5/6j3K9uk8d7/c+fzXWiUsCsPte0VMwOA= 10 | github.com/chromedp/chromedp v0.12.1/go.mod h1:F6+wdq9LKFDMoyxhq46ZLz4VLXrsrCAR3sFqJz4Nqc0= 11 | github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= 12 | github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= 13 | github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 14 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 16 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 17 | github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= 18 | github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= 19 | github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= 20 | github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= 21 | github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= 22 | github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 23 | github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= 24 | github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= 25 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 26 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 27 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 28 | github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 29 | github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 30 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 31 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 32 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 33 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 34 | github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= 35 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 36 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 37 | github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= 38 | github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= 39 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 40 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 41 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 42 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 43 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 44 | github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= 45 | github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= 46 | github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= 47 | github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= 48 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 49 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 50 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 51 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 52 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 53 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 54 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 55 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 56 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 57 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 58 | github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= 59 | github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= 60 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 61 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 62 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 63 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 64 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 65 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 66 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 67 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 68 | golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= 69 | golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= 70 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 71 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 72 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 73 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 74 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 75 | golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= 76 | golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 77 | golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= 78 | golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 79 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 80 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 81 | golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= 82 | golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 83 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 84 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 85 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 86 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 87 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 88 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 89 | -------------------------------------------------------------------------------- /handler/handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/fatih/color" 9 | "writeup-finder.go/utils" 10 | ) 11 | 12 | // ProcessUrls iterates over a list of URLs and processes each one based on its type (Medium or YouTube). 13 | func ProcessUrls(urlList []string, today time.Time, database *sql.DB) int { 14 | articlesFound := 0 15 | 16 | for i, url := range urlList { 17 | utils.PrintPretty(fmt.Sprintf("Processing feed: %s", url), color.FgMagenta, false) 18 | 19 | // Determine the type of feed and process accordingly 20 | if IsYouTubeFeed(url) { 21 | videosFound := ProcessYouTubeFeed(url, today, database) 22 | articlesFound += videosFound 23 | } else { 24 | articlesFound += ProcessMediumFeed(url, today, database) 25 | } 26 | 27 | // Delay processing of the next URL to prevent rate-limiting or server overload 28 | if i < len(urlList)-1 { 29 | time.Sleep(3 * time.Second) 30 | } 31 | } 32 | 33 | return articlesFound 34 | } 35 | -------------------------------------------------------------------------------- /handler/medium.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "github.com/fatih/color" 10 | "writeup-finder.go/utils" 11 | ) 12 | 13 | // processMediumFeed fetches and processes articles from a Medium RSS feed. 14 | func ProcessMediumFeed(url string, today time.Time, database *sql.DB) int { 15 | articlesFound := 0 16 | articles, err := utils.FetchArticles(url) 17 | if err != nil { 18 | log.Printf("Error fetching articles from %s: %v", url, err) 19 | return 0 20 | } 21 | 22 | for _, article := range articles { 23 | if IsNewArticle(article, database, today) { 24 | message := FormatArticleMessage(article) 25 | if err := HandleArticle(article, message, database, false); err != nil { 26 | log.Printf("Error handling article %s: %v", article.GUID, err) 27 | continue 28 | } 29 | fmt.Println(color.GreenString(message)) 30 | articlesFound++ 31 | } 32 | } 33 | return articlesFound 34 | } 35 | -------------------------------------------------------------------------------- /handler/utils.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "log" 8 | "strings" 9 | "time" 10 | 11 | "github.com/chromedp/chromedp" 12 | "github.com/mmcdole/gofeed" 13 | "writeup-finder.go/db" 14 | "writeup-finder.go/global" 15 | "writeup-finder.go/telegram" 16 | "writeup-finder.go/utils" 17 | ) 18 | 19 | // isYouTubeFeed determines if a given URL corresponds to a YouTube RSS feed. 20 | func IsYouTubeFeed(url string) bool { 21 | return strings.HasPrefix(url, "https://www.youtube.com/feeds/") 22 | } 23 | 24 | // IsNewArticle checks if an article is new by comparing its publication date and database presence. 25 | func IsNewArticle(item *gofeed.Item, db *sql.DB, today time.Time) bool { 26 | pubDate, err := utils.ParseDate(item.Published) 27 | if err != nil { 28 | return false 29 | } 30 | 31 | yesterday := today.AddDate(0, 0, -1) 32 | isToday := pubDate.Format(global.DateFormat) == today.Format(global.DateFormat) 33 | isYesterday := pubDate.Format(global.DateFormat) == yesterday.Format(global.DateFormat) 34 | 35 | if !isToday && !isYesterday { 36 | return false 37 | } 38 | 39 | var exists bool 40 | query := "SELECT EXISTS(SELECT 1 FROM articles WHERE title = $1)" 41 | err = db.QueryRow(query, item.Title).Scan(&exists) 42 | utils.HandleError(err, "Error checking if article title exists in database", false) 43 | 44 | return !exists 45 | } 46 | 47 | // FormatArticleMessage creates a formatted string for an article's details. 48 | func FormatArticleMessage(item *gofeed.Item) string { 49 | // Check if the GUID starts with "https://medium.com" 50 | if !strings.HasPrefix(item.GUID, "https://medium.com") { 51 | return fmt.Sprintf("\u25BA %s\nPublished: %s\nLink: %s", item.Title, item.Published, item.GUID) 52 | } 53 | 54 | premium, err := isPremium(item.GUID) 55 | if err != nil { 56 | log.Printf("Error checking premium status for URL %s: %v. Skipping URL.", item.GUID, err) 57 | return fmt.Sprintf("\u25BA %s\nPublished: %s\nLink: %s", item.Title, item.Published, item.GUID) 58 | } 59 | 60 | // If the article is premium, change the domain 61 | if premium { 62 | item.GUID = strings.Replace(item.GUID, "https://medium.com", "https://freedium.cfd", 1) 63 | } 64 | 65 | return fmt.Sprintf("\u25BA %s\nPublished: %s\nLink: %s", item.Title, item.Published, item.GUID) 66 | } 67 | 68 | func isPremium(url string) (bool, error) { 69 | // Custom user agent and allocator options 70 | opts := append(chromedp.DefaultExecAllocatorOptions[:], 71 | chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"), 72 | chromedp.Flag("no-sandbox", true), 73 | ) 74 | 75 | // Create a new context with the allocator 76 | ctx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) 77 | defer cancel() 78 | 79 | // Create a new browser context 80 | ctx, cancel = chromedp.NewContext(ctx) 81 | defer cancel() 82 | 83 | // Set a timeout for the entire operation 84 | ctx, cancel = context.WithTimeout(ctx, 60*time.Second) // Extended timeout 85 | defer cancel() 86 | 87 | var isPremium bool 88 | 89 | // Run the browser tasks 90 | err := chromedp.Run(ctx, 91 | chromedp.Navigate(url), 92 | chromedp.WaitReady("body"), // Wait for the body to load 93 | chromedp.Evaluate(`document.querySelectorAll('[aria-label="Close"]').forEach(btn => btn.click());`, nil), // Close popups 94 | chromedp.Evaluate(`{ 95 | const xpathCheck = document.evaluate( 96 | '//*[contains(text(), "Member-only story")]', 97 | document, 98 | null, 99 | XPathResult.ANY_TYPE, 100 | null 101 | ); 102 | const hasMemberText = xpathCheck.iterateNext() !== null; 103 | 104 | const hasGoldenStar = document.querySelector('svg[fill="#FFC017"]') !== null; 105 | 106 | hasMemberText || hasGoldenStar; 107 | }`, &isPremium), 108 | ) 109 | 110 | if err != nil { 111 | return false, fmt.Errorf("error checking premium status for %s: %v", url, err) 112 | } 113 | 114 | return isPremium, nil 115 | } 116 | 117 | // HandleArticle manages sending an article to Telegram and saving it to the database if enabled. 118 | func HandleArticle(item *gofeed.Item, message string, database *sql.DB, isYoutube bool) error { 119 | if global.SendToTelegramFlag { 120 | fmt.Println("Start Send to Telegram...") 121 | 122 | telegram.SendToTelegram(message, global.ProxyURL, item.Title, isYoutube) 123 | } 124 | 125 | if global.UseDatabase { 126 | db.SaveUrlToDB(database, item.GUID, item.Title) 127 | } 128 | 129 | return nil 130 | } 131 | -------------------------------------------------------------------------------- /handler/youtube.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "github.com/fatih/color" 10 | "github.com/mmcdole/gofeed" 11 | "writeup-finder.go/global" 12 | "writeup-finder.go/utils" 13 | ) 14 | 15 | // processYouTubeFeed fetches and processes videos from a YouTube RSS feed. 16 | func ProcessYouTubeFeed(url string, today time.Time, database *sql.DB) int { 17 | articlesFound := 0 18 | feedParser := gofeed.NewParser() 19 | 20 | feed, err := feedParser.ParseURL(url) 21 | if err != nil { 22 | log.Printf("Error fetching YouTube feed from %s: %v", url, err) 23 | return 0 24 | } 25 | 26 | for _, item := range feed.Items { 27 | pubDate, err := time.Parse(time.RFC3339, item.Published) 28 | if err != nil { 29 | log.Printf("Error parsing publication date for YouTube video: %v", err) 30 | continue 31 | } 32 | 33 | // Determine if the video is new based on publication date 34 | yesterday := today.AddDate(0, 0, -1) 35 | isToday := pubDate.Format(global.DateFormat) == today.Format(global.DateFormat) 36 | isYesterday := pubDate.Format(global.DateFormat) == yesterday.Format(global.DateFormat) 37 | 38 | if !isToday && !isYesterday { 39 | continue 40 | } 41 | 42 | // Check for the video's existence in the database 43 | var exists bool 44 | query := "SELECT EXISTS(SELECT 1 FROM articles WHERE url = $1)" 45 | err = database.QueryRow(query, item.Link).Scan(&exists) 46 | utils.HandleError(err, "Error checking if YouTube video link exists in database", false) 47 | 48 | if exists { 49 | continue 50 | } 51 | 52 | article := &gofeed.Item{ 53 | GUID: item.Link, 54 | Title: item.Title, 55 | Published: item.Published, 56 | } 57 | message := FormatArticleMessage(article) 58 | 59 | if err := HandleArticle(article, message, database, true); err != nil { 60 | log.Printf("Error handling YouTube video %s: %v", item.Link, err) 61 | continue 62 | } 63 | fmt.Println(color.GreenString(message)) 64 | articlesFound++ 65 | } 66 | return articlesFound 67 | } 68 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "writeup-finder.go/command" 5 | 6 | _ "github.com/lib/pq" // Import PostgreSQL driver for database/sql 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | // init configures the logrus logger format. 11 | // It disables timestamps, level truncation, and full timestamps for cleaner log output. 12 | func init() { 13 | log.SetFormatter(&log.TextFormatter{ 14 | DisableTimestamp: true, // Remove timestamps from log output 15 | DisableLevelTruncation: true, // Prevent truncation of log levels 16 | FullTimestamp: false, // Do not use full timestamps 17 | }) 18 | } 19 | 20 | // main is the entry point of the application. 21 | // It calls the Execute function from the command package to start the application. 22 | func main() { 23 | command.Execute() 24 | } 25 | -------------------------------------------------------------------------------- /run_writeUp-finder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source /home/mohammad/Videos/go/proxy.env 4 | 5 | # Define variables 6 | SCRIPT_PATH="$HOME/Videos/go/writeup-finder" # Change this to your actual script directory 7 | 8 | # Function to check proxy connection 9 | check_proxy() { 10 | nc -zv "$PROXY_HOST" "$PROXY_PORT" >/dev/null 2>&1 11 | if [ $? -eq 0 ]; then 12 | return 0 13 | else 14 | return 1 15 | fi 16 | } 17 | 18 | # Function to check if Windscribe VPN is up 19 | check_windscribe() { 20 | pgrep -l windscribe >/dev/null 2>&1 21 | if [ $? -eq 0 ]; then 22 | return 0 23 | else 24 | return 1 25 | fi 26 | } 27 | 28 | # Main logic 29 | if check_windscribe; then 30 | echo "Windscribe VPN is up, running the writeup-finder script without proxy..." 31 | cd "$SCRIPT_PATH" || { echo "Failed to change directory to $SCRIPT_PATH"; exit 1; } 32 | $HOME/Videos/go/writeup-finder/writeup-finder --database --telegram 33 | 34 | elif check_proxy; then 35 | echo "Proxy is up, running the writeup-finder script with proxy..." 36 | cd "$SCRIPT_PATH" || { echo "Failed to change directory to $SCRIPT_PATH"; exit 1; } 37 | $HOME/Videos/go/writeup-finder/writeup-finder --database --telegram --proxy="$PROXY" 38 | 39 | else 40 | echo "Neither Windscribe VPN nor proxy is available. Skipping this attempt." 41 | fi 42 | 43 | -------------------------------------------------------------------------------- /telegram/message.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | // TelegramMessage represents the structure of a message to be sent to Telegram. 4 | type TelegramMessage struct { 5 | ChatID string `json:"chat_id"` 6 | Text string `json:"text"` 7 | MessageThreadID string `json:"message_thread_id,omitempty"` 8 | } 9 | -------------------------------------------------------------------------------- /telegram/proxy.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | 7 | "writeup-finder.go/utils" 8 | ) 9 | 10 | // ValidateProxyURL checks if the provided proxy URL is valid and supported. 11 | // It returns an error if the scheme is unsupported or the hostname is missing. 12 | func ValidateProxyURL(proxyURL string) error { 13 | parsedURL, err := url.Parse(proxyURL) 14 | utils.HandleError(err, "Error:", true) 15 | 16 | switch parsedURL.Scheme { 17 | case "http", "https", "socks5": 18 | default: 19 | return fmt.Errorf("unsupported proxy scheme: %s", parsedURL.Scheme) 20 | } 21 | 22 | if parsedURL.Hostname() == "" { 23 | return fmt.Errorf("missing hostname or IP address in proxy URL") 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /telegram/request.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "net" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/fatih/color" 12 | ) 13 | 14 | // sendRequest sends an HTTP POST request to the Telegram API. 15 | // It handles retries for network errors, rate limiting, and unexpected status codes. 16 | func SendRequest(client *http.Client, apiURL string, jsonData []byte, retryCount *int) error { 17 | resp, err := client.Post(apiURL, "application/json", bytes.NewBuffer(jsonData)) 18 | if err != nil { 19 | if netErr, ok := err.(net.Error); ok && netErr.Timeout() { 20 | fmt.Println(color.RedString("Network timeout, retrying...")) 21 | (*retryCount)++ 22 | return err // Return the error to trigger a retry 23 | } 24 | (*retryCount)++ 25 | return err // Increment retry count for non-retryable errors 26 | } 27 | defer resp.Body.Close() 28 | 29 | if resp.StatusCode == http.StatusOK { 30 | return nil // Success, no need to retry 31 | } 32 | 33 | // Handle rate limiting 34 | if resp.StatusCode == http.StatusTooManyRequests { 35 | retryAfter := time.Duration(rateLimitBase<<*retryCount) * time.Second 36 | fmt.Println(color.YellowString("Rate limit exceeded, retrying after %v...", retryAfter)) 37 | (*retryCount)++ 38 | time.Sleep(retryAfter) 39 | return fmt.Errorf("rate limit exceeded") 40 | } 41 | 42 | // Log unexpected HTTP status codes 43 | log.Printf("Unexpected status code %d: retrying...", resp.StatusCode) 44 | (*retryCount)++ 45 | return fmt.Errorf("failed to send message, status code: %d", resp.StatusCode) 46 | } 47 | -------------------------------------------------------------------------------- /telegram/telegram.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | "writeup-finder.go/utils" 10 | ) 11 | 12 | const ( 13 | maxRetries = 5 // Maximum number of retries for sending a message 14 | retryDelay = 2 * time.Second // Delay between retries 15 | rateLimitBase = 2 // Base multiplier for rate limit backoff 16 | ) 17 | 18 | // SendToTelegram sends a message to a Telegram channel using the provided proxy. 19 | // It handles retries, rate limiting, and thread selection based on the message type (YouTube or keyword-based). 20 | func SendToTelegram(message string, proxyURL string, title string, isYoutube bool) { 21 | botToken := utils.GetEnv("TELEGRAM_BOT_TOKEN") 22 | channelID := utils.GetEnv("CHAT_ID") 23 | mainThreadID := utils.GetEnv("MAIN_THREAD_ID") 24 | youtubeThreadID := utils.GetEnv("YOUTUBE_THREAD_ID") 25 | 26 | var messageThreadID string 27 | 28 | if isYoutube { 29 | messageThreadID = youtubeThreadID 30 | } else { 31 | // Load keywords from the JSON configuration 32 | keywords, err := utils.LoadKeywords("data/keywords.json") 33 | if err != nil { 34 | utils.HandleError(err, "Failed to load keyword patterns", true) 35 | } 36 | 37 | // Determine the message thread ID based on title keywords 38 | messageThreadID = utils.MatchKeyword(title, keywords, mainThreadID) 39 | } 40 | 41 | apiURL := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", botToken) 42 | telegramMessage := TelegramMessage{ 43 | ChatID: channelID, 44 | Text: message, 45 | MessageThreadID: messageThreadID, 46 | } 47 | 48 | jsonData, err := json.Marshal(telegramMessage) 49 | utils.HandleError(err, "Error marshalling Telegram message", false) 50 | 51 | client := utils.CreateHTTPClient(proxyURL) 52 | retryCount := 0 53 | 54 | for { 55 | err := SendRequest(client, apiURL, jsonData, &retryCount) 56 | if err != nil { 57 | if retryCount >= maxRetries { 58 | log.Printf("Failed to send message to Telegram after %d retries: %v", maxRetries, err) 59 | return 60 | } 61 | log.Printf("Retrying request (%d/%d): %v", retryCount, maxRetries, err) 62 | time.Sleep(retryDelay) // Wait before retrying 63 | continue 64 | } 65 | log.Println("Message sent successfully!") 66 | break // Exit the loop if request was successful 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /utils/env.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | 7 | "github.com/joho/godotenv" 8 | ) 9 | 10 | // LoadEnv loads environment variables from a `.env` file located in the specified directory. 11 | // If the `.env` file cannot be loaded, it logs a fatal error and exits the program. 12 | // The `.env` file path is determined by joining the `GITHUB_WORKSPACE` environment variable with `.env`. 13 | func LoadEnv() { 14 | envFile := filepath.Join(os.Getenv("GITHUB_WORKSPACE"), ".env") 15 | err := godotenv.Load(envFile) 16 | 17 | HandleError(err, "Error loading .env file:", true) 18 | } 19 | -------------------------------------------------------------------------------- /utils/filters.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "regexp" 8 | "sort" 9 | ) 10 | 11 | // KeywordPattern represents a compiled regex pattern, its associated thread ID, and priority. 12 | type KeywordPattern struct { 13 | Pattern *regexp.Regexp 14 | ThreadID string 15 | Priority int 16 | } 17 | 18 | // RawKeyword represents a keyword pattern and its associated thread ID and priority as loaded from JSON. 19 | type RawKeyword struct { 20 | Pattern string `json:"pattern"` 21 | ThreadID string `json:"threadID"` 22 | Priority int `json:"priority"` 23 | } 24 | 25 | // KeywordGroup represents a group of keywords with a common name. 26 | type KeywordGroup struct { 27 | Name string `json:"name"` 28 | Keywords []RawKeyword `json:"keywords"` 29 | } 30 | 31 | // LoadKeywords loads keyword patterns from a JSON configuration file and compiles them into regex patterns. 32 | // It also maps thread IDs from environment variables and sorts the keywords by priority. 33 | // Returns a slice of KeywordPattern or an error if the file cannot be read or the regex cannot be compiled. 34 | func LoadKeywords(configPath string) ([]KeywordPattern, error) { 35 | // Map thread IDs from environment variables 36 | threadIDMap := map[string]string{ 37 | "MONEY_THREAD_ID": GetEnv("MONEY_THREAD_ID"), 38 | "BYPASS_THREAD_ID": GetEnv("BYPASS_THREAD_ID"), 39 | "PLATFORMS_THREAD_ID": GetEnv("PLATFORMS_THREAD_ID"), 40 | "TRYHACKME_THREAD_ID": GetEnv("TRYHACKME_THREAD_ID"), 41 | "HACKTHEBOX_THREAD_ID": GetEnv("HACKTHEBOX_THREAD_ID"), 42 | "MOBILE_THREAD_ID": GetEnv("MOBILE_THREAD_ID"), 43 | "RECON_THREAD_ID": GetEnv("RECON_THREAD_ID"), 44 | "PORTSWIGGER_THREAD_ID": GetEnv("PORTSWIGGER_THREAD_ID"), 45 | "BURPSUITE_THREAD_ID": GetEnv("BURPSUITE_THREAD_ID"), 46 | "CTF_THREAD_ID": GetEnv("CTF_THREAD_ID"), 47 | "OS_THREAD_ID": GetEnv("OS_THREAD_ID"), 48 | "VULNERABILITIES_THREAD_ID": GetEnv("VULNERABILITIES_THREAD_ID"), 49 | "TOOLS_THREAD_ID": GetEnv("TOOLS_THREAD_ID"), 50 | "PROGRAMMINGLANGS_THREAD_ID": GetEnv("PROGRAMMINGLANGS_THREAD_ID"), 51 | "CVE_THREAD_ID": GetEnv("CVE_THREAD_ID"), 52 | "OSINT_THREAD_ID": GetEnv("OSINT_THREAD_ID"), 53 | "CRYPTOGRAPHIC_THREAD_ID": GetEnv("CRYPTOGRAPHIC_THREAD_ID"), 54 | "STEGANOGRAPHY_THREAD_ID": GetEnv("STEGANOGRAPHY_THREAD_ID"), 55 | "WEBSCRAPING_THREAD_ID": GetEnv("WEBSCRAPING_THREAD_ID"), 56 | } 57 | 58 | // Load JSON configuration file 59 | file, err := os.Open(configPath) 60 | if err != nil { 61 | return nil, err 62 | } 63 | defer file.Close() 64 | 65 | var rawConfig struct { 66 | Groups []KeywordGroup `json:"groups"` 67 | } 68 | 69 | if err := json.NewDecoder(file).Decode(&rawConfig); err != nil { 70 | return nil, err 71 | } 72 | 73 | // Parse keywords and compile regex patterns 74 | var keywords []KeywordPattern 75 | for _, group := range rawConfig.Groups { 76 | for _, raw := range group.Keywords { 77 | compiledPattern, err := regexp.Compile("(?i)" + raw.Pattern) 78 | if err != nil { 79 | return nil, err // Return an error if regex compilation fails 80 | } 81 | threadID, ok := threadIDMap[raw.ThreadID] 82 | if !ok { 83 | return nil, fmt.Errorf("unknown thread ID: %s", raw.ThreadID) 84 | } 85 | keywords = append(keywords, KeywordPattern{ 86 | Pattern: compiledPattern, 87 | ThreadID: threadID, 88 | Priority: raw.Priority, 89 | }) 90 | } 91 | } 92 | 93 | // Sort keywords by priority (ascending order) 94 | sort.Slice(keywords, func(i, j int) bool { 95 | return keywords[i].Priority < keywords[j].Priority 96 | }) 97 | 98 | return keywords, nil 99 | } 100 | 101 | // MatchKeyword searches for the first keyword pattern that matches the given title. 102 | // It returns the associated thread ID if a match is found, otherwise returns the default thread ID. 103 | func MatchKeyword(title string, keywords []KeywordPattern, defaultThreadID string) string { 104 | for _, keyword := range keywords { 105 | if keyword.Pattern.MatchString(title) { 106 | return keyword.ThreadID 107 | } 108 | } 109 | return defaultThreadID 110 | } 111 | -------------------------------------------------------------------------------- /utils/http.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "time" 7 | ) 8 | 9 | // CreateHTTPClient creates and returns an HTTP client with a 30-second timeout. 10 | // If a proxy URL is provided, it configures the client to use the proxy. 11 | // If the proxy URL is invalid, the function logs an error and returns a client without proxy settings. 12 | func CreateHTTPClient(proxyURL string) *http.Client { 13 | client := &http.Client{ 14 | Timeout: 30 * time.Second, // Set a 30-second timeout for all requests 15 | } 16 | 17 | if proxyURL != "" { 18 | proxy, err := url.Parse(proxyURL) 19 | if err != nil { 20 | HandleError(err, "Error parsing proxy URL", false) 21 | return client 22 | } 23 | client.Transport = &http.Transport{ 24 | Proxy: http.ProxyURL(proxy), 25 | } 26 | } 27 | 28 | return client 29 | } 30 | -------------------------------------------------------------------------------- /utils/rss.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/mmcdole/gofeed" 9 | ) 10 | 11 | // FetchArticles retrieves articles from the given RSS feed URL. 12 | // It returns a list of feed items or an error if the request or parsing fails. 13 | func FetchArticles(feedURL string) ([]*gofeed.Item, error) { 14 | client := &http.Client{ 15 | Timeout: time.Second * 10, // Set a timeout for the request 16 | } 17 | 18 | req, err := http.NewRequest("GET", feedURL, nil) 19 | if err != nil { 20 | HandleError(err, "Error creating request", false) 21 | return nil, err 22 | } 23 | 24 | // Set headers to mimic a browser request 25 | req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:91.0) Gecko/20100101 Firefox/91.0") 26 | req.Header.Set("Accept", "application/rss+xml, application/xml;q=0.9, */*;q=0.8") 27 | 28 | resp, err := client.Do(req) 29 | if err != nil { 30 | HandleError(err, "Error fetching feed", false) 31 | return nil, err 32 | } 33 | defer resp.Body.Close() 34 | 35 | // Check for non-2xx HTTP status codes 36 | if resp.StatusCode < 200 || resp.StatusCode >= 300 { 37 | err := fmt.Errorf("HTTP error: %s, status code: %d", resp.Status, resp.StatusCode) 38 | HandleError(err, "Invalid HTTP response", false) 39 | return nil, err 40 | } 41 | 42 | parser := gofeed.NewParser() 43 | feed, err := parser.Parse(resp.Body) 44 | if err != nil { 45 | HandleError(err, "Error parsing feed", false) 46 | return nil, err 47 | } 48 | 49 | if feed == nil { 50 | err := fmt.Errorf("no feed data received from URL: %s", feedURL) 51 | HandleError(err, "Nil feed data", false) 52 | return nil, err 53 | } 54 | 55 | return feed.Items, nil 56 | } 57 | 58 | // ParseDate attempts to parse a date string using RFC1123Z or RFC1123 formats. 59 | // It returns the parsed time.Time object and any error encountered during parsing. 60 | func ParseDate(dateString string) (time.Time, error) { 61 | parsedTime, err := time.Parse(time.RFC1123Z, dateString) 62 | if err != nil { 63 | // Attempt parsing with an alternative format 64 | parsedTime, err = time.Parse(time.RFC1123, dateString) 65 | } 66 | 67 | HandleError(err, "Error parsing date", false) 68 | 69 | return parsedTime, err 70 | } 71 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "github.com/fatih/color" 11 | ) 12 | 13 | // ReadUrls reads a list of URLs from a text file and returns them as a slice of strings. 14 | // It trims whitespace and skips empty lines. If the file cannot be opened or read, it logs an error. 15 | func ReadUrls(filePath string) []string { 16 | file, err := os.Open(filePath) 17 | HandleError(err, "Error opening URL file", false) 18 | defer file.Close() 19 | 20 | var urls []string 21 | scanner := bufio.NewScanner(file) 22 | for scanner.Scan() { 23 | line := strings.TrimSpace(scanner.Text()) 24 | if line != "" { 25 | urls = append(urls, line) 26 | } 27 | } 28 | 29 | if err := scanner.Err(); err != nil { 30 | HandleError(err, "Error scanning URL file", false) 31 | } 32 | 33 | return urls 34 | } 35 | 36 | // HandleError logs an error message with optional coloring and exits the program if specified. 37 | // It is used to handle errors consistently across the application. 38 | func HandleError(err error, message string, exit bool) { 39 | if err != nil { 40 | fmt.Println(color.RedString("%s: %s", message, err)) 41 | if exit { 42 | os.Exit(1) 43 | } 44 | } 45 | } 46 | 47 | // PrintPretty prints a message with optional coloring and formatting. 48 | // If isTitle is true, it centers the message and adds a border for emphasis. 49 | // Otherwise, it prints the message with a timestamp. 50 | func PrintPretty(message string, colorAttr color.Attribute, isTitle bool) { 51 | timestamp := time.Now().Format("2006-01-02 15:04:05") 52 | colored := color.New(colorAttr).SprintFunc() 53 | 54 | if isTitle { 55 | width := 80 56 | padding := (width - len(message)) / 2 57 | fmt.Println(colored(strings.Repeat("=", width))) 58 | fmt.Printf("%s%s%s\n", strings.Repeat(" ", padding), colored(message), strings.Repeat(" ", width-len(message)-padding)) 59 | fmt.Println(colored(strings.Repeat("=", width))) 60 | } else { 61 | fmt.Println(color.CyanString(timestamp), "-", colored(message)) 62 | } 63 | } 64 | 65 | // GetEnv retrieves the value of an environment variable. 66 | // If the variable is not set, it logs an error and returns an empty string. 67 | func GetEnv(key string) string { 68 | value := os.Getenv(key) 69 | if value == "" { 70 | HandleError(fmt.Errorf("environment variable %s not set", key), "Missing environment variable", false) 71 | } 72 | return value 73 | } 74 | --------------------------------------------------------------------------------