├── .gitignore ├── Dockerfile ├── Dockerfile-sqliteweb ├── LICENSE ├── Makefile ├── README.md ├── api.go ├── bitcoin └── addresses.go ├── commands.go ├── config.toml ├── config └── config.go ├── db └── db.go ├── docker-compose.yml ├── docker-entrypoint.sh ├── encoder └── encoder.go ├── export ├── btc_addresses.go ├── bulletin.go ├── export.go ├── newsletters.go ├── release.go ├── rfc.go └── weeks.go ├── feed_commands.go ├── feeds ├── ctrl.go ├── feed.go └── form.go ├── filters ├── filters.go └── mailchimp_filter.go ├── github ├── auth.go └── github.go ├── go.mod ├── go.sum ├── handlers ├── handlers.go ├── releases.go ├── rfc.go └── rss.go ├── hugobot.pdf ├── jobs.go ├── logging └── log.go ├── main.go ├── parse_test.go ├── posts └── posts.go ├── posts_test.go ├── scheduler.go ├── server.go ├── static ├── bolts.go └── data.go ├── types ├── json.go └── stringlist.go └── utils ├── integers.go ├── map.go ├── paths.go ├── print.go ├── shortid.go ├── time.go └── time_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # ---> Go 2 | 3 | # Test binary, build with `go test -c` 4 | *.test 5 | 6 | # Output of the go coverage tool, specifically when used with LiteIDE 7 | *.out 8 | 9 | *.log 10 | 11 | 12 | # Binary 13 | hugobot 14 | 15 | # Sqlite 16 | *.sqlite-* 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.11-alpine as builder 2 | 3 | MAINTAINER Chakib 4 | 5 | # Copy source 6 | COPY . /go/src/hugobot 7 | 8 | # install dependencies and build 9 | RUN apk add --no-cache --upgrade \ 10 | ca-certificates \ 11 | git \ 12 | openssh \ 13 | make \ 14 | alpine-sdk 15 | 16 | RUN cd /go/src/hugobot \ 17 | && make install 18 | 19 | ################################ 20 | #### FINAL IMAGE 21 | ############################### 22 | 23 | 24 | FROM alpine as final 25 | 26 | ENV WEBSITE_PATH=/website 27 | ENV HUGOBOT_DB_PATH=/db 28 | 29 | RUN apk add --no-cache --upgrade \ 30 | ca-certificates \ 31 | bash \ 32 | sqlite \ 33 | jq 34 | 35 | COPY --from=builder /go/bin/hugobot /bin/ 36 | 37 | 38 | RUN mkdir -p ${HUGOBOT_DB_PATH} 39 | RUN mkdir -p ${WEBSITE_PATH} 40 | 41 | 42 | VOLUME ${HUGOBOT_DB_PATH} 43 | 44 | 45 | # Expose API ports 46 | EXPOSE 8734 47 | 48 | # copy entrypoint 49 | COPY "docker-entrypoint.sh" /entry 50 | 51 | ENTRYPOINT ["/entry"] 52 | CMD ["hugobot", "server"] 53 | -------------------------------------------------------------------------------- /Dockerfile-sqliteweb: -------------------------------------------------------------------------------- 1 | FROM coleifer/sqlite 2 | RUN apk add --no-cache --virtual .build-reqs build-base gcc make \ 3 | && pip install --no-cache-dir cython \ 4 | && pip install --no-cache-dir flask peewee sqlite-web \ 5 | && apk del .build-reqs 6 | EXPOSE 8080 7 | VOLUME /db 8 | WORKDIR /db 9 | CMD sqlite_web -H 0.0.0.0 -x $SQLITE_DATABASE -P 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TARGET=hugobot 2 | 3 | GOINSTALL := GO111MODULE=on go install -v 4 | GOBUILD := GO111MODULE=on go build -v 5 | PKG := hugobot 6 | 7 | .PHONY: all build install 8 | 9 | 10 | all: build 11 | 12 | build: 13 | $(GOBUILD) -o $(TARGET) 14 | 15 | install: 16 | $(GOINSTALL) 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **MIRRORED FROM**: https://git.blob42.xyz/blob42/hugobot 2 | 3 | # HUGOBOT 4 | 5 | *hugobot* is a bot that automates the fetching and 6 | aggregation of content for [Hugo][hugo] data-driven 7 | websites. It has the following features: 8 | 9 | 10 | ## Data fetch 11 | 12 | - Use the `feeds` table to register feeds that will be fetched periodically. 13 | - Currently, it can handle these types of feeds: `RSS`, `Github Releases`, `Newsletters` 14 | - To define your own feed types, implement the `JobHandler` interface (see `handlers/handlers.go`). 15 | - Hugobot automatically fetches new posts from the registered feeds. 16 | - The database uses Sqlite for storage. It has `feeds` and `posts` tables. 17 | - The scheduler can handle an unlimited number of tasks and uses leveldb for caching and resuming jobs. 18 | 19 | ## Hugo export 20 | 21 | - Data is automatically exported to the configured Hugo website path. 22 | - It can export data as `markdown` files or `json/toml` data files. 23 | - You can customize all fields in the exported files. 24 | - You can define custom output formats by using the `FormatHandler` interface. 25 | - You can register custom filters and post-processing for exported posts to prevent altering the raw data stored in the database. 26 | - You can force data export using the CLI. 27 | 28 | ## API 29 | 30 | - It uses `gin-gonic` as the web framework. 31 | - *hugobot* also includes a webserver API that can be used with Hugo [Data Driven Mode][data-driven]. 32 | - You can insert and query data from the database. This feature is still a work in progress, but you can easily add the missing code on the API side to automate inserting and querying data from the database. 33 | - For example, it can be used to automate the generation of Bitcoin addresses for new articles on [bitcointechweekly.com][btw-btc]. 34 | 35 | ## Other 36 | 37 | - Some commands are available through the CLI (`github.com/urfave/cli`), you 38 | can add your own custom commands. 39 | 40 | ## Sqliteweb interface 41 | 42 | - See the Docker files for more information. 43 | 44 | ## First time usage 45 | 46 | - The first time you run the program, it will automatically generate the database. You can add your feeds to the Sqlite database using your preferred Sqlite GUI. 47 | 48 | ## Contribution 49 | 50 | - We welcome pull requests. Our current priority is adding tests. 51 | - Check the [TODO](#TODO) section. 52 | 53 | ## TODO: 54 | 55 | - Add tests. 56 | - Handle more feed formats: `tweets`, `mailing-list emails` ... 57 | - TLS support in the API (not a priority, can be done with a reverse proxy). 58 | 59 | 60 | [data-driven]:https://gohugo.io/templates/data-templates/#data-driven-content 61 | [btw-btc]:https://bitcointechweekly.com/btc/3Jv15g4G5LDnBJPDh1e2ja8NPnADzMxhVh 62 | [hugo]:https://gohugo.io 63 | 64 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "strconv" 7 | 8 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 9 | 10 | "git.blob42.xyz/blob42/hugobot/v3/config" 11 | 12 | "git.blob42.xyz/blob42/hugobot/v3/bitcoin" 13 | 14 | gum "git.blob42.xyz/blob42/gum.git" 15 | "github.com/gin-gonic/gin" 16 | ) 17 | 18 | var ( 19 | apiLogFile *os.File 20 | ) 21 | 22 | type API struct { 23 | router *gin.Engine 24 | } 25 | 26 | func (api *API) Run(m gum.UnitManager) { 27 | 28 | feedsRoute := api.router.Group("/feeds") 29 | { 30 | feedCtrl := &feeds.FeedCtrl{} 31 | 32 | feedsRoute.POST("/", feedCtrl.Create) 33 | feedsRoute.DELETE("/:id", feedCtrl.Delete) 34 | feedsRoute.GET("/", feedCtrl.List) // Get all 35 | //feedsRoute.Get("/:id", feedCtrl.GetById) // Get one 36 | } 37 | 38 | btcRoute := api.router.Group("/btc") 39 | { 40 | btcRoute.GET("/address", bitcoin.GetAddressCtrl) 41 | } 42 | 43 | // Run router 44 | go func() { 45 | 46 | err := api.router.Run(":" + strconv.Itoa(config.C.ApiPort)) 47 | if err != nil { 48 | panic(err) 49 | } 50 | }() 51 | 52 | // Wait for stop signal 53 | <-m.ShouldStop() 54 | 55 | // Shutdown 56 | api.Shutdown() 57 | m.Done() 58 | } 59 | 60 | func (api *API) Shutdown() {} 61 | 62 | func NewApi() *API { 63 | apiLogFile, _ = os.Create(".api.log") 64 | gin.DefaultWriter = io.MultiWriter(apiLogFile, os.Stdout) 65 | 66 | api := &API{ 67 | router: gin.Default(), 68 | } 69 | 70 | return api 71 | } 72 | -------------------------------------------------------------------------------- /bitcoin/addresses.go: -------------------------------------------------------------------------------- 1 | package bitcoin 2 | 3 | import ( 4 | "database/sql" 5 | "log" 6 | "net/http" 7 | 8 | "git.blob42.xyz/blob42/hugobot/v3/db" 9 | 10 | "github.com/gin-gonic/gin" 11 | sqlite3 "github.com/mattn/go-sqlite3" 12 | ) 13 | 14 | var DB = db.DB 15 | 16 | const ( 17 | DBBTCAddressesSchema = `CREATE TABLE IF NOT EXISTS btc_addresses ( 18 | addr_id INTEGER PRIMARY KEY, 19 | address TEXT NOT NULL UNIQUE, 20 | address_position INTEGER NOT NULL DEFAULT 0, 21 | linked_article_title TEXT DEFAULT '', 22 | linked_article_id TEXT NOT NULL DEFAULT '', 23 | used INTEGER NOT NULL DEFAULT 0, 24 | synced INTEGER NOT NULL DEFAULT 0 25 | )` 26 | 27 | QueryUnusedAddress = `SELECT * FROM btc_addresses WHERE used = 0 LIMIT 1 ` 28 | 29 | UpdateAddressQuery = `UPDATE btc_addresses 30 | SET linked_article_id = ?, 31 | linked_article_title = ?, 32 | used = ? 33 | WHERE addr_id = ? 34 | ` 35 | ) 36 | 37 | type BTCAddress struct { 38 | ID int64 `db:"addr_id"` 39 | Address string `db:"address"` 40 | AddrPosition int64 `db:"address_position"` 41 | LinkedArticleTitle string `db:"linked_article_title"` 42 | LinkedArticleID string `db:"linked_article_id"` 43 | Used bool `db:"used"` 44 | Synced bool `db:"synced"` 45 | } 46 | 47 | // TODO: Set address to synced 48 | func (a *BTCAddress) SetSynced() error { 49 | a.Synced = true 50 | query := `UPDATE btc_addresses SET synced = :synced WHERE addr_id = :addr_id` 51 | _, err := DB.Handle.NamedExec(query, a) 52 | if err != nil { 53 | return err 54 | } 55 | 56 | return nil 57 | 58 | } 59 | 60 | func GetAddressByPos(pos int) (*BTCAddress, error) { 61 | var btcAddr BTCAddress 62 | err := DB.Handle.Get(&btcAddr, 63 | "SELECT * FROM btc_addresses WHERE address_position = ?", 64 | pos, 65 | ) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | return &btcAddr, nil 71 | } 72 | 73 | func GetAddressByArticleID(artId string) (*BTCAddress, error) { 74 | var btcAddr BTCAddress 75 | err := DB.Handle.Get(&btcAddr, 76 | "SELECT * FROM btc_addresses WHERE linked_article_id = ?", 77 | artId, 78 | ) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return &btcAddr, nil 84 | } 85 | 86 | func GetAllUsedUnsyncedAddresses() ([]*BTCAddress, error) { 87 | var addrs []*BTCAddress 88 | err := DB.Handle.Select(&addrs, 89 | "SELECT * FROM btc_addresses WHERE used = 1 AND synced = 0", 90 | ) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | return addrs, nil 96 | } 97 | 98 | func GetNextUnused() (*BTCAddress, error) { 99 | var btcAddr BTCAddress 100 | err := DB.Handle.Get(&btcAddr, QueryUnusedAddress) 101 | if err != nil { 102 | return nil, err 103 | } 104 | return &btcAddr, nil 105 | } 106 | 107 | func GetAddressForArticle(artId string, artTitle string) (*BTCAddress, error) { 108 | // Check if article already has an assigned address 109 | addr, err := GetAddressByArticleID(artId) 110 | sqliteErr, isSqliteErr := err.(sqlite3.Error) 111 | 112 | if (isSqliteErr && sqliteErr.Code != sqlite3.ErrNotFound) || 113 | (err != nil && !isSqliteErr && err != sql.ErrNoRows) { 114 | 115 | log.Println("err") 116 | return nil, err 117 | } 118 | 119 | if err == nil { 120 | // If different title update it 121 | if artTitle != addr.LinkedArticleTitle { 122 | addr.LinkedArticleTitle = artTitle 123 | // Store newly assigned address 124 | _, err = DB.Handle.Exec(UpdateAddressQuery, 125 | addr.LinkedArticleID, 126 | addr.LinkedArticleTitle, 127 | addr.Used, 128 | addr.ID, 129 | ) 130 | if err != nil { 131 | return nil, err 132 | } 133 | } 134 | 135 | return addr, nil 136 | } 137 | 138 | // Get next unused address 139 | addr, err = GetNextUnused() 140 | if err != nil { 141 | return nil, err 142 | } 143 | 144 | addr.LinkedArticleID = artId 145 | addr.LinkedArticleTitle = artTitle 146 | addr.Used = true 147 | 148 | // Store newly assigned address 149 | _, err = DB.Handle.Exec(UpdateAddressQuery, 150 | addr.LinkedArticleID, 151 | addr.LinkedArticleTitle, 152 | addr.Used, 153 | addr.ID, 154 | ) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | return addr, nil 160 | } 161 | 162 | func GetAddressCtrl(c *gin.Context) { 163 | artId := c.Query("articleId") 164 | artTitle := c.Query("articleTitle") 165 | 166 | addr, err := GetAddressForArticle(artId, artTitle) 167 | 168 | if err != nil { 169 | c.JSON(http.StatusBadRequest, 170 | gin.H{"status": http.StatusBadRequest, 171 | "error": err.Error()}) 172 | c.Abort() 173 | return 174 | } 175 | 176 | c.JSON(http.StatusOK, gin.H{ 177 | "status": http.StatusOK, 178 | "addr": addr.Address, 179 | }) 180 | 181 | } 182 | 183 | func init() { 184 | _, err := DB.Handle.Exec(DBBTCAddressesSchema) 185 | if err != nil { 186 | log.Fatal(err) 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /commands.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/export" 5 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 6 | "git.blob42.xyz/blob42/hugobot/v3/static" 7 | "log" 8 | 9 | cli "gopkg.in/urfave/cli.v1" 10 | ) 11 | 12 | var startServerCmd = cli.Command{ 13 | Name: "server", 14 | Aliases: []string{"s"}, 15 | Usage: "Run server", 16 | Action: startServer, 17 | } 18 | 19 | var exportCmdGrp = cli.Command{ 20 | Name: "export", 21 | Aliases: []string{"e"}, 22 | Usage: "Export to hugo", 23 | Subcommands: []cli.Command{ 24 | exportPostsCmd, 25 | exportWeeksCmd, 26 | exportBTCAddressesCmd, 27 | }, 28 | } 29 | 30 | var exportBTCAddressesCmd = cli.Command{ 31 | Name: "btc", 32 | Usage: "export bitcoin addresses", 33 | Action: exportAddresses, 34 | } 35 | 36 | var exportWeeksCmd = cli.Command{ 37 | Name: "weeks", 38 | Usage: "export weeks", 39 | Action: exportWeeks, 40 | } 41 | 42 | var exportPostsCmd = cli.Command{ 43 | Name: "posts", 44 | Usage: "Export posts to hugo", 45 | Action: exportPosts, 46 | } 47 | 48 | func startServer(c *cli.Context) { 49 | server() 50 | } 51 | 52 | func exportPosts(c *cli.Context) { 53 | exporter := export.NewHugoExporter() 54 | feeds, err := feeds.ListFeeds() 55 | if err != nil { 56 | log.Fatal(err) 57 | } 58 | 59 | for _, f := range feeds { 60 | exporter.Export(*f) 61 | } 62 | 63 | // Export static data 64 | err = static.HugoExportData() 65 | if err != nil { 66 | log.Fatal(err) 67 | } 68 | 69 | } 70 | 71 | func exportWeeks(c *cli.Context) { 72 | err := export.ExportWeeks() 73 | if err != nil { 74 | log.Fatal(err) 75 | } 76 | 77 | } 78 | 79 | func exportAddresses(c *cli.Context) { 80 | err := export.ExportBTCAddresses() 81 | if err != nil { 82 | log.Fatal(err) 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- 1 | api-port = 8734 2 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "log" 5 | "path" 6 | 7 | "github.com/fatih/structs" 8 | ) 9 | 10 | const ( 11 | BTCQRCodesDir = "qrcodes" 12 | ) 13 | 14 | type Config struct { 15 | WebsitePath string 16 | GithubAccessToken string 17 | RelBitcoinAddrContentPath string 18 | ApiPort int 19 | } 20 | 21 | var ( 22 | C *Config 23 | ) 24 | 25 | func HugoData() string { 26 | return path.Join(C.WebsitePath, "data") 27 | } 28 | 29 | func HugoContent() string { 30 | return path.Join(C.WebsitePath, "content") 31 | } 32 | 33 | func RelBitcoinAddrContentPath() string { 34 | return path.Join(C.WebsitePath, C.RelBitcoinAddrContentPath) 35 | } 36 | 37 | func RegisterConf(conf string, val interface{}) error { 38 | log.Printf("Setting %#v to %#v", conf, val) 39 | s := structs.New(C) 40 | 41 | field, ok := s.FieldOk(conf) 42 | 43 | // Conf option not registered in Config struct 44 | if !ok { 45 | return nil 46 | } 47 | 48 | err := field.Set(val) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | return nil 54 | } 55 | 56 | func init() { 57 | C = new(Config) 58 | } 59 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | "time" 10 | 11 | "github.com/jmoiron/sqlx" 12 | _ "github.com/mattn/go-sqlite3" 13 | ) 14 | 15 | const ( 16 | DBName = "hugobot.sqlite" 17 | DBPragma = ` PRAGMA foreign_keys = ON; ` 18 | DBBasePathEnv = "HUGOBOT_DB_PATH" 19 | ) 20 | 21 | var ( 22 | DBOptions = map[string]string{ 23 | "_journal_mode": "WAL", 24 | } 25 | 26 | DB *Database 27 | ) 28 | 29 | type Database struct { 30 | Handle *sqlx.DB 31 | } 32 | 33 | func (d *Database) Open() error { 34 | 35 | dsnOptions := &url.Values{} 36 | for k, v := range DBOptions { 37 | dsnOptions.Set(k, v) 38 | } 39 | 40 | // Get db base path 41 | path, set := os.LookupEnv(DBBasePathEnv) 42 | if !set { 43 | path = "." 44 | } 45 | path = filepath.Join(path, DBName) 46 | //path = fmt.Sprintf("%s/%s", path, DBName) 47 | 48 | dsn := fmt.Sprintf("file:%s?%s", path, dsnOptions.Encode()) 49 | 50 | log.Printf("Opening sqlite db %s\n", dsn) 51 | 52 | var err error 53 | d.Handle, err = sqlx.Open("sqlite3", dsn) 54 | if err != nil { 55 | log.Fatal(err) 56 | } 57 | 58 | // Execute Pragmas 59 | d.Handle.MustExec(DBPragma) 60 | 61 | return nil 62 | } 63 | 64 | type AutoIncr struct { 65 | ID int64 `json:"id"` 66 | Created time.Time `json:"created"` 67 | } 68 | 69 | func init() { 70 | DB = &Database{} 71 | DB.Open() 72 | } 73 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2.2" 2 | 3 | volumes: 4 | js-deps: 5 | build: 6 | sqlite-db: 7 | 8 | services: 9 | bot: 10 | image: hugobot/hugobot 11 | build: . 12 | 13 | volumes: 14 | - path_to_website:/website 15 | - $PWD:/hugobot 16 | - sqlite-db:/db 17 | 18 | environment: 19 | - BUILD_DIR=/build 20 | 21 | restart: on-failure 22 | 23 | ports: 24 | - "8734:8734" 25 | 26 | working_dir: /hugobot 27 | 28 | 29 | sqlite-web: 30 | image: hugobot/sqlite-web 31 | build: 32 | context: . 33 | dockerfile: ./Dockerfile-sqliteweb 34 | ports: 35 | - "8080" 36 | volumes: 37 | - sqlite-db:/db 38 | 39 | environment: 40 | - SQLITE_DATABASE=hugobot.sqlite 41 | - SQLITE_WEB_PASSWORD=hugobot 42 | 43 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [[ -z "$(ls -A "$HUGOBOT_DB_PATH")" ]];then 6 | echo "WARNING !! $HUGOBOT_DB_PATH is empty, creating new database !" 7 | fi 8 | 9 | if [[ -z "$(ls -A "$WEBSITE_PATH")" ]];then 10 | echo "you need to mount the website path !" 11 | exit 1 12 | fi 13 | 14 | 15 | exec "$@" 16 | -------------------------------------------------------------------------------- /encoder/encoder.go: -------------------------------------------------------------------------------- 1 | package encoder 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/BurntSushi/toml" 9 | ) 10 | 11 | const ( 12 | JSON = iota 13 | TOML 14 | ) 15 | 16 | type Encoder interface { 17 | Encode(v interface{}) error 18 | } 19 | 20 | type ExportEncoder struct { 21 | encoder Encoder 22 | w io.Writer 23 | eType int 24 | } 25 | 26 | func (ee *ExportEncoder) Encode(v interface{}) error { 27 | var err error 28 | 29 | if ee.eType == TOML { 30 | fmt.Fprintf(ee.w, "+++\n") 31 | } 32 | 33 | err = ee.encoder.Encode(v) 34 | 35 | if ee.eType == TOML { 36 | fmt.Fprintf(ee.w, "+++\n") 37 | } 38 | 39 | return err 40 | } 41 | 42 | func NewExportEncoder(w io.Writer, encType int) *ExportEncoder { 43 | 44 | var enc Encoder 45 | 46 | switch encType { 47 | case JSON: 48 | enc = json.NewEncoder(w) 49 | case TOML: 50 | enc = toml.NewEncoder(w) 51 | } 52 | 53 | return &ExportEncoder{ 54 | encoder: enc, 55 | w: w, 56 | eType: encType, 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /export/btc_addresses.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/bitcoin" 5 | "git.blob42.xyz/blob42/hugobot/v3/config" 6 | "git.blob42.xyz/blob42/hugobot/v3/encoder" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | 11 | qrcode "github.com/skip2/go-qrcode" 12 | ) 13 | 14 | func ExportBTCAddresses() error { 15 | unusedAddrs, err := bitcoin.GetAllUsedUnsyncedAddresses() 16 | if err != nil { 17 | return err 18 | } 19 | 20 | for _, a := range unusedAddrs { 21 | //first export the qr codes 22 | log.Println("exporting ", a) 23 | 24 | qrFileName := a.Address + ".png" 25 | 26 | qrCodePath := filepath.Join(config.RelBitcoinAddrContentPath(), 27 | config.BTCQRCodesDir, qrFileName) 28 | 29 | err := qrcode.WriteFile(a.Address, qrcode.Medium, 580, qrCodePath) 30 | if err != nil { 31 | return err 32 | } 33 | 34 | // store the address pages 35 | 36 | filename := a.Address + ".md" 37 | filePath := filepath.Join(config.RelBitcoinAddrContentPath(), filename) 38 | 39 | data := map[string]interface{}{ 40 | "linked_article_id": a.LinkedArticleID, 41 | //"resources": []map[string]interface{}{ 42 | //map[string]interface{}{ 43 | //"src": filepath.Join(config.BTCQRCodesDir, a.Address+".png"), 44 | //}, 45 | //}, 46 | } 47 | 48 | addressPage, err := os.Create(filePath) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | tomlExporter := encoder.NewExportEncoder(addressPage, encoder.TOML) 54 | tomlExporter.Encode(data) 55 | 56 | // Set synced 57 | err = a.SetSynced() 58 | if err != nil { 59 | return err 60 | } 61 | 62 | } 63 | 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /export/bulletin.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/posts" 6 | "strings" 7 | ) 8 | 9 | func BulletinExport(exp Map, feed feeds.Feed, post posts.Post) error { 10 | 11 | bulletinInfo := strings.Split(feed.Section, "/") 12 | 13 | if bulletinInfo[0] == "bulletin" { 14 | exp["bulletin_type"] = bulletinInfo[1] 15 | } 16 | return nil 17 | } 18 | -------------------------------------------------------------------------------- /export/export.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | "git.blob42.xyz/blob42/hugobot/v3/config" 12 | "git.blob42.xyz/blob42/hugobot/v3/encoder" 13 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 14 | "git.blob42.xyz/blob42/hugobot/v3/filters" 15 | "git.blob42.xyz/blob42/hugobot/v3/posts" 16 | "git.blob42.xyz/blob42/hugobot/v3/types" 17 | "git.blob42.xyz/blob42/hugobot/v3/utils" 18 | ) 19 | 20 | var PostMappers []PostMapper 21 | var FeedMappers []FeedMapper 22 | 23 | type Map map[string]interface{} 24 | 25 | type PostMapper func(Map, feeds.Feed, posts.Post) error 26 | type FeedMapper func(Map, feeds.Feed) error 27 | 28 | // Exported version of a post 29 | type PostExport struct { 30 | ID int64 `json:"id"` 31 | Title string `json:"title"` 32 | Link string `json:"link"` 33 | Published time.Time `json:"published"` 34 | Content string `json:"content"` 35 | } 36 | 37 | type PostMap map[int64]Map 38 | 39 | type FeedExport struct { 40 | Name string `json:"name"` 41 | Section string `json:"section"` 42 | Categories types.StringList `json:"categories"` 43 | Posts PostMap `json:"posts"` 44 | } 45 | 46 | type HugoExporter struct{} 47 | 48 | func (he HugoExporter) Handle(feed feeds.Feed) error { 49 | return he.export(feed) 50 | } 51 | 52 | func (he HugoExporter) export(feed feeds.Feed) error { 53 | log.Printf("Exporting %s to %s", feed.Name, config.HugoData()) 54 | 55 | posts, err := posts.GetPostsByFeedId(feed.FeedID) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | if len(posts) == 0 { 61 | log.Printf("nothing to export") 62 | return nil 63 | } 64 | 65 | // Run filters on posts 66 | for _, p := range posts { 67 | filters.RunPostFilterHooks(feed, p) 68 | } 69 | 70 | // Dir and filename 71 | dirPath := filepath.Join(config.HugoData(), feed.Section) 72 | cleanFeedName := strings.Replace(feed.Name, "/", "-", -1) 73 | filePath := filepath.Join(dirPath, cleanFeedName+".json") 74 | 75 | err = utils.Mkdir(dirPath) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | feedExp := Map{ 81 | "name": feed.Name, 82 | "section": feed.Section, 83 | "categories": feed.Categories, 84 | } 85 | 86 | runFeedMappers(feedExp, feed) 87 | 88 | postsMap := make(PostMap) 89 | for _, p := range posts { 90 | exp := Map{ 91 | "id": p.PostID, 92 | "title": p.Title, 93 | "link": p.Link, 94 | "published": p.Published, 95 | "updated": p.Updated, 96 | //"content": p.Content, 97 | } 98 | runPostMappers(exp, feed, *p) 99 | 100 | postsMap[p.PostID] = exp 101 | } 102 | feedExp["posts"] = postsMap 103 | 104 | outputFile, err := os.Create(filePath) 105 | defer outputFile.Close() 106 | if err != nil { 107 | return err 108 | } 109 | 110 | exportEncoder := encoder.NewExportEncoder(outputFile, encoder.JSON) 111 | exportEncoder.Encode(feedExp) 112 | //jsonEnc.Encode(feedExp) 113 | 114 | // Handle feeds which export posts individually as hugo posts 115 | // Like bulletin 116 | 117 | if feed.ExportPosts { 118 | for _, p := range posts { 119 | 120 | exp := map[string]interface{}{ 121 | "id": p.PostID, 122 | "title": p.Title, 123 | "name": feed.Name, 124 | "author": p.Author, 125 | "description": p.PostDescription, 126 | "externalLink": feed.UseExternalLink, 127 | "display_name": feed.DisplayName, 128 | "publishdate": p.Published, 129 | "date": p.Updated, 130 | "issuedate": utils.NextThursday(p.Updated), 131 | "use_data": true, 132 | "slug": p.ShortID, 133 | "link": p.Link, 134 | // Content is written in the post 135 | "content": p.Content, 136 | "categories": feed.Categories, 137 | "tags": strings.Split(p.Tags, ","), 138 | } 139 | 140 | if feed.Publications != "" { 141 | exp["publications"] = strings.Split(feed.Publications, ",") 142 | } 143 | 144 | runPostMappers(exp, feed, *p) 145 | 146 | dirPath := filepath.Join(config.HugoContent(), feed.Section) 147 | cleanFeedName := strings.Replace(feed.Name, "/", "-", -1) 148 | fileName := fmt.Sprintf("%s-%s.md", cleanFeedName, p.ShortID) 149 | filePath := filepath.Join(dirPath, fileName) 150 | 151 | outputFile, err := os.Create(filePath) 152 | 153 | defer outputFile.Close() 154 | 155 | if err != nil { 156 | return err 157 | } 158 | 159 | exportEncoder := encoder.NewExportEncoder(outputFile, encoder.TOML) 160 | exportEncoder.Encode(exp) 161 | } 162 | } 163 | return nil 164 | } 165 | 166 | // Runs in goroutine 167 | func (he HugoExporter) Export(feed feeds.Feed) { 168 | err := he.export(feed) 169 | if err != nil { 170 | log.Fatal(err) 171 | } 172 | } 173 | 174 | func NewHugoExporter() HugoExporter { 175 | // Make sure path exists 176 | err := utils.Mkdir(config.HugoData()) 177 | if err != nil { 178 | log.Fatal(err) 179 | } 180 | return HugoExporter{} 181 | } 182 | 183 | func runPostMappers(e Map, f feeds.Feed, p posts.Post) { 184 | for _, fn := range PostMappers { 185 | err := fn(e, f, p) 186 | if err != nil { 187 | log.Print(err) 188 | } 189 | } 190 | } 191 | 192 | func runFeedMappers(e Map, f feeds.Feed) { 193 | for _, fn := range FeedMappers { 194 | err := fn(e, f) 195 | if err != nil { 196 | log.Print(err) 197 | } 198 | } 199 | } 200 | 201 | func RegisterPostMapper(mapper PostMapper) { 202 | PostMappers = append(PostMappers, mapper) 203 | } 204 | 205 | func RegisterFeedMapper(mapper FeedMapper) { 206 | FeedMappers = append(FeedMappers, mapper) 207 | } 208 | 209 | func init() { 210 | RegisterPostMapper(BulletinExport) 211 | RegisterPostMapper(NewsletterPostLayout) 212 | RegisterPostMapper(RFCExport) 213 | RegisterPostMapper(ReleaseExport) 214 | } 215 | -------------------------------------------------------------------------------- /export/newsletters.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/posts" 6 | "path" 7 | 8 | "github.com/gobuffalo/flect" 9 | ) 10 | 11 | func NewsletterPostLayout(exp Map, feed feeds.Feed, post posts.Post) error { 12 | section := path.Base(flect.Singularize(feed.Section)) 13 | if feed.Section == "bulletin/newsletters" { 14 | exp["layout"] = section 15 | } 16 | 17 | return nil 18 | } 19 | -------------------------------------------------------------------------------- /export/release.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/posts" 6 | ) 7 | 8 | // 9 | func ReleaseExport(exp Map, feed feeds.Feed, post posts.Post) error { 10 | if feed.Section == "bulletin/releases" { 11 | exp["data"] = post.JsonData 12 | } 13 | return nil 14 | } 15 | -------------------------------------------------------------------------------- /export/rfc.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/posts" 6 | ) 7 | 8 | // TODO: This happend in the main export file 9 | func RFCExport(exp Map, feed feeds.Feed, post posts.Post) error { 10 | if feed.Section == "bulletin/rfc" { 11 | exp["data"] = post.JsonData 12 | 13 | } 14 | return nil 15 | } 16 | -------------------------------------------------------------------------------- /export/weeks.go: -------------------------------------------------------------------------------- 1 | // Export all weeks to the weeks content directory 2 | package export 3 | 4 | import ( 5 | "git.blob42.xyz/blob42/hugobot/v3/config" 6 | "git.blob42.xyz/blob42/hugobot/v3/encoder" 7 | "git.blob42.xyz/blob42/hugobot/v3/utils" 8 | "os" 9 | "path/filepath" 10 | "time" 11 | ) 12 | 13 | const ( 14 | FirstWeek = "2017-12-07" 15 | ) 16 | 17 | var ( 18 | WeeksContentDir = "weeks" 19 | ) 20 | 21 | type WeekData struct { 22 | Title string 23 | Date time.Time 24 | } 25 | 26 | func ExportWeeks() error { 27 | firstWeek, err := time.Parse("2006-01-02", FirstWeek) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | WeeksTilNow := utils.GetAllThursdays(firstWeek, time.Now()) 33 | for _, week := range WeeksTilNow { 34 | weekName := week.Format("2006-01-02") 35 | fileName := weekName + ".md" 36 | 37 | weekFile, err := os.Create(filepath.Join(config.HugoContent(), 38 | WeeksContentDir, 39 | fileName)) 40 | 41 | if err != nil { 42 | return err 43 | } 44 | 45 | weekData := WeekData{ 46 | Title: weekName, 47 | Date: week, 48 | } 49 | 50 | tomlExporter := encoder.NewExportEncoder(weekFile, encoder.TOML) 51 | 52 | tomlExporter.Encode(weekData) 53 | } 54 | 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /feed_commands.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/handlers" 6 | "git.blob42.xyz/blob42/hugobot/v3/posts" 7 | "fmt" 8 | "log" 9 | "time" 10 | 11 | cli "gopkg.in/urfave/cli.v1" 12 | ) 13 | 14 | var fetchCmd = cli.Command{ 15 | Name: "fetch", 16 | Aliases: []string{"f"}, 17 | Usage: "Fetch data from feed", 18 | Flags: []cli.Flag{ 19 | cli.StringFlag{ 20 | Name: "since", 21 | Usage: "Fetch data since `TIME`, defaults to last refresh time", 22 | }, 23 | }, 24 | Action: fetchFeeds, 25 | } 26 | 27 | var feedsCmdGroup = cli.Command{ 28 | Name: "feeds", 29 | Usage: "Feeds related commands. default: list feeds", 30 | Flags: []cli.Flag{ 31 | cli.IntFlag{ 32 | Name: "id,i", 33 | Value: 0, 34 | Usage: "Feeds `id`", 35 | }, 36 | }, 37 | Subcommands: []cli.Command{ 38 | fetchCmd, 39 | }, 40 | Action: listFeeds, 41 | } 42 | 43 | func fetchFeeds(c *cli.Context) { 44 | var result []*posts.Post 45 | 46 | fList, err := getFeeds(c.Parent()) 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | 51 | for _, f := range fList { 52 | var handler handlers.FormatHandler 53 | handler = handlers.GetFormatHandler(*f) 54 | 55 | if c.IsSet("since") { 56 | // Parse time 57 | t, err := time.Parse(time.UnixDate, c.String("since")) 58 | if err != nil { 59 | log.Fatal(err) 60 | } 61 | result, err = handler.FetchSince(f.Url, t) 62 | 63 | } else { 64 | result, err = handler.FetchSince(f.Url, f.LastRefresh) 65 | } 66 | 67 | if err != nil { 68 | log.Fatal(err) 69 | } 70 | 71 | for _, post := range result { 72 | log.Printf("%s (updated: %s)", post.Title, post.Updated) 73 | } 74 | log.Println("Total: ", len(result)) 75 | 76 | } 77 | 78 | } 79 | 80 | func listFeeds(c *cli.Context) { 81 | fList, err := getFeeds(c) 82 | if err != nil { 83 | log.Fatal(err) 84 | } 85 | 86 | for _, f := range fList { 87 | fmt.Println(f) 88 | } 89 | } 90 | 91 | func getFeeds(c *cli.Context) ([]*feeds.Feed, error) { 92 | var fList []*feeds.Feed 93 | var err error 94 | 95 | if c.IsSet("id") { 96 | feed, err := feeds.GetById(c.Int64("id")) 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | fList = append(fList, feed) 102 | } else { 103 | fList, err = feeds.ListFeeds() 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | } 109 | 110 | return fList, nil 111 | 112 | } 113 | -------------------------------------------------------------------------------- /feeds/ctrl.go: -------------------------------------------------------------------------------- 1 | package feeds 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/types" 5 | "log" 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/gin-gonic/gin" 10 | sqlite3 "github.com/mattn/go-sqlite3" 11 | ) 12 | 13 | const ( 14 | MsgOK = "OK" 15 | ) 16 | 17 | var ( 18 | ErrNotInt = "expected int" 19 | ) 20 | 21 | type FeedCtrl struct{} 22 | 23 | func (ctrl FeedCtrl) Create(c *gin.Context) { 24 | 25 | var feedForm FeedForm 26 | feedModel := new(Feed) 27 | 28 | if err := c.ShouldBindJSON(&feedForm); err != nil { 29 | c.JSON(http.StatusNotAcceptable, gin.H{ 30 | "status": http.StatusNotAcceptable, 31 | "message": "invalid form", 32 | "form": feedForm}) 33 | c.Abort() 34 | return 35 | } 36 | 37 | feedModel.Name = feedForm.Name 38 | feedModel.Url = feedForm.Url 39 | feedModel.Format = feedForm.Format 40 | feedModel.Section = feedForm.Section 41 | feedModel.Categories = types.StringList(feedForm.Categories) 42 | 43 | err := feedModel.Write() 44 | 45 | if err != nil { 46 | log.Println(err) 47 | c.JSON(http.StatusNotAcceptable, 48 | gin.H{"status": http.StatusNotAcceptable, "error": err.Error()}) 49 | c.Abort() 50 | return 51 | } 52 | 53 | c.JSON(http.StatusOK, gin.H{"status": http.StatusOK, "message": MsgOK}) 54 | } 55 | 56 | func (ctrl FeedCtrl) List(c *gin.Context) { 57 | 58 | feeds, err := ListFeeds() 59 | if err != nil { 60 | c.JSON(http.StatusNotAcceptable, gin.H{ 61 | "error": err.Error(), 62 | "status": http.StatusNotAcceptable, 63 | }) 64 | c.Abort() 65 | return 66 | } 67 | 68 | c.JSON(http.StatusOK, gin.H{"status": http.StatusOK, "result": feeds}) 69 | 70 | } 71 | 72 | func (ctrl FeedCtrl) Delete(c *gin.Context) { 73 | 74 | id, err := strconv.Atoi(c.Param("id")) 75 | if err != nil { 76 | c.JSON(http.StatusNotAcceptable, gin.H{ 77 | "error": ErrNotInt, 78 | "status": http.StatusNotAcceptable, 79 | }) 80 | c.Abort() 81 | return 82 | } 83 | err = DeleteById(id) 84 | 85 | sqlErr, isSqlErr := err.(sqlite3.Error) 86 | if err != nil { 87 | 88 | if isSqlErr { 89 | 90 | c.JSON(http.StatusInternalServerError, 91 | gin.H{ 92 | "error": sqlErr.Error(), 93 | "status": http.StatusInternalServerError, 94 | }) 95 | 96 | } else { 97 | 98 | var status int 99 | 100 | switch err { 101 | case ErrDoesNotExist: 102 | status = http.StatusNotFound 103 | default: 104 | status = http.StatusInternalServerError 105 | } 106 | 107 | c.JSON(status, 108 | gin.H{"error": err.Error(), "status": status}) 109 | 110 | } 111 | 112 | c.Abort() 113 | return 114 | } 115 | 116 | c.JSON(http.StatusOK, gin.H{"status": http.StatusOK, "message": MsgOK}) 117 | } 118 | -------------------------------------------------------------------------------- /feeds/feed.go: -------------------------------------------------------------------------------- 1 | package feeds 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/db" 5 | "git.blob42.xyz/blob42/hugobot/v3/types" 6 | "errors" 7 | "log" 8 | "time" 9 | 10 | sqlite3 "github.com/mattn/go-sqlite3" 11 | ) 12 | 13 | //sqlite> SELECT feeds.name, url, feed_formats.name AS format_name from feeds JOIN feed_formats ON feeds.format = feed_formats.id; 14 | // 15 | var DB = db.DB 16 | 17 | const ( 18 | DBFeedSchema = `CREATE TABLE IF NOT EXISTS feeds ( 19 | feed_id INTEGER PRIMARY KEY, 20 | name TEXT NOT NULL UNIQUE, 21 | display_name TEXT DEFAULT '', 22 | publications TEXT DEFAULT '', 23 | section TEXT DEFAULT '', 24 | categories TEXT DEFAULT '', 25 | description TEXT DEFAULT '', 26 | url TEXT NOT NULL, 27 | export_posts INTEGER DEFAULT 0, 28 | last_refresh timestamp DEFAULT -1, 29 | created timestamp DEFAULT (strftime('%s')), 30 | interval INTEGER DEFAULT 60, 31 | format INTEGER NOT NULL DEFAULT 0, 32 | serial_run INTEGER DEFAULT 0, 33 | use_external_link INTEGER DEFAULT 0, 34 | FOREIGN KEY (format) REFERENCES feed_formats(id) 35 | 36 | 37 | )` 38 | 39 | DBFeedFormatsSchema = `CREATE TABLE IF NOT EXISTS feed_formats ( 40 | id INTEGER PRIMARY KEY, 41 | format_name TEXT NOT NULL UNIQUE 42 | )` 43 | ) 44 | 45 | const ( 46 | QDeleteFeedById = `DELETE FROM feeds WHERE feed_id = ?` 47 | QGetFeed = `SELECT * FROM feeds WHERE feed_id = ?` 48 | QGetFeedByName = `SELECT * FROM feeds WHERE name = ?` 49 | QGetFeedByURL = `SELECT * FROM feeds WHERE url = ?` 50 | QListFeeds = `SELECT 51 | feeds.feed_id, 52 | feeds.name, 53 | feeds.display_name, 54 | feeds.publications, 55 | feeds.section, 56 | feeds.categories, 57 | feeds.description, 58 | feeds.url, 59 | feeds.last_refresh, 60 | feeds.created, 61 | feeds.format, 62 | feeds.serial_run, 63 | feeds.use_external_link, 64 | feeds.interval, 65 | feeds.export_posts, 66 | feed_formats.format_name 67 | FROM feeds 68 | JOIN feed_formats ON feeds.format = feed_formats.id` 69 | ) 70 | 71 | var ( 72 | ErrDoesNotExist = errors.New("does not exist") 73 | ErrAlreadyExists = errors.New("already exists") 74 | ) 75 | 76 | type FeedFormat int 77 | 78 | // Feed Formats 79 | const ( 80 | FormatRSS FeedFormat = iota 81 | FormatHTML 82 | FormatJSON 83 | FormatTweet 84 | FormatRFC 85 | FormatGHRelease 86 | ) 87 | 88 | var FeedFormats = map[FeedFormat]string{ 89 | FormatRSS: "RSS", 90 | FormatHTML: "HTML", 91 | FormatJSON: "JSON", 92 | FormatTweet: "TWEET", 93 | FormatRFC: "RFC", 94 | FormatGHRelease: "GithubRelease", 95 | } 96 | 97 | type Feed struct { 98 | FeedID int64 `json:"id" db:"feed_id"` 99 | Name string `json:"name" db:"name"` 100 | Section string `json:"section,omitempty"` 101 | Categories types.StringList `json:"categories,omitempty"` 102 | Description string `json:"description"` 103 | Url string `json:"url"` 104 | Format FeedFormat `json:"-"` 105 | FormatString string `json:"format" db:"format_name"` 106 | LastRefresh time.Time `db:"last_refresh" json:"last_refresh"` // timestamp time.Unix() 107 | Created time.Time `json:"created"` 108 | DisplayName string `db:"display_name"` 109 | Publications string `json:"-"` 110 | 111 | // This feed's posts should also be exported individually 112 | ExportPosts bool `json:"export_posts" db:"export_posts"` 113 | 114 | // Time in seconds between each polling job on the news feed 115 | Interval float64 `json:"refresh_interval"` 116 | 117 | Serial bool `json:"serial" db:"serial_run"` // Jobs for this feed should run in series 118 | 119 | // Items which only contain summaries and redirect to external content 120 | // like publications and newsletters 121 | UseExternalLink bool `json:"use_external_link" db:"use_external_link"` 122 | } 123 | 124 | func (f *Feed) Write() error { 125 | 126 | query := `INSERT INTO feeds 127 | (name, section, categories, url, format) 128 | VALUES(:name, :section, :categories, :url, :format)` 129 | 130 | _, err := DB.Handle.NamedExec(query, f) 131 | sqlErr, isSqlErr := err.(sqlite3.Error) 132 | if isSqlErr && sqlErr.Code == sqlite3.ErrConstraint { 133 | return ErrAlreadyExists 134 | } 135 | 136 | if err != nil { 137 | return err 138 | } 139 | 140 | return nil 141 | } 142 | 143 | func (f *Feed) UpdateRefreshTime(time time.Time) error { 144 | f.LastRefresh = time 145 | 146 | query := `UPDATE feeds SET last_refresh = ? WHERE feed_id = ?` 147 | _, err := DB.Handle.Exec(query, f.LastRefresh, f.FeedID) 148 | if err != nil { 149 | return err 150 | } 151 | 152 | return nil 153 | } 154 | 155 | func GetById(id int64) (*Feed, error) { 156 | 157 | var feed Feed 158 | err := DB.Handle.Get(&feed, QGetFeed, id) 159 | if err != nil { 160 | return nil, err 161 | } 162 | 163 | feed.FormatString = FeedFormats[feed.Format] 164 | 165 | return &feed, nil 166 | } 167 | 168 | func GetByName(name string) (*Feed, error) { 169 | 170 | var feed Feed 171 | err := DB.Handle.Get(&feed, QGetFeedByName, name) 172 | if err != nil { 173 | return nil, err 174 | } 175 | 176 | feed.FormatString = FeedFormats[feed.Format] 177 | 178 | return &feed, nil 179 | } 180 | 181 | func GetByURL(url string) (*Feed, error) { 182 | 183 | var feed Feed 184 | err := DB.Handle.Get(&feed, QGetFeedByURL, url) 185 | if err != nil { 186 | return nil, err 187 | } 188 | 189 | feed.FormatString = FeedFormats[feed.Format] 190 | 191 | return &feed, nil 192 | 193 | } 194 | 195 | func ListFeeds() ([]*Feed, error) { 196 | var feeds []*Feed 197 | err := DB.Handle.Select(&feeds, QListFeeds) 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | return feeds, nil 203 | } 204 | 205 | func DeleteById(id int) error { 206 | 207 | // If id does not exists return warning 208 | var feedToDelete Feed 209 | err := DB.Handle.Get(&feedToDelete, QGetFeed, id) 210 | if err != nil { 211 | return ErrDoesNotExist 212 | } 213 | 214 | _, err = DB.Handle.Exec(QDeleteFeedById, id) 215 | if err != nil { 216 | return err 217 | } 218 | 219 | return nil 220 | } 221 | 222 | // Returns true if the feed should be refreshed 223 | func (feed *Feed) ShouldRefresh() (float64, bool) { 224 | lastRefresh := feed.LastRefresh 225 | delta := time.Since(lastRefresh).Seconds() // Delta since last refresh 226 | //log.Printf("%s delta %f >= interval %f ?", feed.Name, delta, feed.Interval) 227 | // 228 | // 229 | //log.Printf("refresh %s in %.0f seconds", feed.Name, feed.Interval-delta) 230 | return delta, delta >= feed.Interval 231 | } 232 | 233 | func init() { 234 | _, err := DB.Handle.Exec(DBFeedSchema) 235 | if err != nil { 236 | log.Fatal(err) 237 | } 238 | 239 | _, err = DB.Handle.Exec(DBFeedFormatsSchema) 240 | if err != nil { 241 | log.Fatal(err) 242 | } 243 | 244 | // Populate feed formats 245 | query := `INSERT INTO feed_formats (id, format_name) VALUES (?, ?)` 246 | for k, v := range FeedFormats { 247 | _, err := DB.Handle.Exec(query, k, v) 248 | if err != nil { 249 | 250 | sqlErr, ok := err.(sqlite3.Error) 251 | if ok && sqlErr.ExtendedCode == sqlite3.ErrConstraintUnique { 252 | log.Panic(err) 253 | } 254 | 255 | if !ok { 256 | log.Panic(err) 257 | } 258 | } 259 | 260 | } 261 | 262 | } 263 | -------------------------------------------------------------------------------- /feeds/form.go: -------------------------------------------------------------------------------- 1 | package feeds 2 | 3 | type FeedForm struct { 4 | Name string `form:"name" binding:"required"` 5 | Url string `form:"url" binding:"required"` 6 | Format FeedFormat `form:"format"` 7 | Categories []string `form:"categories"` 8 | Section string `form:"section"` 9 | } 10 | -------------------------------------------------------------------------------- /filters/filters.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/posts" 6 | "log" 7 | ) 8 | 9 | type FilterHook func(feed feeds.Feed, post *posts.Post) error 10 | 11 | var ( 12 | PostFilters []FilterHook 13 | ) 14 | 15 | func RegisterPostFilterHook(hook FilterHook) { 16 | PostFilters = append(PostFilters, hook) 17 | } 18 | 19 | func RunPostFilterHooks(feed feeds.Feed, post *posts.Post) { 20 | for _, h := range PostFilters { 21 | err := h(feed, post) 22 | if err != nil { 23 | log.Fatal(err) 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /filters/mailchimp_filter.go: -------------------------------------------------------------------------------- 1 | package filters 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/posts" 6 | "strings" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | ) 10 | 11 | const ( 12 | PreviewTextSel = ".mcnPreviewText" 13 | ) 14 | 15 | var ( 16 | RemoveSelectors = []string{"style", ".footerContainer", "#awesomewrap", "#templatePreheader", "img", "head"} 17 | ) 18 | 19 | func mailChimpFilter(feed feeds.Feed, post *posts.Post) error { 20 | 21 | // Nothing to do for empty content 22 | if post.PostDescription == post.Content && 23 | post.Content == "" { 24 | return nil 25 | } 26 | 27 | // Same content in both 28 | if post.PostDescription == post.Content { 29 | post.PostDescription = "" 30 | 31 | } 32 | 33 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(post.Content)) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | sel := doc.Find(strings.Join(RemoveSelectors, ",")) 39 | sel.Remove() 40 | 41 | post.Content, err = doc.Html() 42 | 43 | return err 44 | } 45 | 46 | func extractPreviewText(feed feeds.Feed, post *posts.Post) error { 47 | // Ignore filled description 48 | if post.PostDescription != "" { 49 | return nil 50 | } 51 | 52 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(post.Content)) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | sel := doc.Find(PreviewTextSel) 58 | post.PostDescription = sel.Text() 59 | return nil 60 | } 61 | 62 | func init() { 63 | RegisterPostFilterHook(mailChimpFilter) 64 | RegisterPostFilterHook(extractPreviewText) 65 | } 66 | -------------------------------------------------------------------------------- /github/auth.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/config" 5 | "context" 6 | 7 | "github.com/google/go-github/github" 8 | "golang.org/x/oauth2" 9 | ) 10 | 11 | func Auth(ctx context.Context) *github.Client { 12 | 13 | ts := oauth2.StaticTokenSource( 14 | &oauth2.Token{AccessToken: config.C.GithubAccessToken}, 15 | ) 16 | 17 | tc := oauth2.NewClient(ctx, ts) 18 | client := github.NewClient(tc) 19 | 20 | return client 21 | } 22 | -------------------------------------------------------------------------------- /github/github.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "log" 5 | "net/url" 6 | "strings" 7 | "time" 8 | 9 | "github.com/google/go-github/github" 10 | ) 11 | 12 | func ParseOwnerRepo(Url string) (owner, repo string) { 13 | url, err := url.Parse(Url) 14 | 15 | if err != nil { 16 | panic(err) 17 | } 18 | 19 | parts := strings.Split(strings.TrimPrefix(url.Path, "/"), "/") 20 | owner = parts[0] 21 | repo = parts[1] 22 | 23 | return owner, repo 24 | } 25 | 26 | func RespMiddleware(resp *github.Response) { 27 | if resp == nil { 28 | return 29 | } 30 | 31 | log.Printf("Rate remaining: %d/%d (reset: %s)", resp.Rate.Remaining, resp.Rate.Limit, resp.Rate.Reset) 32 | err := github.CheckResponse(resp.Response) 33 | if _, ok := err.(*github.RateLimitError); ok { 34 | log.Printf("HIT RATE LIMIT !!!") 35 | } 36 | } 37 | 38 | type Release struct { 39 | Version string `json:"version"` 40 | ID int64 `json:"ID"` 41 | Date time.Time `json:"date"` 42 | Description string `json:"description"` 43 | URL string `json:"url"` 44 | ShortURL string `json:"short_url"` 45 | HtmlURL string `json:"html_url"` 46 | Name string `json:"name"` 47 | TagName string `json:"tag_name"` 48 | } 49 | 50 | type PR struct { 51 | Title string `json:"title"` 52 | URL string `json:"url"` 53 | HtmlURL string `json:"html_url"` 54 | Number int `json:"number"` 55 | Date time.Time `json:"date"` 56 | IssueURL string `json:"issue_url"` 57 | Body string `json:"body"` 58 | } 59 | 60 | type Issue struct { 61 | Title string `json:"title"` 62 | URL string `json:"url"` 63 | ShortURL string `json:"short_url"` 64 | Number int `json:"number"` 65 | State string `json:"state"` 66 | Updated time.Time `json:"updated"` 67 | Created time.Time `json:"created"` 68 | Comments int `json:"comments"` 69 | HtmlURL string `json:"html_url"` 70 | Open bool `json:"opened_issue"` 71 | IsPR bool `json:"is_pr"` 72 | Merged bool `json:"merged"` // only for PRs 73 | MergedAt time.Time `json:"merged_at"` 74 | IsUpdate bool `json:"is_update"` 75 | } 76 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module git.blob42.xyz/blob42/hugobot/v3 2 | 3 | go 1.20 4 | 5 | require ( 6 | git.blob42.xyz/blob42/gum.git v0.0.0-20190304130815-31be968b7b17 7 | github.com/BurntSushi/toml v1.2.1 8 | github.com/PuerkitoBio/goquery v1.8.1 9 | github.com/beeker1121/goque v2.1.0+incompatible 10 | github.com/fatih/structs v1.1.0 11 | github.com/gin-gonic/gin v1.9.0 12 | github.com/gobuffalo/flect v1.0.0 13 | github.com/gofrs/uuid v4.4.0+incompatible 14 | github.com/google/go-github v17.0.0+incompatible 15 | github.com/jmoiron/sqlx v1.3.5 16 | github.com/mattn/go-sqlite3 v1.14.16 17 | github.com/mmcdole/gofeed v1.2.0 18 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e 19 | github.com/syndtr/goleveldb v1.0.0 20 | github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 21 | github.com/urfave/cli v1.22.12 22 | golang.org/x/oauth2 v0.5.0 23 | gopkg.in/urfave/cli.v1 v1.20.0 24 | ) 25 | 26 | require ( 27 | github.com/andybalholm/cascadia v1.3.1 // indirect 28 | github.com/bytedance/sonic v1.8.0 // indirect 29 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect 30 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 31 | github.com/gin-contrib/sse v0.1.0 // indirect 32 | github.com/go-playground/locales v0.14.1 // indirect 33 | github.com/go-playground/universal-translator v0.18.1 // indirect 34 | github.com/go-playground/validator/v10 v10.11.2 // indirect 35 | github.com/goccy/go-json v0.10.0 // indirect 36 | github.com/golang/protobuf v1.5.2 // indirect 37 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect 38 | github.com/google/go-querystring v1.1.0 // indirect 39 | github.com/json-iterator/go v1.1.12 // indirect 40 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 41 | github.com/leodido/go-urn v1.2.1 // indirect 42 | github.com/mattn/go-isatty v0.0.17 // indirect 43 | github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 // indirect 44 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 45 | github.com/modern-go/reflect2 v1.0.2 // indirect 46 | github.com/pelletier/go-toml/v2 v2.0.6 // indirect 47 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 48 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 49 | github.com/ugorji/go/codec v1.2.9 // indirect 50 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect 51 | golang.org/x/crypto v0.5.0 // indirect 52 | golang.org/x/net v0.7.0 // indirect 53 | golang.org/x/sys v0.5.0 // indirect 54 | golang.org/x/text v0.7.0 // indirect 55 | google.golang.org/appengine v1.6.7 // indirect 56 | google.golang.org/protobuf v1.28.1 // indirect 57 | gopkg.in/yaml.v2 v2.4.0 // indirect 58 | gopkg.in/yaml.v3 v3.0.1 // indirect 59 | ) 60 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | git.blob42.xyz/blob42/gum.git v0.0.0-20190304130815-31be968b7b17 h1:WJcA7ckN0Jue2YtYvhHQK9xeEwjgKwOi0/WNy0WCbBk= 2 | git.blob42.xyz/blob42/gum.git v0.0.0-20190304130815-31be968b7b17/go.mod h1:lTOrYVRIu1rD8y8tB1xU3biiAPfg0iee7w7+iPtpSeg= 3 | github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= 4 | github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 5 | github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= 6 | github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= 7 | github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= 8 | github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= 9 | github.com/beeker1121/goque v2.1.0+incompatible h1:m5pZ5b8nqzojS2DF2ioZphFYQUqGYsDORq6uefUItPM= 10 | github.com/beeker1121/goque v2.1.0+incompatible/go.mod h1:L6dOWBhDOnxUVQsb0wkLve0VCnt2xJW/MI8pdRX4ANw= 11 | github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= 12 | github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA= 13 | github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= 14 | github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= 15 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= 16 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= 17 | github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= 18 | github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 19 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= 23 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 24 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 25 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 26 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 27 | github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= 28 | github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= 29 | github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= 30 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= 31 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= 32 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= 33 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= 34 | github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= 35 | github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= 36 | github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= 37 | github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 38 | github.com/gobuffalo/flect v1.0.0 h1:eBFmskjXZgAOagiTXJH25Nt5sdFwNRcb8DKZsIsAUQI= 39 | github.com/gobuffalo/flect v1.0.0/go.mod h1:l9V6xSb4BlXwsxEMj3FVEub2nkdQjWhPvD8XTTlHPQc= 40 | github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= 41 | github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= 42 | github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= 43 | github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 44 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 45 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 46 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 47 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 48 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 49 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 50 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 51 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 52 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 53 | github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= 54 | github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= 55 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= 56 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 57 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 58 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 59 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 60 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 61 | github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= 62 | github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= 63 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 64 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 65 | github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= 66 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 67 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= 68 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 69 | github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= 70 | github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= 71 | github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= 72 | github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 73 | github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= 74 | github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 75 | github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 76 | github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= 77 | github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= 78 | github.com/mmcdole/gofeed v1.2.0 h1:kuq7tJnDf0pnsDzF820ukuySHxFimAcizpG15gYHIns= 79 | github.com/mmcdole/gofeed v1.2.0/go.mod h1:TEyTG4gw4Q5Co+Hgahx/Oi3E0JHLM8BXtWC+mkJtRsw= 80 | github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354 h1:Z6i7ND25ixRtXFBylIUggqpvLMV1I15yprcqMVB7WZA= 81 | github.com/mmcdole/goxpp v0.0.0-20200921145534-2f3784f67354/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= 82 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 83 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 84 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 85 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 86 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 87 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 88 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 89 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 90 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 91 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 92 | github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= 93 | github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= 94 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 95 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 96 | github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= 97 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 98 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 99 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= 100 | github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= 101 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 102 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 103 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 104 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 105 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 106 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 107 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 108 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 109 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 110 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 111 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 112 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 113 | github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 h1:xzABM9let0HLLqFypcxvLmlvEciCHL7+Lv+4vwZqecI= 114 | github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569/go.mod h1:2Ly+NIftZN4de9zRmENdYbvPQeaVIYKWpLFStLFEBgI= 115 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 116 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 117 | github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= 118 | github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= 119 | github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= 120 | github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= 121 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 122 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= 123 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= 124 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 125 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 126 | golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= 127 | golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= 128 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 129 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 130 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 131 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 132 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 133 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 134 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 135 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 136 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 137 | golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= 138 | golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= 139 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 140 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 141 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 142 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 143 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 144 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 145 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 146 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 147 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 148 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 149 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 150 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 151 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 152 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 153 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 154 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 155 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 156 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 157 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 158 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 159 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 160 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 161 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 162 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 163 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 164 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 165 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 166 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 167 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= 168 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 169 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 170 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 171 | google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= 172 | google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 173 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 174 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 175 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 176 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 177 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 178 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 179 | gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= 180 | gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= 181 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 182 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 183 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 184 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 185 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 186 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 187 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 188 | -------------------------------------------------------------------------------- /handlers/handlers.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/posts" 6 | "log" 7 | "time" 8 | ) 9 | 10 | type JobHandler interface { 11 | // Main handling function 12 | Handle(feeds.Feed) error 13 | } 14 | 15 | type FormatHandler interface { 16 | FetchSince(url string, time time.Time) ([]*posts.Post, error) 17 | JobHandler // Also implements a job handler 18 | } 19 | 20 | func GetFormatHandler(feed feeds.Feed) FormatHandler { 21 | 22 | var handler FormatHandler 23 | 24 | switch feed.Format { 25 | case feeds.FormatRSS: 26 | handler = NewRSSHandler() 27 | case feeds.FormatRFC: 28 | handler = NewRFCHandler() 29 | case feeds.FormatGHRelease: 30 | handler = NewGHReleaseHandler() 31 | default: 32 | log.Printf("WARNING: No format handler for %s", feed.FormatString) 33 | } 34 | 35 | return handler 36 | } 37 | -------------------------------------------------------------------------------- /handlers/releases.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/github" 6 | "git.blob42.xyz/blob42/hugobot/v3/posts" 7 | "git.blob42.xyz/blob42/hugobot/v3/utils" 8 | "context" 9 | "errors" 10 | "log" 11 | "time" 12 | 13 | githubApi "github.com/google/go-github/github" 14 | ) 15 | 16 | const ( 17 | NoReleaseForProjectTreshold = 10 18 | ) 19 | 20 | type GHRelease struct { 21 | ProjectID int64 `json:"project_id"` 22 | ReleaseID int64 `json:"release_id"` 23 | TagID string `json:"tag_id"` 24 | Name string `json:"name"` 25 | IsTagOnly bool `json:"is_tag_only"` 26 | Date time.Time `json:"commit_date"` 27 | TarBall string `json:"tar_ball"` 28 | Link string `json:"link"` 29 | Owner string `json:"owner"` 30 | Repo string `json:"repo"` 31 | } 32 | 33 | type GHReleaseHandler struct { 34 | ctx context.Context 35 | ghClient *githubApi.Client 36 | } 37 | 38 | func (handler GHReleaseHandler) Handle(feed feeds.Feed) error { 39 | posts, err := handler.FetchSince(feed.Url, feed.LastRefresh) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | if posts == nil { 45 | log.Printf("No new posts in feed <%s>", feed.Name) 46 | } 47 | 48 | for _, p := range posts { 49 | var err error 50 | isTagOnly, ok := p.JsonData["is_tag_only"].(bool) 51 | 52 | if !ok { 53 | return errors.New("could not convert is_tag_only to bool") 54 | } 55 | 56 | if isTagOnly { 57 | err = p.Write(feed.FeedID) 58 | } else { 59 | err = p.WriteWithShortId(feed.FeedID, p.JsonData["release_id"]) 60 | } 61 | if err != nil { 62 | return err 63 | } 64 | } 65 | 66 | return nil 67 | } 68 | 69 | func (handler GHReleaseHandler) FetchSince(url string, 70 | after time.Time) ([]*posts.Post, error) { 71 | var results []*posts.Post 72 | 73 | log.Printf("Fetching GH release %s since %v", url, after) 74 | 75 | owner, repo := github.ParseOwnerRepo(url) 76 | 77 | project, resp, err := handler.ghClient.Repositories.Get(handler.ctx, owner, repo) 78 | if resp == nil { 79 | return nil, errors.New("No response") 80 | } 81 | 82 | github.RespMiddleware(resp) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | // Handle releases first, if project has no releases use tags instead 88 | listReleasesOptions := &githubApi.ListOptions{ 89 | PerPage: 100, 90 | } 91 | 92 | releases, resp, err := handler.ghClient.Repositories.ListReleases( 93 | handler.ctx, owner, repo, listReleasesOptions, 94 | ) 95 | github.RespMiddleware(resp) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | // If no releases use tags 101 | if len(releases) <= 0 { 102 | log.Println("no releases, using tags") 103 | var allTags []*githubApi.RepositoryTag 104 | // Handle tags first 105 | listTagOptions := &githubApi.ListOptions{PerPage: 100} 106 | 107 | for { 108 | tags, resp, err := handler.ghClient.Repositories.ListTags( 109 | handler.ctx, owner, repo, listTagOptions, 110 | ) 111 | 112 | if err != nil { 113 | return nil, err 114 | } 115 | 116 | allTags = append(allTags, tags...) 117 | if resp.NextPage == 0 { 118 | break 119 | } 120 | 121 | listTagOptions.Page = resp.NextPage 122 | } 123 | 124 | for _, tag := range allTags { 125 | //var release *githubApi.RepositoryRelease 126 | // 127 | commit, resp, err := handler.ghClient.Repositories.GetCommit( 128 | handler.ctx, owner, repo, tag.GetCommit().GetSHA(), 129 | ) 130 | github.RespMiddleware(resp) 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | if commit.GetCommit().GetCommitter().GetDate().Before(after) { 136 | break 137 | } 138 | 139 | ghRelease := GHRelease{ 140 | ProjectID: project.GetID(), 141 | TagID: tag.GetName(), 142 | IsTagOnly: true, 143 | Date: commit.GetCommit().GetCommitter().GetDate(), 144 | TarBall: tag.GetTarballURL(), 145 | Owner: owner, 146 | Repo: repo, 147 | } 148 | 149 | post := &posts.Post{} 150 | post.Title = tag.GetName() 151 | post.Link = ghRelease.TarBall 152 | post.Published = ghRelease.Date 153 | post.Updated = post.Published 154 | post.JsonData = utils.StructToJsonMap(ghRelease) 155 | post.Author = commit.GetAuthor().GetName() 156 | 157 | results = append(results, post) 158 | } 159 | } else { 160 | 161 | for _, release := range releases { 162 | 163 | ghRelease := GHRelease{ 164 | ProjectID: project.GetID(), 165 | ReleaseID: release.GetID(), 166 | Name: release.GetName(), 167 | TagID: release.GetTagName(), 168 | IsTagOnly: false, 169 | Date: release.GetCreatedAt().Time, 170 | TarBall: release.GetTarballURL(), 171 | Owner: owner, 172 | Repo: repo, 173 | } 174 | 175 | post := &posts.Post{} 176 | post.Title = release.GetName() 177 | post.Link = release.GetHTMLURL() 178 | post.Published = release.GetPublishedAt().Time 179 | post.Updated = release.GetPublishedAt().Time 180 | post.JsonData = utils.StructToJsonMap(ghRelease) 181 | post.Author = release.GetAuthor().GetName() 182 | post.Content = release.GetBody() 183 | 184 | results = append(results, post) 185 | } 186 | 187 | } 188 | 189 | return results, nil 190 | } 191 | 192 | func NewGHReleaseHandler() FormatHandler { 193 | ctxb := context.Background() 194 | client := github.Auth(ctxb) 195 | 196 | return GHReleaseHandler{ 197 | ctx: ctxb, 198 | ghClient: client, 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /handlers/rfc.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 5 | "git.blob42.xyz/blob42/hugobot/v3/github" 6 | "git.blob42.xyz/blob42/hugobot/v3/posts" 7 | "git.blob42.xyz/blob42/hugobot/v3/static" 8 | "git.blob42.xyz/blob42/hugobot/v3/utils" 9 | "context" 10 | "fmt" 11 | "log" 12 | "regexp" 13 | "strconv" 14 | "strings" 15 | "time" 16 | 17 | githubApi "github.com/google/go-github/github" 18 | ) 19 | 20 | var ( 21 | ReBIP = regexp.MustCompile(`bips?[\s-]*(?P[0-9]+)`) 22 | ReBOLT = regexp.MustCompile(`bolt?[\s-]*(?P[0-9]+)`) 23 | ReSLIP = regexp.MustCompile(`slips?[\s-]*(?P[0-9]+)`) 24 | ) 25 | 26 | const ( 27 | BIPLink = "https://github.com/bitcoin/bips/blob/master/bip-%04d.mediawiki" 28 | SLIPLink = "https://github.com/satoshilabs/slips/blob/master/slip-%04d.md" 29 | ) 30 | 31 | const ( 32 | BOLT = 73738971 33 | BIP = 14531737 34 | SLIP = 50844973 35 | ) 36 | 37 | var RFCTypes = map[int64]string{ 38 | BIP: "bip", 39 | BOLT: "bolt", 40 | SLIP: "slip", 41 | } 42 | 43 | type RFCUpdate struct { 44 | RFCID int64 `json:"rfcid"` 45 | RFCType string `json:"rfc_type"` //bip bolt slip 46 | RFCNumber int `json:"rfc_number"` // (bip/bolt/slip id) 47 | Issue *github.Issue `json:"issue"` 48 | RFCLink string `json:"rfc_link"` 49 | } 50 | 51 | type RFCHandler struct { 52 | ctx context.Context 53 | ghClient *githubApi.Client 54 | } 55 | 56 | func (handler RFCHandler) Handle(feed feeds.Feed) error { 57 | 58 | posts, err := handler.FetchSince(feed.Url, feed.LastRefresh) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | if posts == nil { 64 | log.Printf("No new posts in feed <%s>", feed.Name) 65 | } 66 | 67 | for _, p := range posts { 68 | // Since RFCs are based on github issues, we use their id as unique 69 | // id in the local sqlite db 70 | err := p.WriteWithShortId(feed.FeedID, p.JsonData["rfcid"]) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | } 76 | 77 | return nil 78 | } 79 | 80 | func (handler RFCHandler) FetchSince(url string, after time.Time) ([]*posts.Post, error) { 81 | var results []*posts.Post 82 | 83 | log.Printf("Fetching RFC %s since %v", url, after) 84 | 85 | owner, repo := github.ParseOwnerRepo(url) 86 | 87 | project, resp, err := handler.ghClient.Repositories.Get(handler.ctx, owner, repo) 88 | github.RespMiddleware(resp) 89 | 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | //All Issues 95 | var allIssues []*githubApi.Issue 96 | listIssueOptions := &githubApi.IssueListByRepoOptions{ 97 | Since: after, 98 | State: "all", 99 | ListOptions: githubApi.ListOptions{PerPage: 100}, 100 | } 101 | 102 | for { 103 | 104 | issues, resp, err := handler.ghClient.Issues.ListByRepo( 105 | handler.ctx, owner, repo, listIssueOptions) 106 | 107 | if err != nil { 108 | return nil, err 109 | } 110 | 111 | allIssues = append(allIssues, issues...) 112 | 113 | if resp.NextPage == 0 { 114 | break 115 | } 116 | listIssueOptions.Page = resp.NextPage 117 | 118 | } 119 | 120 | for iIndex, issue := range allIssues { 121 | var pr *githubApi.PullRequest 122 | 123 | // base rfc object 124 | rfc := RFCUpdate{ 125 | RFCID: issue.GetID(), 126 | RFCType: RFCTypes[*project.ID], 127 | Issue: &github.Issue{ 128 | Title: issue.GetTitle(), 129 | URL: issue.GetURL(), 130 | Number: issue.GetNumber(), 131 | State: issue.GetState(), 132 | Updated: issue.GetUpdatedAt(), 133 | Created: issue.GetCreatedAt(), 134 | Comments: issue.GetComments(), 135 | HtmlURL: issue.GetHTMLURL(), 136 | }, 137 | } 138 | 139 | if issue.IsPullRequest() { 140 | 141 | log.Printf("parsing %s. Progress %d/%d\n", url, iIndex+1, len(allIssues)) 142 | 143 | pr, resp, err = handler.ghClient.PullRequests.Get( 144 | handler.ctx, owner, repo, issue.GetNumber(), 145 | ) 146 | //github.RespMiddleware(resp) 147 | 148 | if err != nil { 149 | return nil, err 150 | } 151 | 152 | rfc.Issue.IsPR = issue.IsPullRequest() 153 | rfc.Issue.Merged = *pr.Merged 154 | 155 | if rfc.Issue.Merged { 156 | rfc.Issue.MergedAt = *pr.MergedAt 157 | } 158 | } 159 | 160 | // If is open and is not new (update) mark as update 161 | if rfc.Issue.Created != rfc.Issue.Updated && 162 | !rfc.Issue.Merged && 163 | rfc.Issue.State == "open" { 164 | rfc.Issue.IsUpdate = true 165 | } 166 | 167 | rfc.RFCNumber, err = GetRFCNumber(issue.GetTitle()) 168 | if err != nil { 169 | return nil, err 170 | } 171 | 172 | if rfc.RFCNumber != -1 { 173 | 174 | switch rfc.RFCType { 175 | case RFCTypes[BIP]: 176 | rfc.RFCLink = fmt.Sprintf(BIPLink, rfc.RFCNumber) 177 | case RFCTypes[SLIP]: 178 | rfc.RFCLink = fmt.Sprintf(SLIPLink, rfc.RFCNumber) 179 | case RFCTypes[BOLT]: 180 | rfc.RFCLink = static.BoltMap[rfc.RFCNumber] 181 | } 182 | } 183 | 184 | post := &posts.Post{} 185 | post.Title = rfc.Issue.Title 186 | post.Link = rfc.Issue.URL 187 | post.Published = rfc.Issue.Created 188 | post.Updated = rfc.Issue.Updated 189 | post.JsonData = utils.StructToJsonMap(rfc) 190 | 191 | results = append(results, post) 192 | } 193 | 194 | return results, nil 195 | } 196 | 197 | func NewRFCHandler() FormatHandler { 198 | ctxb := context.Background() 199 | client := github.Auth(ctxb) 200 | 201 | return RFCHandler{ 202 | ctx: ctxb, 203 | ghClient: client, 204 | } 205 | } 206 | 207 | func GetRFCNumber(title string) (int, error) { 208 | 209 | // Detect BIP 210 | for _, re := range []*regexp.Regexp{ReBIP, ReBOLT, ReSLIP} { 211 | 212 | matches := re.FindStringSubmatch(strings.ToLower(title)) 213 | 214 | if matches != nil { 215 | res, err := strconv.Atoi(matches[1]) 216 | if err != nil { 217 | return -1, err 218 | } 219 | 220 | return res, nil 221 | } 222 | 223 | } 224 | 225 | return -1, nil 226 | } 227 | -------------------------------------------------------------------------------- /handlers/rss.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/export" 5 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 6 | "git.blob42.xyz/blob42/hugobot/v3/posts" 7 | "log" 8 | "strings" 9 | "time" 10 | 11 | "github.com/fatih/structs" 12 | "github.com/mmcdole/gofeed" 13 | ) 14 | 15 | type RSSHandler struct { 16 | rssFeed *gofeed.Feed 17 | } 18 | 19 | func (handler RSSHandler) Handle(feed feeds.Feed) error { 20 | 21 | posts, err := handler.FetchSince(feed.Url, feed.LastRefresh) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | if posts == nil { 27 | log.Printf("No new posts in feed <%s>", feed.Name) 28 | } 29 | 30 | // Write posts to DB 31 | for _, p := range posts { 32 | err := p.Write(feed.FeedID) 33 | if err != nil { 34 | return err 35 | } 36 | } 37 | 38 | return nil 39 | } 40 | 41 | func (handler RSSHandler) FetchSince(url string, after time.Time) ([]*posts.Post, error) { 42 | var err error 43 | var fetchedPosts []*posts.Post 44 | 45 | log.Printf("Fetching RSS since %v", after) 46 | fp := gofeed.NewParser() 47 | handler.rssFeed, err = fp.ParseURL(url) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | for _, item := range handler.rssFeed.Items { 53 | if item.PublishedParsed.After(after) { 54 | //log.Println(item.Title) 55 | 56 | post := &posts.Post{} 57 | 58 | if item.Author != nil { 59 | post.Author = item.Author.Name 60 | } 61 | 62 | post.Title = item.Title 63 | 64 | // If content is in description 65 | // store them in reverse in the post 66 | if len(item.Content) == 0 && 67 | len(item.Description) > 0 { 68 | post.Content = item.Description 69 | // If content is same as description 70 | } else if item.Content == item.Description { 71 | post.Content = item.Content 72 | post.PostDescription = "" 73 | 74 | } else { 75 | post.Content = item.Content 76 | post.PostDescription = item.Description 77 | } 78 | 79 | post.Link = item.Link 80 | 81 | if item.UpdatedParsed != nil { 82 | post.Updated = *item.UpdatedParsed 83 | } else { 84 | post.Updated = *item.PublishedParsed 85 | } 86 | 87 | if item.PublishedParsed != nil { 88 | post.Published = *item.PublishedParsed 89 | } 90 | 91 | post.Tags = strings.Join(item.Categories, ",") 92 | 93 | item.Content = "" 94 | item.Description = "" 95 | post.JsonData = structs.Map(item) 96 | 97 | fetchedPosts = append(fetchedPosts, post) 98 | } 99 | } 100 | 101 | return fetchedPosts, nil 102 | } 103 | 104 | func NewRSSHandler() FormatHandler { 105 | return RSSHandler{} 106 | } 107 | 108 | func RSSExportMapper(exp export.Map, feed feeds.Feed, post posts.Post) error { 109 | if feed.Format == feeds.FormatRSS { 110 | exp["updated"] = post.Updated 111 | } 112 | 113 | return nil 114 | } 115 | 116 | func init() { 117 | export.RegisterPostMapper(RSSExportMapper) 118 | } 119 | -------------------------------------------------------------------------------- /hugobot.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blob42/hugobot/c4ed0c48ba802f435c3c9c0dbf901a241bfaf36d/hugobot.pdf -------------------------------------------------------------------------------- /jobs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/export" 5 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 6 | "git.blob42.xyz/blob42/hugobot/v3/handlers" 7 | "git.blob42.xyz/blob42/hugobot/v3/posts" 8 | "git.blob42.xyz/blob42/hugobot/v3/utils" 9 | "bytes" 10 | "encoding/gob" 11 | "encoding/json" 12 | "fmt" 13 | "log" 14 | "time" 15 | 16 | "github.com/beeker1121/goque" 17 | "github.com/gofrs/uuid" 18 | "github.com/syndtr/goleveldb/leveldb" 19 | ) 20 | 21 | type JobStatus int 22 | type JobType int 23 | 24 | const ( 25 | JobStatusNew JobStatus = iota 26 | JobStatusQueued 27 | JobStatusDone 28 | JobStatusFailed 29 | ) 30 | 31 | const ( 32 | JobTypeFetch JobType = iota 33 | JobTypeExport 34 | ) 35 | 36 | var ( 37 | JobTypeMap = map[JobType]string{ 38 | JobTypeFetch: "fetch", 39 | JobTypeExport: "export", 40 | } 41 | 42 | JobStatusMap = map[JobStatus]string{ 43 | JobStatusNew: "new", 44 | JobStatusQueued: "queued", 45 | JobStatusDone: "done", 46 | JobStatusFailed: "failed", 47 | } 48 | ) 49 | 50 | func (js JobStatus) String() string { 51 | return JobStatusMap[js] 52 | } 53 | 54 | type Prioritizer interface { 55 | // Return job priority 56 | GetPriority() uint8 57 | } 58 | 59 | // Represents a Job to be done on a feed 60 | // It could be any of: Poll, Fetch, Store 61 | // Should implement Poller 62 | type Job struct { 63 | ID uuid.UUID 64 | Feed *feeds.Feed 65 | Status JobStatus 66 | Data []*posts.Post 67 | 68 | Priority uint8 69 | JobType JobType 70 | Serial bool // Should be run in a serial manner 71 | 72 | Err error 73 | 74 | Prioritizer 75 | } 76 | 77 | type Handler interface { 78 | Handle() 79 | } 80 | 81 | // GoRoutine method 82 | func (job *Job) Handle() { 83 | var err error 84 | 85 | if job.JobType == JobTypeFetch { 86 | handler := handlers.GetFormatHandler(*job.Feed) 87 | err = handler.Handle(*job.Feed) 88 | } else if job.JobType == JobTypeExport { 89 | handler := export.NewHugoExporter() 90 | err = handler.Handle(*job.Feed) 91 | } 92 | 93 | if err != nil { 94 | job.Failed(err) 95 | return 96 | } 97 | //log.Println("Done for job type ", job.JobType) 98 | job.Done() 99 | } 100 | 101 | func (job *Job) Failed(err error) { 102 | errr := job.Feed.UpdateRefreshTime(time.Now()) 103 | if errr != nil { 104 | log.Fatal(errr) 105 | } 106 | 107 | job.Status = JobStatusFailed 108 | job.Err = err 109 | NotifyScheduler(job) 110 | } 111 | 112 | func (job *Job) Done() { 113 | //TODO: only update refresh time after actual fetching 114 | // 115 | err := job.Feed.UpdateRefreshTime(time.Now()) 116 | if err != nil { 117 | log.Fatal(err) 118 | } 119 | 120 | job.Status = JobStatusDone 121 | NotifyScheduler(job) 122 | } 123 | 124 | func (job *Job) GetPriority() uint8 { 125 | return job.Priority 126 | } 127 | 128 | func (job *Job) String() string { 129 | exp := map[string]interface{}{ 130 | "jobId": job.ID, 131 | "feed": job.Feed.Name, 132 | "priority": job.Priority, 133 | "jobType": JobTypeMap[job.JobType], 134 | "serial": job.Serial, 135 | "err": job.Err, 136 | } 137 | 138 | b, err := json.MarshalIndent(exp, "", " ") 139 | if err != nil { 140 | log.Printf("error printing job %s\n", err) 141 | return "" 142 | } 143 | return fmt.Sprintf(string(b)) 144 | 145 | } 146 | 147 | // Decode object from []byte 148 | func JobFromBytes(value []byte) (*Job, error) { 149 | buffer := bytes.NewBuffer(value) 150 | dec := gob.NewDecoder(buffer) 151 | 152 | j := &Job{} 153 | 154 | err := dec.Decode(j) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | return j, nil 160 | } 161 | 162 | // helper function for jobs that accepts any 163 | // value type, which is then encoded into a byte slice using 164 | // encoding/gob. 165 | func (job *Job) ToBytes() ([]byte, error) { 166 | var buffer bytes.Buffer 167 | enc := gob.NewEncoder(&buffer) 168 | if err := enc.Encode(job); err != nil { 169 | return nil, err 170 | } 171 | 172 | return buffer.Bytes(), nil 173 | } 174 | 175 | func NewFetchJob(feed *feeds.Feed, 176 | priority uint8) (*Job, error) { 177 | 178 | uuid, err := uuid.NewV4() 179 | if err != nil { 180 | return nil, err 181 | } 182 | 183 | job := &Job{ 184 | ID: uuid, 185 | Feed: feed, 186 | Status: JobStatusNew, 187 | JobType: JobTypeFetch, 188 | Priority: priority, 189 | Serial: feed.Serial, 190 | } 191 | 192 | return job, nil 193 | } 194 | 195 | func NewExportJob(feed *feeds.Feed, 196 | priority uint8) (*Job, error) { 197 | 198 | uuid, err := uuid.NewV4() 199 | if err != nil { 200 | return nil, err 201 | } 202 | 203 | job := &Job{ 204 | ID: uuid, 205 | Feed: feed, 206 | Status: JobStatusNew, 207 | Priority: priority, 208 | JobType: JobTypeExport, 209 | } 210 | 211 | return job, nil 212 | } 213 | 214 | type Queuer interface { 215 | Enqueue(job *Job) (*Job, error) 216 | Dequeue() (*Job, error) 217 | Close() error 218 | Drop() error // Clsoe and delete all jobs 219 | Length() uint64 220 | //Peek() (*Job, error) 221 | //PeekByID(id uint64) (*Job, error) 222 | 223 | // Returns item located at given offset starting from head 224 | // of queue without removing it 225 | //PeekByOffset(offset uint64) (*Job, error) 226 | } 227 | 228 | // Represents the queue of fetching todo jobs 229 | type JobPool struct { 230 | // Actual jobs queue 231 | Q *goque.PriorityQueue 232 | 233 | // Handle queuing mechanics 234 | Queuer 235 | 236 | maxJobs int 237 | 238 | feedJobMap *leveldb.DB 239 | } 240 | 241 | func (jp *JobPool) Close() error { 242 | jp.Q.Close() 243 | 244 | err := jp.feedJobMap.Close() 245 | return err 246 | } 247 | 248 | func (jp *JobPool) Dequeue() (*Job, error) { 249 | item, err := jp.Q.Dequeue() 250 | if err != nil { 251 | return nil, err 252 | } 253 | j := &Job{} 254 | item.ToObject(j) 255 | 256 | //TODO: This is done when the job is done 257 | //feedId := utils.IntToBytes(j.Feed.ID) 258 | //err = jp.feedJobMap.Delete(feedId, nil) 259 | //if err != nil { 260 | //return nil, err 261 | //} 262 | 263 | return j, nil 264 | } 265 | 266 | func (jp *JobPool) DeleteMarkedJob(job *Job) error { 267 | var err error 268 | 269 | feedId := utils.IntToBytes(job.Feed.FeedID) 270 | err = jp.feedJobMap.Delete(feedId, nil) 271 | 272 | return err 273 | } 274 | 275 | // Mark a job in feedJobMap to avoid duplicates 276 | func (jp *JobPool) MarkUniqJob(job *Job) error { 277 | 278 | // Mark the feed in the feedJobMap to avoid creating duplicates 279 | feedId := utils.IntToBytes(job.Feed.FeedID) 280 | 281 | jobData, err := job.ToBytes() 282 | if err != nil { 283 | return err 284 | } 285 | 286 | err = jp.feedJobMap.Put(feedId, jobData, nil) 287 | if err != nil { 288 | return err 289 | } 290 | 291 | return nil 292 | } 293 | 294 | func (jp *JobPool) Enqueue(job *Job) error { 295 | 296 | // Update job status 297 | job.Status = JobStatusQueued 298 | 299 | // Enqueue the job in the jobpool 300 | item, err := jp.Q.EnqueueObject(job.GetPriority(), job) 301 | if err != nil { 302 | return err 303 | } 304 | 305 | // Recode item to job 306 | j := &Job{} 307 | item.ToObject(j) 308 | 309 | return nil 310 | } 311 | func (jp *JobPool) Drop() { 312 | jp.Q.Drop() 313 | } 314 | 315 | func (jp *JobPool) Length() uint64 { 316 | return jp.Q.Length() 317 | } 318 | 319 | func (jp *JobPool) Peek() (*Job, error) { 320 | item, err := jp.Q.Peek() 321 | if err != nil { 322 | return nil, err 323 | } 324 | 325 | j := &Job{} 326 | item.ToObject(j) 327 | return j, err 328 | } 329 | -------------------------------------------------------------------------------- /logging/log.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "log" 5 | "os" 6 | ) 7 | 8 | const ( 9 | logFileName = ".log" 10 | ) 11 | 12 | var ( 13 | logFile *os.File 14 | ) 15 | 16 | func Close() error { 17 | return logFile.Close() 18 | } 19 | 20 | func init() { 21 | //var err error 22 | //logFile, err = os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) 23 | //if err != nil { 24 | //log.Fatal(err) 25 | //} 26 | 27 | //log.SetOutput(io.MultiWriter(logFile, os.Stdout)) 28 | log.SetOutput(os.Stdout) 29 | } 30 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "git.blob42.xyz/blob42/hugobot/v3/config" 8 | "git.blob42.xyz/blob42/hugobot/v3/logging" 9 | 10 | "github.com/gobuffalo/flect" 11 | altsrc "github.com/urfave/cli/altsrc" 12 | cli "gopkg.in/urfave/cli.v1" 13 | ) 14 | 15 | func tearDown() { 16 | DB.Handle.Close() 17 | logging.Close() 18 | } 19 | 20 | func main() { 21 | defer tearDown() 22 | 23 | app := cli.NewApp() 24 | app.Name = "hugobot" 25 | app.Version = "1.0" 26 | flags := []cli.Flag{ 27 | altsrc.NewStringFlag(cli.StringFlag{ 28 | Name: "website-path", 29 | Usage: "`PATH` to hugo project", 30 | EnvVar: "WEBSITE_PATH", 31 | }), 32 | altsrc.NewStringFlag(cli.StringFlag{ 33 | Name: "github-access-token", 34 | Usage: "Github API Access Token", 35 | EnvVar: "GH_ACCESS_TOKEN", 36 | }), 37 | altsrc.NewStringFlag(cli.StringFlag{ 38 | Name: "rel-bitcoin-addr-content-path", 39 | Usage: "path to bitcoin data relative to hugo path", 40 | }), 41 | altsrc.NewIntFlag(cli.IntFlag{ 42 | Name: "api-port", 43 | Usage: "default bot api port", 44 | }), 45 | 46 | cli.StringFlag{ 47 | Name: "config", 48 | Value: "config.toml", 49 | Usage: "TOML config `FILE` path", 50 | }, 51 | } 52 | 53 | app.Before = func(c *cli.Context) error { 54 | 55 | err := altsrc.InitInputSourceWithContext(flags, 56 | altsrc.NewTomlSourceFromFlagFunc("config"))(c) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | for _, conf := range c.GlobalFlagNames() { 62 | 63 | // find corresponding flag 64 | for _, flag := range flags { 65 | if flag.GetName() == conf { 66 | switch flag.(type) { 67 | case cli.StringFlag: 68 | err = config.RegisterConf(flect.Pascalize(conf), c.GlobalString(conf)) 69 | case *altsrc.StringFlag: 70 | err = config.RegisterConf(flect.Pascalize(conf), c.GlobalString(conf)) 71 | case cli.IntFlag: 72 | err = config.RegisterConf(flect.Pascalize(conf), c.GlobalInt(conf)) 73 | case *altsrc.IntFlag: 74 | err = config.RegisterConf(flect.Pascalize(conf), c.GlobalInt(conf)) 75 | 76 | } 77 | } 78 | } 79 | 80 | } 81 | 82 | return err 83 | } 84 | 85 | app.Flags = flags 86 | 87 | app.Commands = []cli.Command{ 88 | startServerCmd, 89 | exportCmdGrp, 90 | feedsCmdGroup, 91 | } 92 | 93 | if err := app.Run(os.Args); err != nil { 94 | log.Fatal(err) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /parse_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/handlers" 5 | "fmt" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | const ( 12 | rssTestFeed = "https://bitcointechweekly.com/index.xml" 13 | rssTestFeed2 = "https://bitcoinops.org/feed.xml" 14 | ) 15 | 16 | func TestFetch(t *testing.T) { 17 | handler := handlers.NewRSSHandler() 18 | when, _ := time.Parse("Jan 2006", "Jun 2018") 19 | res, err := handler.FetchSince(rssTestFeed2, when) 20 | if err != nil { 21 | t.Error(err) 22 | } 23 | 24 | for i, post := range res { 25 | f, err := os.Create(fmt.Sprintf("%d.html", i)) 26 | if err != nil { 27 | t.Error(err) 28 | } 29 | defer f.Close() 30 | _, err = f.WriteString(post.Content) 31 | if err != nil { 32 | t.Error(err) 33 | } 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /posts/posts.go: -------------------------------------------------------------------------------- 1 | package posts 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/db" 5 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 6 | "git.blob42.xyz/blob42/hugobot/v3/types" 7 | "git.blob42.xyz/blob42/hugobot/v3/utils" 8 | "errors" 9 | "fmt" 10 | "log" 11 | "strconv" 12 | "time" 13 | 14 | sqlite3 "github.com/mattn/go-sqlite3" 15 | ) 16 | 17 | const ( 18 | DBPostsSchema = `CREATE TABLE IF NOT EXISTS posts ( 19 | post_id INTEGER PRIMARY KEY, 20 | feed_id INTEGER NOT NULL, 21 | title TEXT DEFAULT '', 22 | description TEXT DEFAULT '', 23 | link TEXT NOT NULL, 24 | updated timestamp NOT NULL, 25 | published timestamp NOT NULL, 26 | author TEXT DEFAULT '', 27 | content TEXT DEFAULT '', 28 | tags TEXT DEFAULT '', 29 | json_data BLOB DEFAULT '', 30 | short_id TEXT UNIQUE, 31 | FOREIGN KEY (feed_id) REFERENCES feeds(feed_id) 32 | )` 33 | ) 34 | 35 | var ( 36 | ErrDoesNotExist = errors.New("does not exist") 37 | ErrAlreadyExists = errors.New("already exists") 38 | ) 39 | 40 | var DB = db.DB 41 | 42 | type Post struct { 43 | PostID int64 `josn:"id" db:"post_id"` 44 | Title string `json:"title"` 45 | PostDescription string `json:"description" db:"post_description"` 46 | Link string `json:"link"` 47 | Updated time.Time `json:"updated"` 48 | Published time.Time `json:"published"` 49 | Author string `json:"author"` 50 | Content string `json:"content"` 51 | Tags string `json:"tags"` 52 | ShortID string `json:"short_id" db:"short_id"` 53 | JsonData types.JsonMap `json:"data" db:"json_data"` 54 | 55 | feeds.Feed 56 | } 57 | 58 | // Writes with provided short id 59 | func (post *Post) WriteWithShortId(feedId int64, shortId interface{}) error { 60 | var shortid string 61 | 62 | switch v := shortId.(type) { 63 | case int: 64 | shortid = strconv.Itoa(v) 65 | case int64: 66 | shortid = strconv.Itoa(int(v)) 67 | case string: 68 | shortid = v 69 | default: 70 | return fmt.Errorf("Cannot convert %v to string", shortId) 71 | 72 | } 73 | return write(post, feedId, shortid) 74 | } 75 | 76 | // Auto generates shortId 77 | func (post *Post) Write(feedId int64) error { 78 | 79 | shortId, err := utils.GetSIDGenerator().Generate() 80 | if err != nil { 81 | return err 82 | } 83 | 84 | return write(post, feedId, shortId) 85 | } 86 | 87 | func write(post *Post, feedId int64, shortId string) error { 88 | const query = `INSERT OR REPLACE INTO posts ( 89 | feed_id, 90 | title, 91 | description, 92 | link, 93 | updated, 94 | published, 95 | author, 96 | content, 97 | json_data, 98 | short_id, 99 | tags 100 | ) 101 | 102 | VALUES( 103 | ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? 104 | ) 105 | ` 106 | 107 | _, err := DB.Handle.Exec(query, 108 | feedId, 109 | post.Title, 110 | post.PostDescription, 111 | post.Link, 112 | post.Updated, 113 | post.Published, 114 | post.Author, 115 | post.Content, 116 | post.JsonData, 117 | shortId, 118 | post.Tags, 119 | ) 120 | 121 | sqlErr, isSqlErr := err.(sqlite3.Error) 122 | if isSqlErr && sqlErr.Code == sqlite3.ErrConstraint { 123 | return fmt.Errorf("%+v --- %s ", sqlErr, post.Title) 124 | } 125 | 126 | if err != nil { 127 | return err 128 | } 129 | 130 | return nil 131 | } 132 | 133 | func ListPosts() ([]Post, error) { 134 | const query = `SELECT * FROM posts JOIN feeds ON posts.feed_id = feeds.feed_id` 135 | var posts []Post 136 | err := DB.Handle.Select(&posts, query) 137 | if err != nil { 138 | return nil, err 139 | } 140 | return posts, nil 141 | 142 | } 143 | 144 | func GetPostsByFeedId(feedId int64) ([]*Post, error) { 145 | 146 | const query = `SELECT 147 | post_id, 148 | feed_id, 149 | title, 150 | description AS post_description, 151 | link, 152 | updated, 153 | published, 154 | author, 155 | content, 156 | tags, 157 | json_data, 158 | short_id 159 | FROM posts WHERE feed_id = ?` 160 | 161 | var posts []*Post 162 | 163 | err := DB.Handle.Select(&posts, query, feedId) 164 | if err != nil { 165 | return nil, err 166 | } 167 | 168 | return posts, nil 169 | } 170 | 171 | func init() { 172 | _, err := DB.Handle.Exec(DBPostsSchema) 173 | if err != nil { 174 | log.Fatal(err) 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /posts_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/posts" 5 | "log" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestGetPosts(t *testing.T) { 11 | posts, err := posts.ListPosts() 12 | if err != nil { 13 | t.Error(err) 14 | } 15 | log.Println(posts) 16 | for _, p := range posts { 17 | t.Logf("%s <---- %s", p.Title, p.Feed.Name) 18 | } 19 | 20 | } 21 | 22 | func TestMain(m *testing.M) { 23 | code := m.Run() 24 | 25 | defer DB.Handle.Close() 26 | os.Exit(code) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /scheduler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/export" 5 | "git.blob42.xyz/blob42/hugobot/v3/feeds" 6 | "git.blob42.xyz/blob42/hugobot/v3/static" 7 | "git.blob42.xyz/blob42/hugobot/v3/utils" 8 | "errors" 9 | "log" 10 | "math" 11 | "path" 12 | "time" 13 | 14 | gum "git.blob42.xyz/blob42/gum.git" 15 | "github.com/syndtr/goleveldb/leveldb" 16 | 17 | "github.com/beeker1121/goque" 18 | ) 19 | 20 | const ( 21 | StaticDataExportInterval = 3600 * time.Second 22 | JobSchedulerInterval = 60 * time.Second 23 | QDataDir = "./.data" 24 | MaxQueueJobs = 100 25 | MaxSerialJob = 100 26 | 27 | // Used in JobPool to avoid duplicate jobs for the same feed 28 | MapFeedJobFile = "map_feed_job" 29 | ) 30 | 31 | var ( 32 | SourcePriorityRange = Range{Min: 0, Max: 336 * time.Hour.Seconds()} // 2 weeks 33 | TargetPriorityRange = Range{Min: 0, Max: math.MaxUint8} 34 | 35 | SchedulerUpdates = make(chan *Job) 36 | ) 37 | 38 | type Range struct { 39 | Min float64 40 | Max float64 41 | } 42 | 43 | func (r Range) Val() float64 { 44 | return r.Max - r.Min 45 | } 46 | 47 | // JobPool schedluer. Priodically schedule new jobs 48 | type Scheduler struct { 49 | jobs *JobPool 50 | jobUpdates chan *Job 51 | serialJobs chan *Job 52 | } 53 | 54 | func serialRun(inputJobs <-chan *Job) { 55 | for { 56 | j := <-inputJobs 57 | log.Printf("serial run %v", j) 58 | j.Handle() 59 | } 60 | } 61 | 62 | func (s *Scheduler) Run(m gum.UnitManager) { 63 | go serialRun(s.serialJobs) // These jobs run in series 64 | 65 | jobTimer := time.NewTicker(JobSchedulerInterval) 66 | staticExportTimer := time.NewTicker(StaticDataExportInterval) 67 | 68 | for { 69 | select { 70 | case <-jobTimer.C: 71 | log.Println("job heartbeat !") 72 | 73 | j, _ := s.jobs.Peek() 74 | if j != nil { 75 | log.Printf("peeking job: %s\n", j) 76 | } 77 | 78 | // If max pool jobs reached clean the pool 79 | if s.jobs.Length() >= MaxQueueJobs { 80 | s.jobs.Drop() 81 | s.panicAndShutdown(m) 82 | return 83 | } 84 | 85 | s.updateJobs() 86 | 87 | // Spawn job works 88 | s.dispatchJobs() 89 | 90 | case job := <-s.jobUpdates: 91 | log.Printf("job update recieved: %s", JobStatusMap[job.Status]) 92 | 93 | switch job.Status { 94 | 95 | case JobStatusDone: 96 | log.Println("Job is done, removing from feedJobMap. New jobs for this feed can be added now.") 97 | 98 | // Remove job from feedJobMap 99 | err := s.jobs.DeleteMarkedJob(job) 100 | if err != nil { 101 | log.Fatal(err) 102 | } 103 | 104 | // Create export job for successful fetch jobs 105 | if job.JobType == JobTypeFetch { 106 | log.Println("Creating an export job") 107 | expJob, err := NewExportJob(job.Feed, 0) 108 | if err != nil { 109 | log.Fatal(err) 110 | } 111 | 112 | //log.Printf("export job: %+v", expJob) 113 | log.Printf("export job: %s\n", expJob) 114 | 115 | err = s.jobs.Enqueue(expJob) 116 | if err != nil { 117 | log.Fatal(err) 118 | } 119 | } 120 | 121 | case JobStatusFailed: 122 | //TODO: Store all failed jobs somewhere 123 | err := s.jobs.DeleteMarkedJob(job) 124 | if err != nil { 125 | log.Fatal(err) 126 | } 127 | 128 | log.Printf("Job %s failed with error: %s", job.Feed.Name, job.Err) 129 | } 130 | 131 | case <-staticExportTimer.C: 132 | log.Println("-------- export tick --------") 133 | 134 | log.Println("Exporting static data ...") 135 | err := static.HugoExportData() 136 | if err != nil { 137 | log.Fatal(err) 138 | } 139 | 140 | log.Println("Exporting weeks ...") 141 | err = export.ExportWeeks() 142 | if err != nil { 143 | log.Fatal(err) 144 | } 145 | 146 | log.Println("Exporting btc ...") 147 | err = export.ExportBTCAddresses() 148 | if err != nil { 149 | log.Fatal(err) 150 | } 151 | 152 | case <-m.ShouldStop(): 153 | s.Shutdown() 154 | m.Done() 155 | 156 | } 157 | } 158 | } 159 | 160 | func (s *Scheduler) panicAndShutdown(m gum.UnitManager) { 161 | 162 | //err := s.jobs.Drop() 163 | //if err != nil { 164 | //log.Fatal(err) 165 | //} 166 | //TODO 167 | m.Panic(errors.New("max job queue exceeded")) 168 | 169 | s.Shutdown() 170 | 171 | m.Done() 172 | } 173 | 174 | func (s *Scheduler) Shutdown() { 175 | // Flush ongoing jobs back to job queue 176 | 177 | iter := s.jobs.feedJobMap.NewIterator(nil, nil) 178 | var markedDelete [][]byte 179 | for iter.Next() { 180 | key := iter.Key() 181 | log.Printf("Putting job %s back to queue", key) 182 | value := iter.Value() 183 | //log.Println("value ", value) 184 | job, err := JobFromBytes(value) 185 | if err != nil { 186 | log.Fatal(err) 187 | } 188 | 189 | err = s.jobs.Enqueue(job) 190 | if err != nil { 191 | log.Fatal(err) 192 | } 193 | 194 | markedDelete = append(markedDelete, key) 195 | 196 | } 197 | iter.Release() 198 | err := iter.Error() 199 | if err != nil { 200 | log.Fatal(err) 201 | } 202 | 203 | for _, k := range markedDelete { 204 | 205 | err := s.jobs.feedJobMap.Delete(k, nil) 206 | if err != nil { 207 | log.Fatal(err) 208 | } 209 | } 210 | 211 | // Close jobpool queue 212 | err = s.jobs.Close() 213 | if err != nil { 214 | panic(err) 215 | } 216 | 217 | } 218 | 219 | // Dispatchs all jobs in the job pool to task workers 220 | func (s *Scheduler) dispatchJobs() { 221 | //log.Println("dispatching ...") 222 | jobsLength := int(s.jobs.Length()) 223 | for i := 0; i < jobsLength; i++ { 224 | log.Printf("Dequeing %d/%d", i, s.jobs.Length()) 225 | 226 | j, err := s.jobs.Dequeue() 227 | if err != nil { 228 | log.Fatal(err) 229 | } 230 | 231 | log.Println("Dispatching ", j) 232 | 233 | if j.Serial { 234 | s.serialJobs <- j 235 | } else { 236 | go j.Handle() 237 | } 238 | } 239 | 240 | } 241 | 242 | // Gets all available feeds and creates 243 | // a new Job if time.Now() - feed.last_refresh >= feed.interval 244 | func (s *Scheduler) updateJobs() { 245 | // 246 | // Get all feeds 247 | // 248 | // For each feed compare Now() vs last refresh 249 | // If now() - last_refresh >= refresh interval -> create job 250 | 251 | feeds, err := feeds.ListFeeds() 252 | if err != nil { 253 | log.Fatal(err) 254 | } 255 | 256 | //log.Printf("updating jobs for %d feeds\n", len(feeds)) 257 | 258 | // Check all jobs 259 | for _, f := range feeds { 260 | //log.Printf("checking feed %s: %s\n", f.Name, f.Url) 261 | //log.Printf("Seconds since last refresh %f", time.Since(f.LastRefresh.Time).Seconds()) 262 | //log.Printf("Refresh interval %f", f.Interval) 263 | 264 | if delta, ok := f.ShouldRefresh(); ok { 265 | log.Printf("Refreshing %s -- %f seconds since last.", f.Name, delta) 266 | 267 | // If there is already a job with this feed id skip and return an empty job 268 | feedId := utils.IntToBytes(f.FeedID) 269 | 270 | _, err := s.jobs.feedJobMap.Get(feedId, nil) 271 | if err == nil { 272 | log.Println("Job already exists for feed") 273 | } else if err != leveldb.ErrNotFound { 274 | log.Fatal(err) 275 | } else { 276 | 277 | // Priority is based on the delta time since last refresh 278 | // bigger delta == higher priority 279 | 280 | // Convert priority to smaller range priority 281 | // We use original range of `0 - 1 month` in seconds 282 | // Target range is uint8 `0 - 255` 283 | prio := MapPriority(delta) 284 | 285 | job, err := NewFetchJob(f, prio) 286 | if err != nil { 287 | panic(err) 288 | } 289 | 290 | err = s.jobs.Enqueue(job) 291 | if err != nil { 292 | panic(err) 293 | } 294 | 295 | err = s.jobs.MarkUniqJob(job) 296 | if err != nil { 297 | panic(err) 298 | } 299 | 300 | } 301 | 302 | } 303 | 304 | } 305 | 306 | jLen := s.jobs.Length() 307 | if jLen > 0 { 308 | log.Printf("jobs length = %+v\n", jLen) 309 | } 310 | } 311 | 312 | func NewScheduler() gum.WorkUnit { 313 | // Priority queue for jobs 314 | q, err := goque.OpenPriorityQueue(QDataDir, goque.DESC) 315 | if err != nil { 316 | panic(err) 317 | } 318 | 319 | // map[FEED_ID][JOB] 320 | // Used to avoid duplicate jobs in the queue for the same feed 321 | feedJobMapDB, err := leveldb.OpenFile(path.Join(QDataDir, MapFeedJobFile), nil) 322 | if err != nil { 323 | panic(err) 324 | } 325 | 326 | jobPool := &JobPool{ 327 | Q: q, 328 | maxJobs: MaxQueueJobs, 329 | feedJobMap: feedJobMapDB, 330 | } 331 | 332 | // Restore all ongoing jobs in feedJobMap to the pool 333 | iter := feedJobMapDB.NewIterator(nil, nil) 334 | var markedDelete [][]byte 335 | for iter.Next() { 336 | key := iter.Key() 337 | value := iter.Value() 338 | 339 | job, err := JobFromBytes(value) 340 | if err != nil { 341 | log.Fatal(err) 342 | } 343 | 344 | log.Printf("Restoring uncomplete job %s back to job queue", job.ID) 345 | 346 | err = jobPool.Enqueue(job) 347 | if err != nil { 348 | log.Fatal(err) 349 | } 350 | 351 | markedDelete = append(markedDelete, key) 352 | } 353 | 354 | iter.Release() 355 | err = iter.Error() 356 | if err != nil { 357 | log.Fatal(err) 358 | } 359 | 360 | serialJobs := make(chan *Job, MaxSerialJob) 361 | 362 | return &Scheduler{ 363 | 364 | jobs: jobPool, 365 | jobUpdates: SchedulerUpdates, 366 | serialJobs: serialJobs, 367 | } 368 | } 369 | 370 | func NotifyScheduler(job *Job) { 371 | SchedulerUpdates <- job 372 | } 373 | 374 | func MapPriority(val float64) uint8 { 375 | newVal := (((val - SourcePriorityRange.Min) * TargetPriorityRange.Val()) / 376 | SourcePriorityRange.Val()) + TargetPriorityRange.Min 377 | 378 | return uint8(newVal) 379 | } 380 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/db" 5 | "log" 6 | "os" 7 | "syscall" 8 | "time" 9 | 10 | gum "git.blob42.xyz/blob42/gum.git" 11 | ) 12 | 13 | var ( 14 | DB = db.DB 15 | quit chan bool 16 | ) 17 | 18 | func shutdown(c <-chan os.Signal) { 19 | ticker := time.NewTicker(JobSchedulerInterval) 20 | 21 | for { 22 | select { 23 | case <-ticker.C: 24 | log.Println("shutdown goroutine") 25 | 26 | default: 27 | for sig := range c { 28 | switch sig { 29 | 30 | case os.Interrupt: 31 | log.Println("shutting down ... ") 32 | DB.Handle.Close() 33 | quit <- true 34 | 35 | } 36 | 37 | } 38 | } 39 | 40 | } 41 | } 42 | 43 | func server() { 44 | 45 | manager := gum.NewManager() 46 | 47 | manager.ShutdownOn(syscall.SIGINT) 48 | manager.ShutdownOn(syscall.SIGTERM) 49 | 50 | // Jobs scheduler 51 | scheduler := NewScheduler() 52 | manager.AddUnit(scheduler) 53 | 54 | // API 55 | api := NewApi() 56 | manager.AddUnit(api) 57 | 58 | go manager.Run() 59 | 60 | <-manager.Quit 61 | 62 | } 63 | -------------------------------------------------------------------------------- /static/bolts.go: -------------------------------------------------------------------------------- 1 | package static 2 | 3 | var BoltMap = map[int]string{ 4 | 0: "https://github.com/lightningnetwork/lightning-rfc/blob/master/00-introduction.md", 5 | 1: "https://github.com/lightningnetwork/lightning-rfc/blob/master/01-messaging.md", 6 | 2: "https://github.com/lightningnetwork/lightning-rfc/blob/master/02-peer-protocol.md", 7 | 3: "https://github.com/lightningnetwork/lightning-rfc/blob/master/03-transactions.md", 8 | 4: "https://github.com/lightningnetwork/lightning-rfc/blob/master/04-onion-routing.md", 9 | 5: "https://github.com/lightningnetwork/lightning-rfc/blob/master/05-onchain.md", 10 | 7: "https://github.com/lightningnetwork/lightning-rfc/blob/master/07-routing-gossip.md", 11 | 8: "https://github.com/lightningnetwork/lightning-rfc/blob/master/08-transport.md", 12 | 9: "https://github.com/lightningnetwork/lightning-rfc/blob/master/09-features.md", 13 | 10: "https://github.com/lightningnetwork/lightning-rfc/blob/master/10-dns-bootstrap.md", 14 | 11: "https://github.com/lightningnetwork/lightning-rfc/blob/master/11-payment-encoding.md", 15 | } 16 | 17 | var BoltNames = map[int]string{ 18 | 0: "introduction", 19 | 1: "messaging", 20 | 2: "peer protocol", 21 | 3: "transactions", 22 | 4: "onion routing", 23 | 5: "onchain", 24 | 7: "routing gossip", 25 | 8: "transport", 26 | 9: "features", 27 | 10: "dns bootstrap", 28 | 11: "payment encoding", 29 | } 30 | -------------------------------------------------------------------------------- /static/data.go: -------------------------------------------------------------------------------- 1 | package static 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/config" 5 | "encoding/json" 6 | "os" 7 | "path/filepath" 8 | ) 9 | 10 | var data = map[string]interface{}{ 11 | "bolts": map[string]interface{}{ 12 | "names": BoltNames, 13 | }, 14 | } 15 | 16 | // Json Export Static Data 17 | func HugoExportData() error { 18 | dirPath := filepath.Join(config.HugoData()) 19 | for k, v := range data { 20 | filePath := filepath.Join(dirPath, k+".json") 21 | outputFile, err := os.Create(filePath) 22 | defer outputFile.Close() 23 | if err != nil { 24 | return err 25 | } 26 | 27 | jsonEnc := json.NewEncoder(outputFile) 28 | jsonEnc.Encode(v) 29 | } 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /types/json.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "bytes" 5 | "database/sql/driver" 6 | "encoding/json" 7 | "errors" 8 | ) 9 | 10 | type JsonMap map[string]interface{} 11 | 12 | func (m *JsonMap) Scan(src interface{}) error { 13 | 14 | val, ok := src.([]byte) 15 | if !ok { 16 | return errors.New("not []byte") 17 | } 18 | 19 | jsonDecoder := json.NewDecoder(bytes.NewBuffer(val)) 20 | 21 | err := jsonDecoder.Decode(m) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | return nil 27 | } 28 | 29 | func (m JsonMap) Value() (driver.Value, error) { 30 | 31 | val, err := json.Marshal(m) 32 | if err != nil { 33 | return driver.Value(nil), err 34 | } 35 | 36 | return driver.Value(val), nil 37 | 38 | } 39 | -------------------------------------------------------------------------------- /types/stringlist.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "database/sql/driver" 5 | "errors" 6 | "strings" 7 | ) 8 | 9 | type StringList []string 10 | 11 | func (sl *StringList) Scan(src interface{}) error { 12 | 13 | switch v := src.(type) { 14 | case string: 15 | *sl = strings.Split(v, ",") 16 | case []byte: 17 | *sl = strings.Split(string(v), ",") 18 | default: 19 | return errors.New("Could not scan to []string") 20 | } 21 | 22 | return nil 23 | } 24 | 25 | func (sl StringList) Value() (driver.Value, error) { 26 | result := strings.Join(sl, ",") 27 | return driver.Value(result), nil 28 | } 29 | -------------------------------------------------------------------------------- /utils/integers.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "encoding/binary" 4 | 5 | func IntToBytes(x int64) []byte { 6 | buf := make([]byte, 4) 7 | binary.PutVarint(buf, x) 8 | return buf 9 | } 10 | -------------------------------------------------------------------------------- /utils/map.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "git.blob42.xyz/blob42/hugobot/v3/types" 5 | 6 | "github.com/fatih/structs" 7 | ) 8 | 9 | func StructToJsonMap(in interface{}) types.JsonMap { 10 | out := make(types.JsonMap) 11 | 12 | s := structs.New(in) 13 | for _, f := range s.Fields() { 14 | out[f.Tag("json")] = f.Value() 15 | } 16 | 17 | return out 18 | } 19 | -------------------------------------------------------------------------------- /utils/paths.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | const ( 9 | MkdirMode = 0770 10 | ) 11 | 12 | func Mkdir(path ...string) error { 13 | joined := filepath.Join(path...) 14 | 15 | return os.MkdirAll(joined, MkdirMode) 16 | } 17 | -------------------------------------------------------------------------------- /utils/print.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | ) 7 | 8 | func PrettyPrint(v interface{}) (err error) { 9 | b, err := json.MarshalIndent(v, "", " ") 10 | if err == nil { 11 | fmt.Println(string(b)) 12 | } 13 | return 14 | } 15 | -------------------------------------------------------------------------------- /utils/shortid.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/teris-io/shortid" 5 | ) 6 | 7 | const ( 8 | Seed = 322124 9 | ) 10 | 11 | var ( 12 | sid *shortid.Shortid 13 | ) 14 | 15 | func GetSIDGenerator() *shortid.Shortid { 16 | return sid 17 | } 18 | 19 | func init() { 20 | var err error 21 | sid, err = shortid.New(1, shortid.DefaultABC, Seed) 22 | if err != nil { 23 | panic(err) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /utils/time.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "log" 5 | "time" 6 | ) 7 | 8 | func NextThursday(t time.Time) time.Time { 9 | weekday := t.Weekday() 10 | 11 | t, err := time.Parse("2006-01-02", t.Format("2006-01-02")) 12 | if err != nil { 13 | log.Println(err) 14 | } 15 | nextThursday := t 16 | 17 | if weekday < 4 { 18 | nextThursday = t.AddDate(0, 0, int(4-weekday)) 19 | } else if weekday > 4 { 20 | nextThursday = t.AddDate(0, 0, int((7-weekday)+4)) 21 | } 22 | 23 | return nextThursday 24 | } 25 | 26 | // Returns all thursdays starting from now up to the input date 27 | func GetAllThursdays(from time.Time, to time.Time) []time.Time { 28 | var dates []time.Time 29 | 30 | //log.Printf("Parsing from %s", from) 31 | 32 | firstWeek := NextThursday(from) 33 | lastWeek := NextThursday(to) 34 | 35 | //log.Printf("First thursday is %s", firstWeek) 36 | 37 | cursorWeek := firstWeek 38 | for cursorWeek.Before(lastWeek) { 39 | dates = append(dates, cursorWeek) 40 | cursorWeek = cursorWeek.AddDate(0, 0, 7) 41 | } 42 | 43 | if !cursorWeek.Before(lastWeek) && 44 | cursorWeek.Weekday() == time.Thursday { 45 | dates = append(dates, cursorWeek) 46 | } 47 | 48 | return dates 49 | } 50 | -------------------------------------------------------------------------------- /utils/time_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestGetAllThursdays(t *testing.T) { 9 | tt, err := time.Parse("2006-01-02", "2017-12-07") 10 | if err != nil { 11 | t.Error(err) 12 | } 13 | 14 | dates := GetAllThursdays(tt, time.Now()) 15 | 16 | if dates[0] != NextThursday(tt) { 17 | t.Error("starting date") 18 | } 19 | 20 | t.Log(NextThursday(time.Now())) 21 | t.Log(dates[len(dates)-1]) 22 | 23 | if dates[len(dates)-1] != NextThursday(time.Now()) { 24 | t.Error("end date") 25 | } 26 | } 27 | --------------------------------------------------------------------------------