├── .gitattributes ├── .gitignore ├── .goreleaser.yml ├── LICENSE ├── README.md ├── cmd └── cli │ └── main.go └── internal ├── app └── cli │ ├── cli.go │ ├── commands │ ├── root.go │ ├── search.go │ ├── update.go │ └── version.go │ ├── log │ └── log.go │ └── terminal │ └── terminal.go └── pkg ├── config └── config.go ├── connector ├── api.go ├── connector.go └── svn.go ├── context └── context.go ├── search ├── misc.go ├── misc_test.go ├── results.go └── search.go ├── slurper ├── archive.go ├── client.go ├── extensions.go ├── failed.go ├── results.go ├── revisions.go ├── slurper.go └── svn.go ├── stats └── stats.go └── utils ├── filesystem.go ├── utils.go └── utils_test.go /.gitattributes: -------------------------------------------------------------------------------- 1 | testdata/* linguist-vendored -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Pprof related files 11 | *.prof 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # IDE Related 17 | /.idea/ 18 | 19 | # Ignore goreleaser related files folder 20 | /dist/ 21 | releasenotes.md 22 | 23 | # WordPress Slurper Created Files 24 | /plugins/ 25 | /themes/ 26 | /readmes/ 27 | /searches/ 28 | .last-revision 29 | .last-revision-plugins 30 | .last-revision-themes -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: wpds 2 | 3 | builds: 4 | 5 | - 6 | 7 | main: ./cmd/cli/main.go 8 | 9 | binary: wpds 10 | 11 | env: 12 | - CGO_ENABLED=0 13 | 14 | goos: 15 | - windows 16 | - darwin 17 | - linux 18 | 19 | goarch: 20 | - amd64 21 | - arm 22 | - arm64 23 | - 386 24 | 25 | goarm: 26 | - 6 27 | - 7 28 | 29 | ignore: 30 | - goos: darwin 31 | goarch: 386 32 | 33 | archive: 34 | 35 | name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" 36 | 37 | format_overrides: 38 | - goos: windows 39 | format: zip 40 | 41 | replacements: 42 | amd64: 64-bit 43 | 386: 32-bit 44 | darwin: macOS 45 | 46 | files: 47 | - README.md 48 | - LICENSE 49 | 50 | checksum: 51 | 52 | name_template: "{{ .ProjectName }}_checksums.txt" 53 | 54 | release: 55 | 56 | draft: true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | {description} 294 | Copyright (C) {year} {fullname} 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Release](https://img.shields.io/badge/version-v0.4.0-blue.svg)](https://github.com/PeterBooker/WordPress-Directory-Slurper/releases/tag/v0.4.0) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/PeterBooker/WordPress-Directory-Slurper)](https://goreportcard.com/report/github.com/PeterBooker/WordPress-Directory-Slurper) 3 | [![License](https://img.shields.io/badge/license-GPL--2.0%2B-red.svg)](https://github.com/PeterBooker/WordPress-Directory-Slurper/blob/master/LICENSE) 4 | 5 | # WPDS (WordPress Directory Slurper) 6 | WPDS is a cross-platform CLI tool built with [Go](https://golang.org/). Slurps down the latest version of every Plugin and/or Theme in the WordPress Directory. Comes with inbuilt searching and formatted search summaries. Based on the Plugin and Theme Directory Slurpers by [markjaquith](https://github.com/markjaquith/WordPress-Plugin-Directory-Slurper), [clorith](https://github.com/Clorith/WordPress-Plugin-Directory-Slurper), [ipstenu](https://github.com/Ipstenu/WordPress-Theme-Directory-Slurper) and [chriscct7](https://github.com/chriscct7/WordPress-Plugin-Directory-Slurper/). 7 | 8 | Note: WPDS is still in early development and therefore may contain bugs or miss features. 9 | 10 | ## Dependencies 11 | 12 | None. WPDS is a self-contained executable. 13 | 14 | ## Install 15 | 16 | Download the relevant file for your operating system from the [releases](https://github.com/PeterBooker/WordPress-Directory-Slurper/releases) page, then either run it from the directory you want it to work in or put it into your PATH and it will use the current working directory. 17 | 18 | ## Examples 19 | 20 | ### Slurp Plugin Directory 21 | 22 | ``` 23 | wpds update plugins 24 | ``` 25 | 26 | This will start a new slurp or continue the existing slurp using the latest revision found in `/plugins/.last-revision`. 27 | 28 | ## Features 29 | 30 | - [x] Download the Plugin Directory 31 | - [x] Update the Plugin Directory files 32 | - [x] Download the Theme Directory 33 | - [x] Update the Theme Directory files 34 | - [x] In-built Searching 35 | - [ ] Search Summary Generation 36 | 37 | ## FAQ 38 | 39 | ### Why did you remake the previous tools in Go? 40 | 41 | Building the CLI tool in Go removes any requirements and provides full cross platform support, making it easier for everyone to use. 42 | 43 | It also allowed me to build the search functionality into the tool, removing further requirements. 44 | 45 | ### Why download the zip files? Why not use SVN? 46 | 47 | An SVN checkout of the entire repository is a BEAST of a thing. You don't want it, 48 | trust me. Updates and cleanups can take **hours** or even **days** to complete. 49 | 50 | ### How long will it take? 51 | 52 | Your first update will take a while (at last testing around 2-3 hours using the default settings). 53 | 54 | But subsequent updates are smarter. The script tracks the SVN revision number of your latest update and then asks the plugins SVN repository for a list of plugins that have changed since. Only those changed plugins are updated after the initial sync. 55 | 56 | ### How much disk space do I need? 57 | 58 | As of April 2018: 59 | 60 | 73,629 Plugins- extracting 518,453 folders, 2,690,570 files, taking up 40.7 GB of space. 61 | 62 | ### Something went wrong, how do I do a partial update? 63 | 64 | The last successful update revision number is stored in `plugins/.last-revision`. 65 | You can just overwrite that and the next `wpds update plugins` will start after that revision. 66 | 67 | ### What is this thing actually doing to my computer? 68 | 69 | Once downloads have started, the CLI tool will display its progress including how many of the themes/plugins it has downloaded out of the total and an estimated completion time. 70 | 71 | ## License 72 | 73 | GNU General Public License 2.0, see [LICENSE](https://github.com/PeterBooker/WordPress-Directory-Slurper/blob/master/LICENSE). -------------------------------------------------------------------------------- /cmd/cli/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/peterbooker/wpds/internal/app/cli" 5 | ) 6 | 7 | func main() { 8 | 9 | // Setup and Run CLI 10 | cli.Execute() 11 | 12 | } 13 | -------------------------------------------------------------------------------- /internal/app/cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/peterbooker/wpds/internal/app/cli/commands" 8 | ) 9 | 10 | // Execute ... 11 | func Execute() { 12 | 13 | if err := commands.Execute(); err != nil { 14 | fmt.Println(err) 15 | os.Exit(1) 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /internal/app/cli/commands/root.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "os" 5 | "runtime/pprof" 6 | 7 | "github.com/peterbooker/wpds/internal/app/cli/log" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | var ( 12 | CPUProf string 13 | MemProf string 14 | C int 15 | V bool 16 | F string 17 | L string 18 | ) 19 | 20 | var rootCmd = &cobra.Command{ 21 | Use: "wpds", 22 | Short: "WPDS is a tool for downloading and searching the WordPress Plugin/Theme Directories.", 23 | Long: `WPDS is a tool for downloading and searching the WordPress Plugin/Theme Directories.`, 24 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 25 | 26 | // Start Memory Profile 27 | if MemProf != "" { 28 | f, err := os.Create("start_" + MemProf) 29 | if err != nil { 30 | panic(err) 31 | } 32 | pprof.WriteHeapProfile(f) 33 | f.Close() 34 | return 35 | } 36 | 37 | // Setup the global logger using flag values 38 | // Runs before every command 39 | log.Setup(V, L) 40 | 41 | }, 42 | PersistentPostRun: func(cmd *cobra.Command, args []string) { 43 | 44 | // End Memory Profile 45 | if MemProf != "" { 46 | f, err := os.Create("end_" + MemProf) 47 | if err != nil { 48 | panic(err) 49 | } 50 | pprof.WriteHeapProfile(f) 51 | f.Close() 52 | return 53 | } 54 | 55 | }, 56 | } 57 | 58 | func init() { 59 | 60 | // Debug / Profiling Flags 61 | rootCmd.PersistentFlags().StringVar(&CPUProf, "cpuprof", "", "Filename of CPU profiling file.") 62 | rootCmd.PersistentFlags().StringVar(&MemProf, "memprof", "", "Filename of Memory profiling file.") 63 | 64 | // General Flags 65 | rootCmd.PersistentFlags().IntVarP(&C, "concurrent-actions", "c", 50, "Maximum number of concurrent actions (valid between 10-1000).") 66 | rootCmd.PersistentFlags().StringVarP(&L, "log", "l", "", "Destination of file to log output to.") 67 | rootCmd.PersistentFlags().BoolVarP(&V, "verbose", "v", false, "Verbose mode changed output from log to stdout.") 68 | rootCmd.PersistentFlags().StringVarP(&F, "files", "f", "all", "Extension files to download e.g. all or readme.") 69 | 70 | } 71 | 72 | // Execute processes CLI commands 73 | func Execute() error { 74 | return rootCmd.Execute() 75 | } 76 | -------------------------------------------------------------------------------- /internal/app/cli/commands/search.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/peterbooker/wpds/internal/pkg/search" 8 | "github.com/peterbooker/wpds/internal/pkg/stats" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | func init() { 13 | rootCmd.AddCommand(searchCmd) 14 | searchCmd.AddCommand(pluginsSearchCmd) 15 | searchCmd.AddCommand(themesSearchCmd) 16 | } 17 | 18 | var searchCmd = &cobra.Command{ 19 | Use: "search", 20 | Short: "Search files downloaded from the WordPress Directory.", 21 | Long: `Search files downloaded from the WordPress Directory.`, 22 | } 23 | 24 | var pluginsSearchCmd = &cobra.Command{ 25 | Use: "plugins", 26 | Short: "Search the Plugin files downloaded from the WordPress Directory.", 27 | Long: `Search the Plugin files downloaded from the WordPress Directory.`, 28 | Args: cobra.ExactArgs(1), 29 | Run: func(cmd *cobra.Command, args []string) { 30 | 31 | // Search Input 32 | input := args[0] 33 | 34 | // Get Working Directory 35 | wd, _ := os.Getwd() 36 | 37 | // Create new Stats 38 | stats := stats.New() 39 | 40 | // Setup Whitelist 41 | whitelist := []string{} 42 | 43 | // Setup Context 44 | ctx := &search.Context{ 45 | ExtensionType: cmd.Use, 46 | FileType: F, 47 | ExtWhitelist: whitelist, 48 | WorkingDirectory: wd, 49 | Stats: stats, 50 | } 51 | 52 | log.Println("Search All Plugin files...") 53 | 54 | s := search.Setup(input, ctx) 55 | 56 | err := s.Run() 57 | if err != nil { 58 | log.Fatal(err) 59 | } 60 | 61 | }, 62 | } 63 | 64 | var themesSearchCmd = &cobra.Command{ 65 | Use: "themes", 66 | Short: "Search the Theme files downloaded from the WordPress Directory.", 67 | Long: `Search the Theme files downloaded from the WordPress Directory.`, 68 | Args: cobra.ExactArgs(1), 69 | Run: func(cmd *cobra.Command, args []string) { 70 | 71 | // Search Input 72 | input := args[0] 73 | 74 | // Get Working Directory 75 | wd, _ := os.Getwd() 76 | 77 | // Create new Stats 78 | stats := stats.New() 79 | 80 | // Setup Whitelist 81 | whitelist := []string{} 82 | 83 | // Setup Context 84 | ctx := &search.Context{ 85 | ExtensionType: cmd.Use, 86 | FileType: F, 87 | ExtWhitelist: whitelist, 88 | WorkingDirectory: wd, 89 | Stats: stats, 90 | } 91 | 92 | log.Println("Search All Theme files...") 93 | 94 | s := search.Setup(input, ctx) 95 | 96 | err := s.Run() 97 | if err != nil { 98 | log.Fatal(err) 99 | } 100 | 101 | }, 102 | } 103 | -------------------------------------------------------------------------------- /internal/app/cli/commands/update.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "runtime/pprof" 7 | 8 | "github.com/peterbooker/wpds/internal/pkg/config" 9 | "github.com/peterbooker/wpds/internal/pkg/context" 10 | "github.com/peterbooker/wpds/internal/pkg/slurper" 11 | "github.com/peterbooker/wpds/internal/pkg/stats" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | func init() { 16 | rootCmd.AddCommand(updateCmd) 17 | updateCmd.AddCommand(pluginsUpdateCmd) 18 | updateCmd.AddCommand(themesUpdateCmd) 19 | } 20 | 21 | var updateCmd = &cobra.Command{ 22 | Use: "update", 23 | Short: "Update files from the WordPress Directory.", 24 | Long: `Update Plugin or Theme files from their WordPress Directory.`, 25 | } 26 | 27 | var pluginsUpdateCmd = &cobra.Command{ 28 | Use: "plugins", 29 | Short: "Update Plugin files.", 30 | Long: ``, 31 | Example: `wpds update plugins -c 250`, 32 | Run: func(cmd *cobra.Command, args []string) { 33 | 34 | if CPUProf != "" { 35 | f, err := os.Create(CPUProf) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | pprof.StartCPUProfile(f) 40 | defer pprof.StopCPUProfile() 41 | } 42 | 43 | if (C < 10) || (C > 10000) { 44 | log.Printf("Flag (concurrent-actions, c) out of permitted range (10-10000).\n") 45 | os.Exit(1) 46 | } 47 | 48 | // Get Config Details 49 | name := config.GetName() 50 | version := config.GetVersion() 51 | 52 | // Check if SVN is installed 53 | // Used if available, as it is more reliable than the HTTP API 54 | connector := "api" 55 | if slurper.CheckForSVN() { 56 | connector = "svn" 57 | } 58 | 59 | // Get Working Directory 60 | wd, _ := os.Getwd() 61 | 62 | // Create new Stats 63 | stats := stats.New() 64 | 65 | ctx := &context.Context{ 66 | Name: name, 67 | Version: version, 68 | ConcurrentActions: C, 69 | ExtensionType: "plugins", 70 | FileType: F, 71 | Connector: connector, 72 | CurrentRevision: 0, 73 | LatestRevision: 0, 74 | WorkingDirectory: wd, 75 | Stats: stats, 76 | } 77 | 78 | log.Println("Updating Plugins...") 79 | 80 | slurper.StartUpdate(ctx) 81 | 82 | }, 83 | } 84 | 85 | var themesUpdateCmd = &cobra.Command{ 86 | Use: "themes", 87 | Short: "Update Theme files.", 88 | Long: ``, 89 | Example: `wpds update themes -c 250`, 90 | Run: func(cmd *cobra.Command, args []string) { 91 | 92 | if CPUProf != "" { 93 | f, err := os.Create(CPUProf) 94 | if err != nil { 95 | log.Fatal(err) 96 | } 97 | pprof.StartCPUProfile(f) 98 | defer pprof.StopCPUProfile() 99 | } 100 | 101 | if (C < 10) || (C > 10000) { 102 | log.Printf("Flag (concurrent-actions, c) out of permitted range (10-1000).\n") 103 | os.Exit(1) 104 | } 105 | 106 | // Get Config Details 107 | name := config.GetName() 108 | version := config.GetVersion() 109 | 110 | // Check if SVN is installed 111 | // Used if available, as it is more reliable than the HTTP API 112 | connector := "api" 113 | if slurper.CheckForSVN() { 114 | connector = "svn" 115 | } 116 | 117 | // Get Working Directory 118 | wd, _ := os.Getwd() 119 | 120 | // Create new Stats 121 | stats := stats.New() 122 | 123 | ctx := &context.Context{ 124 | Name: name, 125 | Version: version, 126 | ConcurrentActions: C, 127 | ExtensionType: "themes", 128 | FileType: F, 129 | Connector: connector, 130 | CurrentRevision: 0, 131 | LatestRevision: 0, 132 | WorkingDirectory: wd, 133 | Stats: stats, 134 | } 135 | 136 | log.Println("Updating Themes...") 137 | 138 | slurper.StartUpdate(ctx) 139 | 140 | }, 141 | } 142 | -------------------------------------------------------------------------------- /internal/app/cli/commands/version.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/peterbooker/wpds/internal/pkg/config" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | func init() { 11 | rootCmd.AddCommand(versionCmd) 12 | } 13 | 14 | var versionCmd = &cobra.Command{ 15 | 16 | Use: "version", 17 | Short: "Print the version number of WPDS", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | 20 | command := cmd.Use 21 | name := config.GetName() 22 | version := config.GetVersion() 23 | 24 | fmt.Printf("%s %s %s\n", name, command, version) 25 | 26 | }, 27 | } 28 | -------------------------------------------------------------------------------- /internal/app/cli/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | ) 10 | 11 | const ( 12 | // DefaultLogFilename is the default log filename. 13 | DefaultLogFilename = "debug.log" 14 | ) 15 | 16 | func init() { 17 | 18 | // Default logger is silent, logs to ioutil.Discard 19 | log.SetOutput(ioutil.Discard) 20 | log.SetPrefix("") 21 | log.SetFlags(log.Ldate | log.Ltime) 22 | 23 | } 24 | 25 | // Setup configures the global logger 26 | func Setup(verboseFlag bool, logFlag string) { 27 | 28 | // If logFlag is not empty then log to the file specified. 29 | if logFlag != "" { 30 | 31 | logFile, err := os.OpenFile(logFlag, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666) 32 | if err != nil { 33 | fmt.Printf("Error: Could not open log file: %s\n.", logFlag) 34 | os.Exit(1) 35 | } 36 | 37 | log.SetOutput(logFile) 38 | 39 | } 40 | 41 | // If verboseFlag is true then log to Stdout. 42 | if verboseFlag { 43 | 44 | log.SetOutput(os.Stdout) 45 | 46 | } 47 | 48 | // If logFlag is not empty and verboseFlag is true then log to both logFile and Stdout. 49 | if logFlag != "" && verboseFlag { 50 | 51 | logFile, err := os.OpenFile(logFlag, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666) 52 | if err != nil { 53 | fmt.Printf("Error: Could not open log file: %s\n.", logFlag) 54 | os.Exit(1) 55 | } 56 | 57 | mw := io.MultiWriter(os.Stdout, logFile) 58 | log.SetOutput(mw) 59 | 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /internal/app/cli/terminal/terminal.go: -------------------------------------------------------------------------------- 1 | package terminal 2 | 3 | import () 4 | -------------------------------------------------------------------------------- /internal/pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | const ( 4 | name = "WPDS" 5 | version = "0.7.0" 6 | ) 7 | 8 | // GetName returns the CLI Name 9 | func GetName() string { 10 | 11 | return name 12 | 13 | } 14 | 15 | // GetVersion returns the CLI Version 16 | func GetVersion() string { 17 | 18 | return version 19 | 20 | } 21 | -------------------------------------------------------------------------------- /internal/pkg/connector/api.go: -------------------------------------------------------------------------------- 1 | package connector 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "regexp" 8 | "strconv" 9 | "time" 10 | 11 | "github.com/peterbooker/wpds/internal/pkg/context" 12 | ) 13 | 14 | const ( 15 | wpAllPluginsListURL = "http://plugins.svn.wordpress.org/" 16 | wpAllThemesListURL = "http://themes.svn.wordpress.org/" 17 | wpLatestPluginsRevisionURL = "http://plugins.trac.wordpress.org/log/?format=changelog&stop_rev=HEAD" 18 | wpLatestThemesRevisionURL = "http://themes.trac.wordpress.org/log/?format=changelog&stop_rev=HEAD" 19 | wpPluginChangelogURL = "https://plugins.trac.wordpress.org/log/?verbose=on&mode=follow_copy&format=changelog&rev=%d&limit=%d" 20 | wpThemeChangelogURL = "https://themes.trac.wordpress.org/log/?verbose=on&mode=follow_copy&format=changelog&rev=%d&limit=%d" 21 | ) 22 | 23 | var ( 24 | regexAPILatestRevision = regexp.MustCompile(`\[(\d*)\]`) 25 | regexAPIFullExtensionsList = regexp.MustCompile(`.+?\>(\S+?)\/\<`) 26 | ) 27 | 28 | // API implements the DirectoryConnector inferface. 29 | // It uses an HTTP API to communicate with the WordPress Directory SVN Repositories. 30 | type API struct { 31 | currentRevision int 32 | latestRevision int 33 | extensions []string 34 | } 35 | 36 | // GetLatestRevision gets the latest revision of the target directory. 37 | func (api API) GetLatestRevision(ctx *context.Context) (int, error) { 38 | 39 | var URL string 40 | 41 | client := &http.Client{ 42 | Timeout: 30 * time.Second, 43 | } 44 | 45 | URL = fmt.Sprintf("http://%s.trac.wordpress.org/log/?format=changelog&stop_rev=HEAD", ctx.ExtensionType) 46 | 47 | req, err := http.NewRequest("GET", URL, nil) 48 | if err != nil { 49 | return 0, err 50 | } 51 | 52 | // Set User Agent e.g. wpds/1.1.3 53 | userAgent := fmt.Sprintf("%s/%s", ctx.Name, ctx.Version) 54 | req.Header.Set("User-Agent", userAgent) 55 | 56 | resp, err := client.Do(req) 57 | if err != nil { 58 | return 0, err 59 | } 60 | 61 | if resp.StatusCode != 200 { 62 | return 0, fmt.Errorf("Invalid HTTP response code: %d", resp.StatusCode) 63 | } 64 | 65 | defer resp.Body.Close() 66 | bBytes, err := ioutil.ReadAll(resp.Body) 67 | bString := string(bBytes) 68 | 69 | revs := regexAPILatestRevision.FindAllStringSubmatch(bString, 1) 70 | 71 | api.latestRevision, err = strconv.Atoi(revs[0][1]) 72 | if err != nil { 73 | return 0, err 74 | } 75 | 76 | return api.latestRevision, nil 77 | 78 | } 79 | 80 | // GetFullExtensionsList gets the fill list of all Extensions. 81 | func (api *API) GetFullExtensionsList(ctx *context.Context) ([]string, error) { 82 | 83 | client := &http.Client{ 84 | Timeout: 60 * time.Second, 85 | } 86 | 87 | URL := fmt.Sprintf("https://%s.svn.wordpress.org/", ctx.ExtensionType) 88 | 89 | req, err := http.NewRequest("GET", URL, nil) 90 | if err != nil { 91 | return api.extensions, err 92 | } 93 | 94 | // Set User Agent e.g. wpds/1.1.3 95 | userAgent := fmt.Sprintf("%s/%s", ctx.Name, ctx.Version) 96 | req.Header.Set("User-Agent", userAgent) 97 | 98 | resp, err := client.Do(req) 99 | if err != nil { 100 | return api.extensions, err 101 | } 102 | 103 | if resp.StatusCode != 200 { 104 | return api.extensions, fmt.Errorf("Invalid HTTP response code: %d", resp.StatusCode) 105 | } 106 | 107 | defer resp.Body.Close() 108 | bBytes, err := ioutil.ReadAll(resp.Body) 109 | bString := string(bBytes) 110 | 111 | matches := regexAPIFullExtensionsList.FindAllStringSubmatch(bString, -1) 112 | 113 | // Add all matches to extension list 114 | for _, extension := range matches { 115 | 116 | api.extensions = append(api.extensions, extension[1]) 117 | 118 | } 119 | 120 | return api.extensions, nil 121 | 122 | } 123 | 124 | // GetUpdatedExtensionsList an updated list of Extensions. 125 | func (api *API) GetUpdatedExtensionsList(ctx *context.Context) ([]string, error) { 126 | 127 | client := &http.Client{ 128 | Timeout: 60 * time.Second, 129 | } 130 | 131 | URL := fmt.Sprintf("https://%s.trac.wordpress.org/log/?verbose=on&mode=follow_copy&format=changelog&rev=%d&limit=%d", ctx.ExtensionType, ctx.CurrentRevision, ctx.LatestRevision) 132 | 133 | req, err := http.NewRequest("GET", URL, nil) 134 | if err != nil { 135 | return api.extensions, err 136 | } 137 | 138 | // Set User Agent e.g. wpds/1.1.3 139 | userAgent := fmt.Sprintf("%s/%s", ctx.Name, ctx.Version) 140 | req.Header.Set("User-Agent", userAgent) 141 | 142 | resp, err := client.Do(req) 143 | if err != nil { 144 | return api.extensions, err 145 | } 146 | 147 | if resp.StatusCode != 200 { 148 | return api.extensions, fmt.Errorf("Invalid HTTP response code: %d", resp.StatusCode) 149 | } 150 | 151 | defer resp.Body.Close() 152 | bBytes, err := ioutil.ReadAll(resp.Body) 153 | bString := string(bBytes) 154 | 155 | matches := regexAPIFullExtensionsList.FindAllStringSubmatch(bString, 1) 156 | 157 | found := make(map[string]bool) 158 | 159 | // Get the desired substring match and remove duplicates 160 | for _, extension := range matches { 161 | 162 | if !found[extension[1]] { 163 | found[extension[1]] = true 164 | api.extensions = append(api.extensions, extension[1]) 165 | } 166 | 167 | } 168 | 169 | return api.extensions, nil 170 | 171 | } 172 | -------------------------------------------------------------------------------- /internal/pkg/connector/connector.go: -------------------------------------------------------------------------------- 1 | package connector 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/peterbooker/wpds/internal/pkg/context" 9 | ) 10 | 11 | // DirectoryConnector implements the methods required to interact with the WordPress Directories 12 | // Implemented via external HTTP API and local SVN client 13 | type DirectoryConnector interface { 14 | GetLatestRevision(ctx *context.Context) (int, error) 15 | GetFullExtensionsList(ctx *context.Context) ([]string, error) 16 | GetUpdatedExtensionsList(ctx *context.Context) ([]string, error) 17 | } 18 | 19 | // Init returns a connector used to communicate with the WordPress Directory SVN repositories. 20 | // Implemented via an external HTTP API or local SVN client 21 | func Init(cType string) DirectoryConnector { 22 | 23 | switch cType { 24 | 25 | case "svn": 26 | return &SVN{} 27 | 28 | case "api": 29 | return &API{} 30 | 31 | default: 32 | // No supported connector found. 33 | fmt.Printf("The defined connector '%s' is not supported.", strings.ToUpper(cType)) 34 | os.Exit(1) 35 | 36 | } 37 | 38 | return nil 39 | 40 | } 41 | -------------------------------------------------------------------------------- /internal/pkg/connector/svn.go: -------------------------------------------------------------------------------- 1 | package connector 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "regexp" 7 | "strconv" 8 | 9 | "github.com/peterbooker/wpds/internal/pkg/context" 10 | ) 11 | 12 | var ( 13 | regexSVNUpdatedExtensionsList = regexp.MustCompile(`.{1,} \/(.+?)\/`) 14 | regexSVNFullExtensionsList = regexp.MustCompile(`(.+?)\/`) 15 | regexSVNLatestRevision = regexp.MustCompile(`r([0-9]+)`) 16 | ) 17 | 18 | // SVN implements the DirectoryConnector inferface. 19 | // It uses a local SVN client to communicate with the WordPress Directory SVN Repositories. 20 | type SVN struct { 21 | currentRevision int 22 | latestRevision int 23 | extensions []string 24 | } 25 | 26 | // GetLatestRevision gets the latest revision of the target directory. 27 | func (svn *SVN) GetLatestRevision(ctx *context.Context) (int, error) { 28 | 29 | URL := fmt.Sprintf("https://%s.svn.wordpress.org/", ctx.ExtensionType) 30 | 31 | args := []string{"log", "-v", "-q", URL, "-r", "HEAD"} 32 | 33 | out, _ := exec.Command("svn", args...).Output() 34 | 35 | matches := regexSVNLatestRevision.FindAllStringSubmatch(string(out), -1) 36 | 37 | var err error 38 | 39 | svn.latestRevision, err = strconv.Atoi(matches[0][1]) 40 | if err != nil { 41 | return 0, err 42 | } 43 | 44 | return svn.latestRevision, nil 45 | 46 | } 47 | 48 | // GetFullExtensionsList gets the fill list of all Extensions. 49 | func (svn *SVN) GetFullExtensionsList(ctx *context.Context) ([]string, error) { 50 | 51 | URL := fmt.Sprintf("https://%s.svn.wordpress.org/", ctx.ExtensionType) 52 | 53 | args := []string{"list", URL} 54 | 55 | out, _ := exec.Command("svn", args...).Output() 56 | 57 | matches := regexSVNFullExtensionsList.FindAllStringSubmatch(string(out), -1) 58 | 59 | // Add all matches to extension list 60 | for _, extension := range matches { 61 | 62 | svn.extensions = append(svn.extensions, extension[1]) 63 | 64 | } 65 | 66 | return svn.extensions, nil 67 | 68 | } 69 | 70 | // GetUpdatedExtensionsList gets an updated list of Extensions. 71 | func (svn *SVN) GetUpdatedExtensionsList(ctx *context.Context) ([]string, error) { 72 | 73 | diff := fmt.Sprintf("%d:%d", ctx.CurrentRevision, ctx.LatestRevision) 74 | 75 | URL := fmt.Sprintf("https://%s.svn.wordpress.org/", ctx.ExtensionType) 76 | 77 | args := []string{"log", "-v", "-q", URL, "-r", diff} 78 | 79 | out, _ := exec.Command("svn", args...).Output() 80 | 81 | groups := regexSVNUpdatedExtensionsList.FindAllStringSubmatch(string(out), -1) 82 | 83 | found := make(map[string]bool) 84 | 85 | // Get the desired substring match and remove duplicates 86 | for _, extension := range groups { 87 | 88 | if !found[extension[1]] { 89 | found[extension[1]] = true 90 | svn.extensions = append(svn.extensions, extension[1]) 91 | } 92 | 93 | } 94 | 95 | return svn.extensions, nil 96 | 97 | } 98 | -------------------------------------------------------------------------------- /internal/pkg/context/context.go: -------------------------------------------------------------------------------- 1 | package context 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/peterbooker/wpds/internal/pkg/stats" 7 | ) 8 | 9 | // Context contains the data required for Slurping 10 | type Context struct { 11 | Name string 12 | Version string 13 | ConcurrentActions int 14 | ExtensionType string 15 | FileType string 16 | Connector string 17 | CurrentRevision int 18 | LatestRevision int 19 | WorkingDirectory string 20 | FailedList []string 21 | Stats *stats.Stats 22 | Client *http.Client 23 | } 24 | 25 | // SearchContext contains data required for Searching 26 | type SearchContext struct { 27 | ExtensionType string 28 | FileType string 29 | ExtWhitelist []string 30 | WorkingDirectory string 31 | Stats *stats.Stats 32 | } 33 | -------------------------------------------------------------------------------- /internal/pkg/search/misc.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import ( 4 | "bytes" 5 | "unicode/utf8" 6 | ) 7 | 8 | const maxLen = 512 9 | 10 | // isUTF8 checks upto the first 512 bytes for valid UTF8 encoding. 11 | func isUTF8(data []byte) bool { 12 | if len(data) > maxLen { 13 | data = data[:maxLen] 14 | } 15 | return utf8.Valid(data) 16 | } 17 | 18 | // getLineNum counts the lines in data. 19 | // TODO: Check this is always accurate. 20 | func getLineNum(data []byte) int { 21 | return 1 + bytes.Count(data, []byte("\n")) 22 | } 23 | 24 | // getMatchLines finds the index for the start of the first line and the end of the last line. 25 | func getMatchLineIndexes(data []byte, match []int) (int, int) { 26 | start := getStartIndex(data, match[0]) 27 | end := getEndIndex(data, match[1]) 28 | return start, end 29 | } 30 | 31 | // getStartIndex begins from the start of the match and finds the start of that line. 32 | func getStartIndex(data []byte, match int) int { 33 | start := match 34 | if match > 0 { 35 | 36 | for i := match - 1; i > 0; i-- { 37 | if data[i] == '\n' { 38 | start = i + 1 39 | break 40 | } 41 | } 42 | 43 | } 44 | return start 45 | } 46 | 47 | // getEndIndex begins from the end of the match and finds the end of that line. 48 | func getEndIndex(data []byte, match int) int { 49 | end := match 50 | max := len(data) 51 | if match < max { 52 | 53 | for i := match + 1; i <= max; i++ { 54 | if i == max { 55 | end = i 56 | break 57 | } 58 | if data[i] == '\n' || data[i] == '\r' { 59 | end = i - 1 60 | break 61 | } 62 | } 63 | 64 | } 65 | return end 66 | } 67 | -------------------------------------------------------------------------------- /internal/pkg/search/misc_test.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import "testing" 4 | 5 | func TestGetLineNum(t *testing.T) { 6 | var num int 7 | var data []byte 8 | data = []byte("first line\r\nsecond line\nthird line") 9 | num = getLineNum(data) 10 | if num != 3 { 11 | t.Error("Expected 3, got ", num) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /internal/pkg/search/results.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/dustin/go-humanize" 7 | ) 8 | 9 | func printSummary(ctx *Context) { 10 | 11 | switch ctx.ExtensionType { 12 | case "plugins": 13 | fmt.Printf("\n== Plugin Search Summary ==\n") 14 | case "themes": 15 | fmt.Printf("\n== Theme Search Summary ==\n") 16 | } 17 | 18 | fmt.Printf("Time Taken: %s\n", ctx.Stats.GetTimeTaken()) 19 | 20 | totalExtensions := int64(ctx.Stats.GetTotalExtensionsFailed()) 21 | totalFiles := int64(ctx.Stats.GetTotalFiles()) 22 | totalMatches := int64(ctx.Stats.GetTotalFiles()) 23 | 24 | fmt.Printf("Total Extensions: %s\n", humanize.Comma(totalExtensions)) 25 | fmt.Printf("Total Files: %s\n", humanize.Comma(totalFiles)) 26 | fmt.Printf("Total Matches: %s\n", humanize.Comma(totalMatches)) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /internal/pkg/search/search.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "regexp" 10 | "sync" 11 | 12 | "github.com/fatih/color" 13 | "github.com/peterbooker/wpds/internal/pkg/stats" 14 | "github.com/peterbooker/wpds/internal/pkg/utils" 15 | ) 16 | 17 | const ( 18 | // MB holds the number of bytes in a megabyte 19 | MB = 1000000 20 | ) 21 | 22 | // Search ... 23 | type Search struct { 24 | input *regexp.Regexp 25 | path string 26 | Results []Match 27 | context *Context 28 | } 29 | 30 | // Match contains details for a search match. 31 | type Match struct { 32 | Extension string 33 | Filename string 34 | Path string 35 | Line int 36 | Text string 37 | } 38 | 39 | // Context ... 40 | type Context struct { 41 | ExtensionType string 42 | FileType string 43 | ExtWhitelist []string 44 | WorkingDirectory string 45 | Stats *stats.Stats 46 | } 47 | 48 | // Setup ... 49 | func Setup(input string, ctx *Context) *Search { 50 | 51 | searchPath := filepath.Join(ctx.WorkingDirectory, ctx.ExtensionType) 52 | 53 | if !utils.DirExists(searchPath) || utils.IsDirEmpty(searchPath) { 54 | log.Fatalf("Nothing to search at specified location: %s", searchPath) 55 | } 56 | 57 | // TODO: Implement this as context option 58 | ignoreCase := false 59 | 60 | var regex *regexp.Regexp 61 | var err error 62 | 63 | if ignoreCase { 64 | regex, err = regexp.Compile(`(?i)(` + input + `)`) 65 | } else { 66 | regex, err = regexp.Compile(`(` + input + `)`) 67 | } 68 | 69 | if err != nil { 70 | log.Fatalf("Cannot compile regex, invalid syntax: %s\n", input) 71 | } 72 | 73 | return &Search{ 74 | input: regex, 75 | path: searchPath, 76 | context: ctx, 77 | } 78 | 79 | } 80 | 81 | // Run ... 82 | func (s *Search) Run() error { 83 | 84 | extensions := ListDirNames(s.path) 85 | 86 | fmt.Printf("Total Extensions: %d\n", len(extensions)) 87 | 88 | var wg sync.WaitGroup 89 | 90 | limiter := make(chan struct{}, 18) 91 | 92 | for _, extension := range extensions { 93 | 94 | limiter <- struct{}{} 95 | 96 | path := filepath.Join(s.path, extension) 97 | 98 | wg.Add(1) 99 | // Start a goroutine to fetch the folder. 100 | go func(root string, extension string) { 101 | 102 | // Decrement the counter when the goroutine completes. 103 | defer wg.Done() 104 | 105 | filepath.Walk(root, func(path string, info os.FileInfo, err error) error { 106 | 107 | pathOffset := len(root) - len(extension) 108 | 109 | if err != nil { 110 | //fmt.Println(err) 111 | } 112 | 113 | // Directories cannot be searched 114 | if info.IsDir() { 115 | return nil 116 | } 117 | 118 | if info.Size() > (20 * MB) { 119 | return fmt.Errorf("File exceeds valid size for searching. Filename: %s Size: %d", path, info.Size()) 120 | } 121 | 122 | data, err := ioutil.ReadFile(path) 123 | if err != nil { 124 | return err 125 | } 126 | 127 | // Ignore files which are not utf8 encoded e.g. binary files- images, etc. 128 | if !isUTF8(data) { 129 | return nil 130 | } 131 | 132 | matches := s.input.FindAllIndex(data, -1) 133 | 134 | if matches != nil { 135 | 136 | for _, match := range matches { 137 | 138 | start, end := getMatchLineIndexes(data, match) 139 | 140 | line := getLineNum(data[:end]) 141 | 142 | var preText, matchText, postText, fullText string 143 | preText = string(data[start:match[0]]) 144 | matchText = string(data[match[0]:match[1]]) 145 | postText = string(data[match[1]:end]) 146 | fullText = string(data[start:end]) 147 | 148 | yellow := color.New(color.FgYellow).SprintFunc() 149 | red := color.New(color.FgRed).SprintFunc() 150 | green := color.New(color.FgGreen).SprintFunc() 151 | 152 | fmt.Println("") 153 | fmt.Printf("%s\n", green(path[pathOffset:len(path)])) 154 | fmt.Printf("%s: %s%s%s\n", red(line), preText, yellow(matchText), postText) 155 | 156 | m := Match{ 157 | Extension: extension, 158 | Filename: extension + string(os.PathSeparator) + info.Name(), 159 | Path: path, 160 | Line: line, 161 | Text: fullText, 162 | } 163 | 164 | s.Results = append(s.Results, m) 165 | 166 | } 167 | 168 | } 169 | 170 | return nil 171 | 172 | }) 173 | 174 | <-limiter 175 | 176 | }(path, extension) 177 | 178 | } 179 | 180 | wg.Wait() 181 | 182 | printSummary(s.context) 183 | 184 | // TODO: Write search results to file. 185 | 186 | return nil 187 | 188 | } 189 | 190 | // ListDirNames lists all Directories for a type of extension. 191 | func ListDirNames(path string) []string { 192 | 193 | var dirs []string 194 | 195 | files, err := ioutil.ReadDir(path) 196 | if err != nil { 197 | log.Fatal(err) 198 | } 199 | 200 | for _, file := range files { 201 | 202 | if file.IsDir() { 203 | dirs = append(dirs, file.Name()) 204 | } 205 | 206 | } 207 | 208 | return dirs 209 | 210 | } 211 | -------------------------------------------------------------------------------- /internal/pkg/slurper/archive.go: -------------------------------------------------------------------------------- 1 | package slurper 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "io" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/peterbooker/wpds/internal/pkg/context" 12 | "github.com/peterbooker/wpds/internal/pkg/utils" 13 | ) 14 | 15 | // ExtractZip extracts the archive containing extension data. 16 | func ExtractZip(content []byte, length int64, dest string, ctx *context.Context) (uint64, error) { 17 | 18 | zr, err := zip.NewReader(bytes.NewReader(content), length) 19 | if err != nil { 20 | return 0, err 21 | } 22 | 23 | path := filepath.Join(wd, ctx.ExtensionType, dest) 24 | 25 | if utils.DirExists(path) { 26 | err := utils.RemoveDir(path) 27 | if err != nil { 28 | log.Printf("Cannot delete extension folder: %s\n", path) 29 | } 30 | } 31 | 32 | err = utils.CreateDir(path) 33 | if err != nil { 34 | log.Printf("Cannot create extension folder: %s\n", path) 35 | } 36 | 37 | // Used to avoid open file descriptors. 38 | // TODO: Check it actually helps. 39 | writeFile := func(zf *zip.File) { 40 | 41 | // If this is a Directory, create it and move on. 42 | if zf.FileInfo().IsDir() { 43 | folder := filepath.Join(wd, ctx.ExtensionType, zf.Name) 44 | utils.CreateDir(folder) 45 | return 46 | } 47 | 48 | fr, err := zf.Open() 49 | if err != nil { 50 | log.Printf("Unable to read file: %s\n", zf.Name) 51 | return 52 | } 53 | defer fr.Close() 54 | 55 | path := filepath.FromSlash(filepath.Join(wd, ctx.ExtensionType, zf.Name)) 56 | dt := filepath.Dir(path) 57 | 58 | // Make the directory required by this File. 59 | utils.CreateDir(dt) 60 | 61 | // Create File. 62 | f, err := os.Create(path) 63 | if err != nil { 64 | log.Printf("Unable to create file: %s\n", path) 65 | return 66 | } 67 | 68 | defer f.Close() 69 | 70 | // Copy contents to the File. 71 | _, err = io.Copy(f, fr) 72 | if err != nil { 73 | log.Printf("Problem writing contents to file <%s>: %s\n", path, err) 74 | f.Close() 75 | return 76 | } 77 | 78 | err = f.Close() 79 | if err != nil { 80 | log.Printf("Problem closing file <%s>: %s\n", path, err) 81 | return 82 | } 83 | 84 | return 85 | 86 | } 87 | 88 | var size uint64 89 | 90 | // Create each File in the Archive. 91 | for _, zf := range zr.File { 92 | writeFile(zf) 93 | size += zf.UncompressedSize64 94 | ctx.Stats.IncrementTotalFiles() 95 | } 96 | 97 | return size, nil 98 | 99 | } 100 | -------------------------------------------------------------------------------- /internal/pkg/slurper/client.go: -------------------------------------------------------------------------------- 1 | package slurper 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net" 8 | "net/http" 9 | "os" 10 | "runtime" 11 | "time" 12 | ) 13 | 14 | // NewClient returns a new HTTP client configured for large numbers of requests. 15 | func NewClient(timeout int, max int) *http.Client { 16 | 17 | var netTransport = &http.Transport{ 18 | Proxy: http.ProxyFromEnvironment, 19 | DialContext: (&net.Dialer{ 20 | Timeout: 30 * time.Second, 21 | KeepAlive: 30 * time.Second, 22 | DualStack: true, 23 | }).DialContext, 24 | MaxIdleConns: 100, 25 | IdleConnTimeout: 90 * time.Second, 26 | TLSHandshakeTimeout: 10 * time.Second, 27 | ExpectContinueTimeout: 1 * time.Second, 28 | MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, 29 | } 30 | 31 | var netClient = &http.Client{ 32 | Timeout: time.Second * time.Duration(timeout), 33 | Transport: netTransport, 34 | } 35 | 36 | return netClient 37 | 38 | } 39 | 40 | // NewRequest sets up and creates a new HTTP request to the given URL 41 | func NewRequest(URL string, timeout int, concurrent int) ([]byte, error) { 42 | 43 | client := NewClient(timeout, concurrent) 44 | 45 | req, err := http.NewRequest("GET", URL, nil) 46 | if err != nil { 47 | log.Println(err) 48 | } 49 | 50 | req.Header.Set("User-Agent", "wpds/0.5.0") 51 | 52 | resp, err := client.Do(req) 53 | if err != nil { 54 | log.Println("Failed HTTP Request") 55 | os.Exit(1) 56 | } 57 | 58 | defer resp.Body.Close() 59 | 60 | if resp.StatusCode != 200 { 61 | return []byte{}, fmt.Errorf("Invalid HTTP Status Code: %d", resp.StatusCode) 62 | 63 | } 64 | 65 | content, err := ioutil.ReadAll(resp.Body) 66 | if err != nil { 67 | return []byte{}, err 68 | } 69 | 70 | return content, nil 71 | 72 | } 73 | -------------------------------------------------------------------------------- /internal/pkg/slurper/extensions.go: -------------------------------------------------------------------------------- 1 | package slurper 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "sync" 11 | "time" 12 | "unicode/utf8" 13 | 14 | retry "github.com/giantswarm/retry-go" 15 | "github.com/peterbooker/wpds/internal/pkg/config" 16 | "github.com/peterbooker/wpds/internal/pkg/context" 17 | "github.com/peterbooker/wpds/internal/pkg/utils" 18 | pb "gopkg.in/cheggaaa/pb.v2" 19 | ) 20 | 21 | const tmpl = `{{counters .}} {{bar . "[" "=" ">" "-" "]"}} {{rtime .}} {{percent .}}` 22 | 23 | // fetchExtensions uses a list of extensions (themes or plugins) to download and extract their archives. 24 | func fetchExtensions(extensions []string, ctx *context.Context) error { 25 | 26 | ctx.Client = NewClient(180, ctx.ConcurrentActions) 27 | 28 | // Use WaitGroup to ensure all Gorountines have finished downloading/extracting. 29 | var wg sync.WaitGroup 30 | 31 | bar := pb.ProgressBarTemplate(tmpl).Start(len(extensions)) 32 | 33 | limiter := make(chan struct{}, ctx.ConcurrentActions) 34 | 35 | // Make Plugins Dir ready for extracting plugins 36 | path := filepath.Join(wd, ctx.ExtensionType) 37 | err := utils.CreateDir(path) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | // Look through extensions and start a Goroutine to download and extract the files. 43 | for _, name := range extensions { 44 | 45 | // Will block if more than max Goroutines already running. 46 | limiter <- struct{}{} 47 | 48 | wg.Add(1) 49 | 50 | go func(name string, ctx *context.Context, wg *sync.WaitGroup) { 51 | defer wg.Done() 52 | defer bar.Increment() 53 | getExtension(name, ctx) 54 | <-limiter 55 | }(name, ctx, &wg) 56 | 57 | } 58 | 59 | // If any previous downloads failed, retry the failures. 60 | if len(ctx.FailedList) > 0 { 61 | 62 | t := bar.Total() 63 | bar.SetTotal(t + int64(len(ctx.FailedList))) 64 | 65 | for _, name := range extensions { 66 | 67 | // Will block if more than max Goroutines already running. 68 | limiter <- struct{}{} 69 | 70 | wg.Add(1) 71 | go func(name string, ctx *context.Context, wg *sync.WaitGroup) { 72 | defer wg.Done() 73 | defer bar.Increment() 74 | getExtension(name, ctx) 75 | <-limiter 76 | }(name, ctx, &wg) 77 | } 78 | 79 | } 80 | 81 | wg.Wait() 82 | 83 | bar.Finish() 84 | 85 | return nil 86 | 87 | } 88 | 89 | // getExtension fetches the relevant data for the extension e.g. All files, readme.txt, etc. 90 | func getExtension(name string, ctx *context.Context) { 91 | 92 | var data []byte 93 | var err error 94 | var size uint64 95 | 96 | // isExtensionNameValid? 97 | if !isValidName(name) { 98 | ctx.Stats.IncrementTotalExtensionsClosed() 99 | return 100 | } 101 | 102 | switch ctx.FileType { 103 | case "all": 104 | 105 | // Gets the data of the archive file. 106 | fetch := func() error { 107 | data, err = getExtensionZip(name, ctx) 108 | return err 109 | } 110 | err := retry.Do(fetch, retry.Timeout(600*time.Second), retry.MaxTries(3), retry.Sleep(5*time.Second)) 111 | if err != nil { 112 | extensionFailure(name, ctx) 113 | return 114 | } 115 | 116 | // Received 404 response, not an error but we have no data so no more actions to take. 117 | if len(data) == 0 { 118 | ctx.Stats.IncrementTotalExtensionsClosed() 119 | return 120 | } 121 | 122 | // Extracts the archive data to disk. 123 | size, err = ExtractZip(data, int64(len(data)), name, ctx) 124 | if err != nil { 125 | extensionFailure(name, ctx) 126 | return 127 | } 128 | 129 | break 130 | 131 | case "readme": 132 | 133 | // Gets the data of the readme file. 134 | fetch := func() error { 135 | data, err = getExtensionReadme(name, ctx) 136 | return err 137 | } 138 | err := retry.Do(fetch, retry.Timeout(600*time.Second), retry.MaxTries(3), retry.Sleep(5*time.Second)) 139 | if err != nil { 140 | extensionFailure(name, ctx) 141 | return 142 | } 143 | 144 | // Received 404 response, not an error but we have no data so no more actions to take. 145 | if len(data) == 0 { 146 | ctx.Stats.IncrementTotalExtensionsClosed() 147 | return 148 | } 149 | 150 | // Writes the readme file to disk. 151 | size, err = writeReadme(data, name, ctx) 152 | if err != nil { 153 | extensionFailure(name, ctx) 154 | return 155 | } 156 | 157 | break 158 | 159 | } 160 | 161 | ctx.Stats.IncrementTotalExtensions() 162 | ctx.Stats.IncreaseTotalSize(size) 163 | 164 | return 165 | 166 | } 167 | 168 | // getExtensionZip gets the extension archive. 169 | func getExtensionZip(name string, ctx *context.Context) ([]byte, error) { 170 | 171 | var URL string 172 | var content []byte 173 | 174 | switch ctx.ExtensionType { 175 | case "plugins": 176 | URL = utils.EncodeURL(fmt.Sprintf(wpPluginDownloadURL, name)) 177 | case "themes": 178 | URL = utils.EncodeURL(fmt.Sprintf(wpThemeDownloadURL, name)) 179 | } 180 | 181 | req, err := http.NewRequest("GET", URL, nil) 182 | if err != nil { 183 | log.Println(err) 184 | return content, err 185 | } 186 | 187 | // Dynamically set User-Agent from config 188 | req.Header.Set("User-Agent", config.GetName()+"/"+config.GetVersion()) 189 | 190 | resp, err := ctx.Client.Do(req) 191 | if err != nil { 192 | return content, err 193 | } 194 | defer resp.Body.Close() 195 | 196 | if resp.StatusCode != 200 { 197 | 198 | // Code 404 is acceptable, it means the plugin/theme is no longer available. 199 | if resp.StatusCode == 404 { 200 | return content, nil 201 | } 202 | 203 | log.Printf("Downloading the extension '%s' failed. Response code: %d\n", name, resp.StatusCode) 204 | 205 | return content, err 206 | 207 | } 208 | 209 | content, err = ioutil.ReadAll(resp.Body) 210 | if err != nil { 211 | return content, err 212 | } 213 | 214 | return content, nil 215 | 216 | } 217 | 218 | // getExtensionReadme gets the extension readme. 219 | func getExtensionReadme(name string, ctx *context.Context) ([]byte, error) { 220 | 221 | var URL string 222 | var content []byte 223 | 224 | switch ctx.ExtensionType { 225 | case "plugins": 226 | URL = utils.EncodeURL(fmt.Sprintf(wpPluginReadmeURL, name)) 227 | case "themes": 228 | URL = utils.EncodeURL(fmt.Sprintf(wpThemeReadmeURL, name)) 229 | } 230 | 231 | req, err := http.NewRequest("GET", URL, nil) 232 | if err != nil { 233 | log.Println(err) 234 | return content, err 235 | } 236 | 237 | // Dynamically set User-Agent from config 238 | req.Header.Set("User-Agent", config.GetName()+"/"+config.GetVersion()) 239 | 240 | resp, err := ctx.Client.Do(req) 241 | if err != nil { 242 | return content, err 243 | } 244 | defer resp.Body.Close() 245 | 246 | if resp.StatusCode != 200 { 247 | 248 | // Code 404 is acceptable, means the plugin/theme is no longer available. 249 | if resp.StatusCode == 404 { 250 | return content, nil 251 | } 252 | 253 | log.Printf("Downloading the extension '%s' failed. Response code: %d\n", name, resp.StatusCode) 254 | 255 | return content, err 256 | 257 | } 258 | 259 | content, err = ioutil.ReadAll(resp.Body) 260 | if err != nil { 261 | return content, err 262 | } 263 | 264 | return content, err 265 | 266 | } 267 | 268 | // writeReadme writes the readme file to disk. 269 | func writeReadme(content []byte, name string, ctx *context.Context) (uint64, error) { 270 | 271 | base := filepath.Join(wd, ctx.ExtensionType, name) 272 | 273 | if utils.DirExists(base) { 274 | err := utils.RemoveDir(base) 275 | if err != nil { 276 | log.Printf("Cannot delete extension folder: %s\n", base) 277 | } 278 | } 279 | 280 | // Create base dir 281 | err := utils.CreateDir(base) 282 | if err != nil { 283 | log.Printf("Cannot create extension folder: %s\n", base) 284 | } 285 | 286 | path := filepath.Join(wd, ctx.ExtensionType, name, "readme.txt") 287 | 288 | if _, err := os.Stat(path); os.IsNotExist(err) { 289 | 290 | f, err := os.Create(path) 291 | defer f.Close() 292 | if err != nil { 293 | return 0, err 294 | } 295 | 296 | } 297 | 298 | f, err := os.OpenFile(path, os.O_RDWR, 0777) 299 | if err != nil { 300 | return 0, err 301 | } 302 | 303 | defer f.Close() 304 | 305 | _, err = f.Write(content) 306 | if err != nil { 307 | return 0, err 308 | } 309 | 310 | fi, err := f.Stat() 311 | if err != nil { 312 | return 0, err 313 | } 314 | 315 | return uint64(fi.Size()), nil 316 | 317 | } 318 | 319 | // isValidName checks if the extension name is utf8 encoded, anything not will have been closed in the repository. 320 | func isValidName(name string) bool { 321 | return utf8.Valid([]byte(name)) 322 | } 323 | -------------------------------------------------------------------------------- /internal/pkg/slurper/failed.go: -------------------------------------------------------------------------------- 1 | package slurper 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/peterbooker/wpds/internal/pkg/context" 10 | ) 11 | 12 | const ( 13 | filename = ".failed-downloads" 14 | ) 15 | 16 | // extensionFailure appends an extension name to the .failed-downloads file. 17 | // TODO: Create a single buffered writer for this, to avoid many Goroutines writing to a single file. 18 | func extensionFailure(name string, ctx *context.Context) { 19 | 20 | ctx.Stats.IncrementTotalExtensionsFailed() 21 | 22 | path := filepath.Join(wd, ctx.ExtensionType, filename) 23 | 24 | if _, err := os.Stat(path); os.IsNotExist(err) { 25 | 26 | f, err := os.Create(path) 27 | defer f.Close() 28 | if err != nil { 29 | return 30 | } 31 | 32 | } 33 | 34 | f, err := os.OpenFile(path, os.O_APPEND, 0777) 35 | if err != nil { 36 | return 37 | } 38 | 39 | defer f.Close() 40 | 41 | _, err = f.WriteString(name + "\n") 42 | if err != nil { 43 | return 44 | } 45 | 46 | return 47 | 48 | } 49 | 50 | // failuresExist checks if the .failed-downloads file exists. 51 | func failuresExist(extType string) bool { 52 | 53 | wd, _ := os.Getwd() 54 | 55 | path := filepath.Join(wd, extType, filename) 56 | 57 | if _, err := os.Stat(path); err == nil { 58 | return true 59 | } 60 | 61 | return false 62 | 63 | } 64 | 65 | // getFailedList reads the .failed-downloads file and returns a list of extensions. 66 | func getFailedList(extType string) []string { 67 | 68 | var list []string 69 | 70 | wd, _ := os.Getwd() 71 | 72 | path := filepath.Join(wd, extType, filename) 73 | 74 | file, err := os.OpenFile(path, os.O_RDONLY, 0644) 75 | if err != nil { 76 | return list 77 | } 78 | defer file.Close() 79 | 80 | r := bufio.NewReader(file) 81 | 82 | for { 83 | 84 | line, _, err := r.ReadLine() 85 | if err == io.EOF { 86 | break 87 | } 88 | if err != nil { 89 | continue 90 | } 91 | 92 | list = append(list, string(line)) 93 | 94 | } 95 | 96 | return list 97 | 98 | } 99 | 100 | // removeFailedList deletes the .failed-downloads file. 101 | func removeFailedList(extType string) error { 102 | 103 | wd, _ := os.Getwd() 104 | 105 | path := filepath.Join(wd, extType, filename) 106 | 107 | return os.Remove(path) 108 | 109 | } 110 | -------------------------------------------------------------------------------- /internal/pkg/slurper/results.go: -------------------------------------------------------------------------------- 1 | package slurper 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/dustin/go-humanize" 7 | "github.com/peterbooker/wpds/internal/pkg/context" 8 | ) 9 | 10 | func printResults(ctx *context.Context) { 11 | 12 | fmt.Printf("\n== Command Results ==\n") 13 | fmt.Printf("Time Taken: %s\n", ctx.Stats.GetTimeTaken()) 14 | 15 | switch ctx.ExtensionType { 16 | case "plugins": 17 | pluginResults(ctx) 18 | case "themes": 19 | themeResults(ctx) 20 | } 21 | 22 | failedDownloads := int64(ctx.Stats.GetTotalExtensionsFailed()) 23 | totalFiles := int64(ctx.Stats.GetTotalFiles()) 24 | totalFileSize := ctx.Stats.GetTotalSize() 25 | 26 | fmt.Printf("Failed Downloads: %s\n", humanize.Comma(failedDownloads)) 27 | fmt.Printf("Total Files: %s\n", humanize.Comma(totalFiles)) 28 | fmt.Printf("Total Disk Size: %s\n", humanize.Bytes(totalFileSize)) 29 | 30 | } 31 | 32 | func pluginResults(ctx *context.Context) { 33 | 34 | totalPlugins := int64(ctx.Stats.GetTotalExtensions()) 35 | closedPlugins := int64(ctx.Stats.GetTotalExtensionsClosed()) 36 | 37 | fmt.Printf("Total Plugins: %s\n", humanize.Comma(totalPlugins)) 38 | fmt.Printf("Closed/Disabled Plugins: %s\n", humanize.Comma(closedPlugins)) 39 | 40 | } 41 | 42 | func themeResults(ctx *context.Context) { 43 | 44 | totalThemes := int64(ctx.Stats.GetTotalExtensions()) 45 | closedThemes := int64(ctx.Stats.GetTotalExtensionsClosed()) 46 | 47 | fmt.Printf("Total Themes: %s\n", humanize.Comma(totalThemes)) 48 | fmt.Printf("Unapproved/Closed Themes: %s\n", humanize.Comma(closedThemes)) 49 | 50 | } 51 | -------------------------------------------------------------------------------- /internal/pkg/slurper/revisions.go: -------------------------------------------------------------------------------- 1 | package slurper 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | "strconv" 11 | "time" 12 | ) 13 | 14 | func getLatestRevision(extType string) (int, error) { 15 | 16 | var revision int 17 | var rURL string 18 | 19 | c := &http.Client{ 20 | Timeout: 30 * time.Second, 21 | } 22 | 23 | switch extType { 24 | case "plugins": 25 | rURL = wpLatestPluginsRevisionURL 26 | case "themes": 27 | rURL = wpLatestThemesRevisionURL 28 | } 29 | 30 | resp, err := c.Get(rURL) 31 | if err != nil { 32 | return 0, err 33 | } 34 | 35 | if resp.StatusCode != 200 { 36 | return 0, fmt.Errorf("Invalid HTTP Response") 37 | } 38 | 39 | defer resp.Body.Close() 40 | bBytes, err := ioutil.ReadAll(resp.Body) 41 | bString := string(bBytes) 42 | 43 | revs := regexRevision.FindAllStringSubmatch(bString, 1) 44 | 45 | revision, err = strconv.Atoi(revs[0][1]) 46 | if err != nil { 47 | return 0, err 48 | } 49 | 50 | return revision, nil 51 | 52 | } 53 | 54 | func getCurrentRevision(extType string) (int, error) { 55 | 56 | var revision int 57 | 58 | fname := ".last-revision" 59 | 60 | path := filepath.Join(wd, extType, fname) 61 | 62 | data, err := ioutil.ReadFile(path) 63 | if err != nil { 64 | return 0, err 65 | } 66 | 67 | revision, err = strconv.Atoi(string(data)) 68 | if err != nil { 69 | return 0, err 70 | } 71 | 72 | return revision, nil 73 | 74 | } 75 | 76 | func setCurrentRevision(rev int, extType string) error { 77 | 78 | fname := ".last-revision" 79 | 80 | path := filepath.Join(wd, extType, fname) 81 | 82 | f, err := os.Create(path) 83 | defer f.Close() 84 | if err != nil { 85 | return err 86 | } 87 | 88 | revision := strconv.Itoa(rev) 89 | 90 | _, err = io.WriteString(f, revision) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | return nil 96 | 97 | } 98 | -------------------------------------------------------------------------------- /internal/pkg/slurper/slurper.go: -------------------------------------------------------------------------------- 1 | package slurper 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "regexp" 9 | 10 | "github.com/peterbooker/wpds/internal/pkg/connector" 11 | "github.com/peterbooker/wpds/internal/pkg/context" 12 | "github.com/peterbooker/wpds/internal/pkg/utils" 13 | ) 14 | 15 | const ( 16 | // TODO: Check if these are needed. Was useful early on but probably not necessary anymore. 17 | wpAllPluginsListURL = "http://plugins.svn.wordpress.org/" 18 | wpAllThemesListURL = "http://themes.svn.wordpress.org/" 19 | wpLatestPluginsRevisionURL = "http://plugins.trac.wordpress.org/log/?format=changelog&stop_rev=HEAD" 20 | wpLatestThemesRevisionURL = "http://themes.trac.wordpress.org/log/?format=changelog&stop_rev=HEAD" 21 | wpPluginChangelogURL = "https://plugins.trac.wordpress.org/log/?verbose=on&mode=follow_copy&format=changelog&rev=%d&limit=%d" 22 | wpThemeChangelogURL = "https://themes.trac.wordpress.org/log/?verbose=on&mode=follow_copy&format=changelog&rev=%d&limit=%d" 23 | wpPluginDownloadURL = "http://downloads.wordpress.org/plugin/%s.latest-stable.zip?nostats=1" 24 | wpThemeDownloadURL = "http://downloads.wordpress.org/theme/%s.latest-stable.zip?nostats=1" 25 | wpPluginReadmeURL = "https://plugins.svn.wordpress.org/%s/trunk/readme.txt" 26 | wpThemeReadmeURL = "https://theme.svn.wordpress.org/%s/trunk/readme.txt" 27 | wpPluginInfoURL = "https://api.wordpress.org/plugins/info/1.1/?action=plugin_information&request[slug]=%s&request[fields][active_installs]=1" 28 | wpThemeInfoURL = "https://api.wordpress.org/themes/info/1.1/?action=plugin_information&request[slug]=%s&request[fields][active_installs]=1" 29 | ) 30 | 31 | var ( 32 | regexRevision = regexp.MustCompile(`\[(\d*)\]`) 33 | regexHTMLRevision = regexp.MustCompile(`[0-9]+`) 34 | regexUpdatedItems = regexp.MustCompile(`\* ([^/A-Z ]+)`) 35 | ) 36 | 37 | var ( 38 | wd, _ = os.Getwd() 39 | ) 40 | 41 | // StartUpdate begins the update 'plugin/theme' command. 42 | // It begins by checking for an existing folder. 43 | // TODO: Check for folder and .last-revision file as that is needed to update an existing slurp. 44 | func StartUpdate(ctx *context.Context) { 45 | 46 | var fresh bool 47 | 48 | path := filepath.Join(wd, ctx.ExtensionType) 49 | 50 | if utils.DirExists(path) { 51 | // Dir exists, check if empty 52 | if utils.IsDirEmpty(path) { 53 | fresh = true 54 | } else { 55 | fresh = false 56 | } 57 | } else { 58 | // No existing slurp folder 59 | fresh = true 60 | } 61 | 62 | if fresh { 63 | // Begin fresh Directory Slurp 64 | err := newSlurp(ctx) 65 | if err != nil { 66 | log.Println(err) 67 | os.Exit(1) 68 | } 69 | } else { 70 | // Continue Existing Slurp Directory 71 | err := updateSlurp(ctx) 72 | if err != nil { 73 | log.Println(err) 74 | os.Exit(1) 75 | } 76 | } 77 | 78 | // Print Results of Command 79 | printResults(ctx) 80 | 81 | } 82 | 83 | // Begins a fresh slurp. 84 | func newSlurp(ctx *context.Context) error { 85 | 86 | var extensions []string 87 | var revision int 88 | var err error 89 | 90 | conn := connector.Init(ctx.Connector) 91 | 92 | extensions, err = conn.GetFullExtensionsList(ctx) 93 | if err != nil { 94 | return err 95 | } 96 | 97 | revision, err = conn.GetLatestRevision(ctx) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | err = fetchExtensions(extensions, ctx) 103 | if err != nil { 104 | return err 105 | } 106 | 107 | err = setCurrentRevision(revision, ctx.ExtensionType) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | return nil 113 | 114 | } 115 | 116 | // Updates a current slurp. 117 | func updateSlurp(ctx *context.Context) error { 118 | 119 | var extensions []string 120 | 121 | conn := connector.Init(ctx.Connector) 122 | 123 | currentRevision, err := getCurrentRevision(ctx.ExtensionType) 124 | if err != nil { 125 | return err 126 | } 127 | 128 | ctx.CurrentRevision = currentRevision 129 | 130 | latestRevision, err := conn.GetLatestRevision(ctx) 131 | if err != nil { 132 | return err 133 | } 134 | 135 | ctx.LatestRevision = latestRevision 136 | 137 | revisionDiff := latestRevision - currentRevision 138 | 139 | if revisionDiff <= 0 { 140 | fmt.Printf("No updates available. Revision: %d/%d.\n", currentRevision, latestRevision) 141 | os.Exit(1) 142 | } 143 | 144 | extensions, err = conn.GetUpdatedExtensionsList(ctx) 145 | if err != nil { 146 | return err 147 | } 148 | 149 | err = fetchExtensions(extensions, ctx) 150 | if err != nil { 151 | return err 152 | } 153 | 154 | err = setCurrentRevision(latestRevision, ctx.ExtensionType) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | // Check for failed downloads and retry. 160 | var failures []string 161 | 162 | if failuresExist(ctx.ExtensionType) { 163 | 164 | failures = getFailedList(ctx.ExtensionType) 165 | err := removeFailedList(ctx.ExtensionType) 166 | if err != nil { 167 | log.Printf("Failed to delete the %s .failed-downloads file.\n", ctx.ExtensionType) 168 | } 169 | 170 | log.Println("Failed downloads detected. Attempting to download again.") 171 | err = fetchExtensions(failures, ctx) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | } 177 | 178 | return nil 179 | 180 | } 181 | -------------------------------------------------------------------------------- /internal/pkg/slurper/svn.go: -------------------------------------------------------------------------------- 1 | package slurper 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "regexp" 7 | "strconv" 8 | ) 9 | 10 | var ( 11 | regexSVNRevisions = regexp.MustCompile(`.{1,} \/(.+?)\/`) 12 | regexSVNLatest = regexp.MustCompile(`r([0-9]+)`) 13 | ) 14 | 15 | // CheckForSVN checks if the SVN CLI tool is available. 16 | func CheckForSVN() bool { 17 | 18 | _, err := exec.LookPath("svn") 19 | if err != nil { 20 | return false 21 | } 22 | 23 | return true 24 | 25 | } 26 | 27 | // getSVNUpdatedExtensions gets a list of extensions which were updated between the given revisions. 28 | func getSVNUpdatedExtensions(cRev, lRev int, extType string) []string { 29 | 30 | diff := fmt.Sprintf("%d:%d", cRev, lRev) 31 | 32 | URL := fmt.Sprintf("https://%s.svn.wordpress.org/", extType) 33 | 34 | args := []string{"log", "-v", "-q", URL, "-r", diff} 35 | 36 | out, _ := exec.Command("svn", args...).Output() 37 | 38 | var items []string 39 | 40 | itemsGroups := regexSVNRevisions.FindAllStringSubmatch(string(out), -1) 41 | 42 | found := make(map[string]bool) 43 | 44 | // Get the desired substring match and remove duplicates 45 | for _, item := range itemsGroups { 46 | 47 | if !found[item[1]] { 48 | found[item[1]] = true 49 | items = append(items, item[1]) 50 | } 51 | 52 | } 53 | 54 | return items 55 | 56 | } 57 | 58 | // getSVNLatestRevision gets the latest revision from the target repository. 59 | func getSVNLatestRevision(extType string) (int, error) { 60 | 61 | URL := fmt.Sprintf("https://%s.svn.wordpress.org/", extType) 62 | 63 | args := []string{"log", "-v", "-q", URL, "-r", "HEAD"} 64 | 65 | out, _ := exec.Command("svn", args...).Output() 66 | 67 | itemsGroups := regexSVNLatest.FindAllStringSubmatch(string(out), -1) 68 | 69 | revision, err := strconv.Atoi(itemsGroups[0][1]) 70 | 71 | if err != nil { 72 | return 0, err 73 | } 74 | 75 | return revision, nil 76 | 77 | } 78 | -------------------------------------------------------------------------------- /internal/pkg/stats/stats.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | // Stats holds stats on bulk processing 9 | type Stats struct { 10 | mu sync.RWMutex 11 | StartTime time.Time 12 | TotalExtensions int 13 | TotalExtensionsClosed int 14 | TotalExtensionsFailed int 15 | TotalFiles int 16 | TotalSize uint64 17 | } 18 | 19 | // New constructs and returns a Stats pointer 20 | func New() *Stats { 21 | 22 | stats := &Stats{ 23 | StartTime: time.Now(), 24 | TotalExtensions: 0, 25 | TotalExtensionsClosed: 0, 26 | TotalFiles: 0, 27 | TotalSize: 0, 28 | } 29 | 30 | return stats 31 | 32 | } 33 | 34 | func (s *Stats) GetTimeTaken() time.Duration { 35 | 36 | end := time.Now() 37 | 38 | s.mu.RLock() 39 | totalTime := end.Sub(s.StartTime).Round(time.Second) 40 | s.mu.RUnlock() 41 | 42 | return totalTime 43 | 44 | } 45 | 46 | func (s *Stats) IncrementTotalExtensions() { 47 | 48 | s.mu.Lock() 49 | s.TotalExtensions++ 50 | s.mu.Unlock() 51 | 52 | } 53 | 54 | func (s *Stats) GetTotalExtensions() int { 55 | 56 | s.mu.RLock() 57 | totalExtensions := s.TotalExtensions 58 | s.mu.RUnlock() 59 | 60 | return totalExtensions 61 | 62 | } 63 | 64 | func (s *Stats) IncrementTotalExtensionsClosed() { 65 | 66 | s.mu.Lock() 67 | s.TotalExtensionsClosed++ 68 | s.mu.Unlock() 69 | 70 | } 71 | 72 | func (s *Stats) GetTotalExtensionsClosed() int { 73 | 74 | s.mu.RLock() 75 | totalExtensionsClosed := s.TotalExtensionsClosed 76 | s.mu.RUnlock() 77 | 78 | return totalExtensionsClosed 79 | 80 | } 81 | 82 | func (s *Stats) IncrementTotalExtensionsFailed() { 83 | 84 | s.mu.Lock() 85 | s.TotalExtensionsFailed++ 86 | s.mu.Unlock() 87 | 88 | } 89 | 90 | func (s *Stats) GetTotalExtensionsFailed() int { 91 | 92 | s.mu.RLock() 93 | totalExtensionsFailed := s.TotalExtensionsFailed 94 | s.mu.RUnlock() 95 | 96 | return totalExtensionsFailed 97 | 98 | } 99 | 100 | func (s *Stats) IncrementTotalFiles() { 101 | 102 | s.mu.Lock() 103 | s.TotalFiles++ 104 | s.mu.Unlock() 105 | 106 | } 107 | 108 | func (s *Stats) GetTotalFiles() int { 109 | 110 | s.mu.RLock() 111 | totalFiles := s.TotalFiles 112 | s.mu.RUnlock() 113 | 114 | return totalFiles 115 | 116 | } 117 | 118 | func (s *Stats) IncreaseTotalSize(size uint64) { 119 | 120 | s.mu.Lock() 121 | s.TotalSize += size 122 | s.mu.Unlock() 123 | 124 | } 125 | 126 | func (s *Stats) GetTotalSize() uint64 { 127 | 128 | s.mu.RLock() 129 | totalSize := s.TotalSize 130 | s.mu.RUnlock() 131 | 132 | return totalSize 133 | 134 | } 135 | -------------------------------------------------------------------------------- /internal/pkg/utils/filesystem.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | // CreateDir creates the directory at path. 9 | func CreateDir(path string) error { 10 | 11 | err := os.MkdirAll(path, 0755) 12 | 13 | return err 14 | 15 | } 16 | 17 | // DirExists checks if the path exists. 18 | func DirExists(path string) bool { 19 | 20 | if _, err := os.Stat(path); err == nil { 21 | return true 22 | } 23 | 24 | return false 25 | 26 | } 27 | 28 | // RemoveDir deletes the path and everything contained in it. 29 | func RemoveDir(path string) error { 30 | 31 | err := os.RemoveAll(path) 32 | 33 | return err 34 | 35 | } 36 | 37 | // IsDirEmpty checks if the given directory is empty or not. 38 | func IsDirEmpty(path string) bool { 39 | 40 | f, err := os.Open(path) 41 | if err != nil { 42 | return false 43 | } 44 | defer f.Close() 45 | 46 | // Read first file 47 | _, err = f.Readdir(1) 48 | 49 | // If EOF then the Dir is empty 50 | if err == io.EOF { 51 | return true 52 | } 53 | 54 | return false 55 | 56 | } 57 | -------------------------------------------------------------------------------- /internal/pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "net/url" 7 | ) 8 | 9 | // EncodeURL properly encodes the URL for compatibility with special characters 10 | // e.g. 新浪微博 and ЯндексФотки 11 | func EncodeURL(rawURL string) string { 12 | 13 | u, _ := url.Parse(rawURL) 14 | 15 | URL := u.String() 16 | 17 | return URL 18 | 19 | } 20 | 21 | // CheckClose is used to check the return from Close in a defer statement. 22 | func CheckClose(c io.Closer, err *error) { 23 | cErr := c.Close() 24 | if *err == nil { 25 | *err = cErr 26 | } 27 | } 28 | 29 | // DrainAndClose discards all data from rd and closes it. 30 | func DrainAndClose(rd io.ReadCloser, err *error) { 31 | if rd == nil { 32 | return 33 | } 34 | _, _ = io.Copy(ioutil.Discard, rd) 35 | cErr := rd.Close() 36 | if err != nil && *err == nil { 37 | *err = cErr 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /internal/pkg/utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | ) 7 | 8 | func TestEncodeURL(t *testing.T) { 9 | 10 | cases := map[string]struct{ text, expected string }{ 11 | "Plain": {"wp-seo", "wp-seo"}, 12 | "Special": {"µmint", "%C2%B5mint"}, 13 | "Symbols": {"★-wpsymbols-★", "%E2%98%85-wpsymbols-%E2%98%85"}, 14 | "Russian": {"ЯндексФотки", "%D0%AF%D0%BD%D0%B4%D0%B5%D0%BA%D1%81%D0%A4%D0%BE%D1%82%D0%BA%D0%B8"}, 15 | "Arabic": {"لينوكس-ويكى", "%D9%84%D9%8A%D9%86%D9%88%D9%83%D8%B3-%D9%88%D9%8A%D9%83%D9%89"}, 16 | "Chinese": {"豆瓣秀-for-wordpress", "%E8%B1%86%E7%93%A3%E7%A7%80-for-wordpress"}, 17 | } 18 | 19 | for k, v := range cases { 20 | u, _ := url.Parse(v.text) 21 | actual := u.String() 22 | if actual != v.expected { 23 | t.Errorf("%s - Raw: %s Encoded: %s Expected: %s\n", k, v.text, u.String(), v.expected) 24 | } 25 | 26 | } 27 | 28 | } 29 | --------------------------------------------------------------------------------