├── .github ├── FUNDING.yml ├── build ├── run-tests.sh └── workflows │ ├── pull_request.yml │ ├── push.yml │ └── release.yml ├── .gitignore ├── API.md ├── Dockerfile ├── HACKING.md ├── LICENSE ├── README.md ├── SCREENSHOTS.md ├── _media ├── node-runs.png ├── node-summary.png └── single-run.png ├── cmd_metrics.go ├── cmd_metrics_test.go ├── cmd_prune.go ├── cmd_prune_test.go ├── cmd_serve.go ├── cmd_serve_test.go ├── cmd_version.go ├── cmd_version_test.go ├── cmd_yaml.go ├── data ├── css │ └── bootstrap.min.css ├── favicon.ico ├── fonts │ └── glyphicons-halflings-regular.woff2 ├── index.template ├── js │ ├── Chart.bundle.min.js │ ├── bootstrap.min.js │ ├── jquery-1.12.4.min.js │ └── jquery.tablesorter.min.js ├── node.template ├── radiator.template ├── report.template ├── results.template ├── robots.txt └── valid.yaml ├── db.go ├── db_test.go ├── go.mod ├── go.sum ├── main.go ├── samples └── systemd_service.txt ├── static.go ├── static_test.go ├── timespan.go ├── timespan_test.go ├── yaml_parser.go └── yaml_parser_test.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: skx 4 | custom: https://steve.fi/donate/ 5 | -------------------------------------------------------------------------------- /.github/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The basename of our binary 4 | BASE="puppet-summary" 5 | 6 | # I don't even .. 7 | go env -w GOFLAGS="-buildvcs=false" 8 | 9 | # 10 | # We could build on multiple platforms/archs 11 | # 12 | # Except sqlite3 is a CGO binary, so we can't. 13 | # 14 | BUILD_PLATFORMS="linux" 15 | BUILD_ARCHS="amd64" 16 | 17 | # For each platform 18 | for OS in ${BUILD_PLATFORMS[@]}; do 19 | 20 | # For each arch 21 | for ARCH in ${BUILD_ARCHS[@]}; do 22 | 23 | # Setup a suffix for the binary 24 | SUFFIX="${OS}" 25 | 26 | # i386 is better than 386 27 | if [ "$ARCH" = "386" ]; then 28 | SUFFIX="${SUFFIX}-i386" 29 | else 30 | SUFFIX="${SUFFIX}-${ARCH}" 31 | fi 32 | 33 | # Windows binaries should end in .EXE 34 | if [ "$OS" = "windows" ]; then 35 | SUFFIX="${SUFFIX}.exe" 36 | fi 37 | 38 | echo "Building for ${OS} [${ARCH}] -> ${BASE}-${SUFFIX}" 39 | 40 | # Run the build 41 | export GOARCH=${ARCH} 42 | export GOOS=${OS} 43 | export CGO_ENABLED=1 44 | 45 | go build -ldflags "-X main.version=$(git describe --tags)" -o "${BASE}-${SUFFIX}" 46 | 47 | done 48 | done 49 | -------------------------------------------------------------------------------- /.github/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # I don't even .. 4 | go env -w GOFLAGS="-buildvcs=false" 5 | 6 | # Install the lint-tool, and the shadow-tool 7 | go install golang.org/x/lint/golint@latest 8 | go install golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow@latest 9 | go install honnef.co/go/tools/cmd/staticcheck@latest 10 | 11 | 12 | 13 | # Run the static-check tool. 14 | t=$(mktemp) 15 | staticcheck -checks all ./... > $t 16 | if [ -s $t ]; then 17 | echo "Found errors via 'staticcheck'" 18 | cat $t 19 | rm $t 20 | exit 1 21 | fi 22 | rm $t 23 | 24 | # At this point failures cause aborts 25 | set -e 26 | 27 | # Run the linter 28 | echo "Launching linter .." 29 | golint -set_exit_status ./... 30 | echo "Completed linter .." 31 | 32 | # Run the shadow-checker 33 | echo "Launching shadowed-variable check .." 34 | go vet -vettool=$(which shadow) ./... 35 | echo "Completed shadowed-variable check .." 36 | 37 | # Run golang tests 38 | go test ./... 39 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | on: pull_request 2 | name: Pull Request 3 | jobs: 4 | test: 5 | name: Test 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@master 9 | - name: Test 10 | uses: skx/github-action-tester@master 11 | -------------------------------------------------------------------------------- /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - master 5 | name: Push Event 6 | jobs: 7 | test: 8 | name: Test 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@master 12 | - name: Test 13 | uses: skx/github-action-tester@master 14 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [created] 4 | name: Handle Release 5 | jobs: 6 | upload: 7 | name: Upload 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@master 11 | - name: Generate the artifacts 12 | uses: skx/github-action-build@master 13 | - name: Upload the artifacts 14 | uses: skx/github-action-publish-binaries@master 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | with: 18 | args: puppet-* 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ps.db* 2 | reports 3 | puppet-summary 4 | puppet-summary-* 5 | -------------------------------------------------------------------------------- /API.md: -------------------------------------------------------------------------------- 1 | HTTP End-Points 2 | --------------- 3 | 4 | The following HTTP end-points are implemented by the server: 5 | 6 | * `GET /` 7 | * Show all known-nodes and their current status. 8 | * `GET /node/${fqdn}` 9 | * Shows the last N (max 50) runs of puppet against the given node. 10 | * This includes a graph of run-time. 11 | * `GET /radiator` 12 | * This shows a simple dashboard/radiator view. 13 | * `GET /report/${n}` 14 | * This shows useful output of a given run. 15 | * `POST /search` 16 | * This allows you to search against node-names. 17 | * `POST /upload` 18 | * Store a report, this is expected to be invoked solely by the puppet-master. 19 | 20 | 21 | Scripting End-Points 22 | -------------------- 23 | 24 | Each of the HTTP end-points can be used for automation, and scripting, with the exception of the `POST /upload` route, and the `POST /search` handler. 25 | 26 | By default the various handlers return HTML-responses, but they can each be configured to return: 27 | 28 | * JSON 29 | * XML 30 | 31 | To receive a non-HTML response you can either: 32 | 33 | * Submit an appropriate `Accept` HTTP-header when making your request. 34 | * Append a `?accept=XXX` parameter to your URL. 35 | 36 | To view your list of nodes you might try any of these requests, for example: 37 | 38 | $ curl -H Accept:application/json http://localhost:3001/ 39 | $ curl -H Accept:application/xml http://localhost:3001/ 40 | $ curl http://localhost:3001/?accept=application/json 41 | $ curl http://localhost:3001/?accept=application/xml 42 | 43 | Similarly the radiator-view might be used like so: 44 | 45 | $ curl -H Accept:application/xml http://localhost:3001/radiator/ 46 | 47 | changed 48 | 0 49 | 0 50 | 51 | 52 | failed 53 | 0 54 | .. 55 | 56 | Or: 57 | 58 | $ curl http://localhost:3001/radiator/?accept=application/json 59 | 60 | 61 | 62 | API Endpoints 63 | ------------- 64 | 65 | In addition to the scripting posibilities available with the multi-format 66 | responses there is also a simple end-point which is designed to return a 67 | list of all the nodes in the given state: 68 | 69 | * `GET /api/state/$state` 70 | 71 | This will default to JSON, but you can choose JSON, XML, or pain-text, via the 72 | Accept: header or `?accept=application/json` parameter, for example: 73 | 74 | 75 | $ curl -H Accept:text/plain http://localhost:3001/api/state/unchanged 76 | $ curl -H Accept:application/xml http://localhost:3001/api/state/unchanged 77 | $ curl http://localhost:3001/api/state/unchanged?accept=text/plain 78 | $ curl http://localhost:3001/api/state/unchanged?accept=application/xml 79 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # First build puppet-summary 2 | FROM alpine 3 | RUN apk --no-cache add go git musl-dev 4 | RUN go get -u github.com/skx/puppet-summary 5 | 6 | # Now put it in a container without all the build tools 7 | FROM alpine 8 | WORKDIR /root/ 9 | COPY --from=0 /root/go/bin/puppet-summary . 10 | ENV PORT=3001 11 | EXPOSE 3001 12 | VOLUME /app 13 | ENTRYPOINT ["/root/puppet-summary", "serve", "-host","0.0.0.0", "-db-file", "/app/db1.sqlite", "-auto-prune" ] 14 | -------------------------------------------------------------------------------- /HACKING.md: -------------------------------------------------------------------------------- 1 | 2 | ## Editing the HTML Templates 3 | 4 | The generated HTML views are stored inside the compiled binary to ease 5 | deployment. If you wish to tweak the look & feel by editing them then 6 | you're more then welcome. 7 | 8 | The raw HTML-templates are located beneath `data/`, and you can edit them 9 | then rebuild the compiled versions via `implant`. 10 | 11 | If you don't already have `implant` installed fetch it like so: 12 | 13 | go get -u github.com/skx/implant/ 14 | 15 | Now regenerate the compiled version(s) of the templates and rebuild the 16 | binary to make your changes: 17 | 18 | implant -input data/ -output static.go 19 | go build . 20 | 21 | 22 | ## Test Coverage 23 | 24 | To test the coverage of the test-suite you can use the `cover` tool: 25 | 26 | go get golang.org/x/tools/cmd/cover 27 | go test -coverprofile fmt 28 | 29 | Once you've done that you can view the coverage of various functions via: 30 | 31 | go tool cover -func=fmt 32 | 33 | To view the coverage report in HTML, via your browser this is good: 34 | 35 | go test -coverprofile=cover.out 36 | go tool cover -html=cover.out -o foo.html 37 | firefox foo.html 38 | 39 | 40 | # Running a container 41 | 42 | This project now ships a `Dockerfile`. The goal is to build a small image with 43 | `puppet-summary` installed on it. This uses multi-stage builds for docker and 44 | thus requires docker version 17.05 or higher. The container is based on alpine 45 | linux and should be around 20MB. 46 | 47 | To build: 48 | 49 | docker build -t puppet-summary: . 50 | 51 | To run: 52 | 53 | docker run -d -v app:/app -p 3001:3001 puppet-summary 54 | 55 | **NOTE**: When running in a container the `-auto-prune` flag is applied, with the intention that reports will be pruned on a weekly basis. 56 | 57 | 58 | # Cross compiling puppet-summary 59 | 60 | In this example, the compilation is happening on x86_64 Fedora or a Debian 9 amd64 system with a target of Raspbian on ARM (raspberry PI). 61 | 62 | ## Install the packages you need. 63 | 64 | ### Fedora 65 | `# dnf install binutils-arm-linux-gnu cross-gcc-common cross-binutils-common gcc-c++-arm-linux-gnu kernel-cross-headers glibc-arm-linux-gnu glibc-arm-linux-gnu-devel` 66 | 67 | ### Debian 68 | `# apt-get install cpp-6-arm-linux-gnueabihf g++-6-arm-linux-gnueabihf gcc-6-arm-linux-gnueabihf gcc-6-arm-linux-gnueabihf-base gccgo-6-arm-linux-gnueabihf` 69 | 70 | ## Manually fix pthreads 71 | 72 | _Note:_ This is only required on Fedora builders. 73 | 74 | The way cgo works for cross compiles, it assumes a sysroot, which is normal. However, the way pthreads is called in the github.com/mattn/go-sqlite3 package, it requires and absolute path, but that path is relative to the sysroot provided. 75 | 76 | `# pushd /usr/arm-linux-gnu; ln -s /usr .; popd` 77 | 78 | ## Compile 79 | 80 | I use `-v` when cross compiling because it will give much more info if something errors out. 81 | 82 | ### Fedora 83 | 84 | `$ CC=arm-linux-gnu-gcc CGO_ENABLED=1 GOOS=linux GOARCH=arm CGO_LDFLAGS=--sysroot=/usr/arm-linux-gnu CGO_CFLAGS=--sysroot=/usr/arm-linux-gnu go build -v .` 85 | 86 | ### Debian 87 | 88 | `$ CC=arm-linux-gnueabihf-gcc-6 CGO_ENABLED=1 GOOS=linux GOARCH=arm CGO_LDFLAGS=--sysroot=/usr/arm-linux-gnu CGO_CFLAGS=--sysroot=/usr/arm-linux-gnu go build -v .` 89 | 90 | ## Verify build 91 | 92 | You should have a generated binary now, which you can inspect via: 93 | 94 | `$ file puppet-summary` 95 | 96 | This should show something like: 97 | 98 | `puppet-summary: ELF 32-bit LSB executable, ARM, EABI5 version 1 (SYSV), dynamically linked, interpreter /lib/ld-linux-armhf.so.3, for GNU/Linux 3.2.0, BuildID[sha1]=810382dc0c531df0de230c2f681925d9ebf59fd6, with debug_info, not stripped` 99 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Go Report Card](https://goreportcard.com/badge/github.com/skx/puppet-summary)](https://goreportcard.com/report/github.com/skx/puppet-summary) 2 | [![license](https://img.shields.io/github/license/skx/puppet-summary.svg)](https://github.com/skx/puppet-summary/blob/master/LICENSE) 3 | [![Release](https://img.shields.io/github/release/skx/puppet-summary.svg)](https://github.com/skx/puppet-summary/releases/latest) 4 | [![gocover store](http://gocover.io/_badge/github.com/skx/puppet-summary)](http://gocover.io/github.com/skx/puppet-summary) 5 | 6 | Table of Contents 7 | ================= 8 | 9 | * [Puppet Summary](#puppet-summary) 10 | * [Puppet Reporting](#puppet-reporting) 11 | * [Installation](#installation) 12 | * [Source Installation](#source-installation) 13 | * [Execution](#execution) 14 | * [Importing Puppet State](#importing-puppet-state) 15 | * [Maintenance](#maintenance) 16 | * [Metrics](#metrics) 17 | * [Notes On Deployment](#notes-on-deployment) 18 | * [Service file for systemd](#service-file-for-systemd) 19 | * [Github Setup](#github-setup) 20 | 21 | Puppet Summary 22 | ============== 23 | 24 | This is a simple [golang](https://golang.org/) based project which is designed to offer a dashboard of your current puppet-infrastructure: 25 | 26 | * Listing all known-nodes, and their current state. 27 | * Viewing the last few runs of a given system. 28 | * etc. 29 | 30 | This project is directly inspired by the [puppet-dashboard](https://github.com/sodabrew/puppet-dashboard) project, reasons why you might prefer _this_ project: 31 | 32 | * It is actively maintained. 33 | * Unlike [puppet-dashboard](https://github.com/sodabrew/puppet-dashboard/issues/341). 34 | * Deployment is significantly simpler. 35 | * This project only involves deploying a single binary. 36 | * It allows you to submit metrics to a carbon-receiver. 37 | * The metrics include a distinct count of each state, allowing you to raise alerts when nodes in the failed state are present. 38 | * The output can be used for scripting, and automation. 39 | * All output is available as [JSON/XML](API.md) in addition to human-viewable HTML. 40 | 41 | You can get a good idea of what the project does by looking at the screens: 42 | 43 | * [SCREENSHOTS.md](SCREENSHOTS.md) 44 | 45 | You can also consult the API documentation: 46 | 47 | * [API.md](API.md) 48 | 49 | 50 | 51 | ## Puppet Reporting 52 | 53 | The puppet-server has integrated support for submitting reports to 54 | a central location, via HTTP POSTs. This project is designed to be 55 | a target for such submission: 56 | 57 | * Your puppet-master submits reports to this software. 58 | * The reports are saved locally, as YAML files, beneath `./reports` 59 | * They are parsed and a simple SQLite database keeps track of them. 60 | * The SQLite database is used to present a visualization layer. 61 | * Which you can see [in the screenshots](screenshots/). 62 | 63 | The reports are expected to be pruned over time, but as the SQLite database 64 | only contains a summary of the available data it will not grow excessively. 65 | 66 | > The software has [been reported](https://github.com/skx/puppet-summary/issues/42) to cope with 16k reports per day, archive approximately 27Gb of data over 14 days! 67 | 68 | 69 | 70 | ## Installation 71 | 72 | Installing the service can be done in one of two ways, depending on whether you have the [go](https://golang.org/) toolchain available: 73 | 74 | * Download the appropriate binary from our [project release page](https://github.com/skx/puppet-summary/releases). 75 | * Install from source, as documented below. 76 | 77 | 78 | ### Source Installation 79 | 80 | If you're planning to make changes to the code, or examine it, then the obvious approach to installing from source is to clone the code, then build and install it from that local clone: 81 | 82 | git clone https://github.com/skx/puppet-summary 83 | cd puppet-summary 84 | go install . 85 | 86 | You could install directly from source, without cloning the repository as an interim step, by running: 87 | 88 | go install github.com/skx/puppet-summary@master 89 | 90 | In either case you'll find a binary named `puppet-summary` placed inside a directory named `bin` beneath the golang GOPATH directory. To see exactly where this is please run: 91 | 92 | echo $(go env GOPATH)/bin 93 | 94 | (Typically you'd find binaries deployed to the directory `~/go/bin`, however this might vary.) 95 | 96 | 97 | 98 | ## Execution 99 | 100 | Once installed you can launch it directly like so: 101 | 102 | $ puppet-summary serve 103 | Launching the server on http://127.0.0.1:3001 104 | 105 | If you wish to change the host/port you can do so like this: 106 | 107 | $ puppet-summary serve -host 10.10.10.10 -port 4321 108 | Launching the server on http://10.10.10.10:4321 109 | 110 | To have it listen on any available IP address, use one of these examples: 111 | 112 | $ puppet-summary serve -host "" -port 4321 113 | $ puppet-summary serve -host 0.0.0.0 -port 4321 114 | 115 | Other sub-commands are described later, or can be viewed via: 116 | 117 | $ puppet-summary help 118 | 119 | 120 | 121 | ## Importing Puppet State 122 | 123 | Once you've got an instance of `puppet-summary` installed and running 124 | the next step is to populate it with report data. The expectation is 125 | that you'll update your puppet server to send the reports to it directly, 126 | by editing `puppet.conf` on your puppet-master: 127 | 128 | [master] 129 | reports = store, http 130 | reporturl = http://localhost:3001/upload 131 | 132 | * If you're running the dashboard on a different host you'll need to use the external IP/hostname here. 133 | * Once you've changed your master's configuration don't forget to restart the service! 134 | 135 | If you __don't__ wish to change your puppet-server initially you can test 136 | what it would look like by importing the existing YAML reports from your 137 | puppet-master. Something like this should do the job: 138 | 139 | # cd /var/lib/puppet/reports 140 | # find . -name '*.yaml' -exec \ 141 | curl --data-binary @\{\} http://localhost:3001/upload \; 142 | 143 | * That assumes that your reports are located beneath `/var/lib/puppet/reports`, 144 | but that is a reasonable default. 145 | * It also assumes you're running the `puppet-summary` instance upon the puppet-master, if you're on a different host remember to change the URI. 146 | 147 | 148 | 149 | ## Maintenance 150 | 151 | Over time your reports will start to consuming ever-increasing amounts of disk-space so they should be pruned. To prune (read: delete) old reports run: 152 | 153 | puppet-summary prune -days 7 -prefix ./reports/ 154 | 155 | That will remove the saved YAML files from disk which are over 7 days old, and it will _also_ remove the associated database entries that refer to them. 156 | 157 | If you're happy with the default pruning behaviour, which is particularly useful when you're running this software in a container, described in [HACKING.md](HACKING.md), you can prune old reports automatically once per week without the need to add a cron-job like so: 158 | 159 | puppet-summary serve -auto-prune [options..] 160 | 161 | If you don't do this you'll need to __add a cronjob__ to ensure that the prune-subcommand runs regularly. 162 | 163 | Nodes which had previously submitted updates to your puppet-master, and `puppet-summary` service, but which have failed to do so "recently", will be listed in the web-based user-interface, in the "orphaned" column. Orphaned nodes will be reaped over time, via the `days` option just discussed. If you explicitly wish to clean removed-hosts you can do so via: 164 | 165 | puppet-summary prune -verbose -orphaned 166 | 167 | 168 | 169 | ## Metrics 170 | 171 | If you have a carbon-server running locally you can also submit metrics 172 | to it : 173 | 174 | puppet-summary metrics \ 175 | -host carbon.example.com \ 176 | -port 2003 \ 177 | -prefix puppet.example_com [-nop] 178 | 179 | The metrics include the count of nodes in each state, `changed`, `unchanged`, `failed`, and `orphaned` and can be used to raise alerts when things fail. When running with `-nop` the metrics will be dumped to the console instead of submitted. 180 | 181 | 182 | 183 | ## Notes On Deployment 184 | 185 | If you can run this software upon your puppet-master then that's the ideal, that way your puppet-master would be configured to uploaded your reports to `127.0.0.1:3001/upload`, and the dashboard itself may be viewed via a reverse-proxy. 186 | 187 | The appeal of allowing submissions from the loopback is that your reverse-proxy can deny access to the upload end-point, ensuring nobody else can submit details. A simple nginx configure might look like this: 188 | 189 | server { 190 | server_name reports.example.com; 191 | listen [::]:80 default ipv6only=off; 192 | 193 | ## Puppet-master is the only host that needs access here 194 | ## it is configured to POST to localhost:3001 directly 195 | ## so we can disable access here. 196 | location /upload { 197 | deny all; 198 | } 199 | 200 | ## send all traffic to the back-end 201 | location / { 202 | proxy_pass http://127.0.0.1:3001; 203 | proxy_redirect off; 204 | proxy_set_header X-Forwarded-For $remote_addr; 205 | } 206 | } 207 | 208 | * Please don't run this application as root. 209 | * The defaults are sane, YAML files are stored beneath `./reports`, and the SQLite database is located at "`./ps.db`. 210 | * Both these values can be changed, but if you change them you'll need to remember to change for all appropriate actions. 211 | * For example "`puppet-summary serve -db-file ./new.db`", "`puppet-summary metrics -db-file ./new.db`", and "`puppet-summary prune -db-file ./new.db`". 212 | 213 | 214 | ### Service file for systemd 215 | 216 | You can find instructions on how to create a service file for systemd in the [samples](samples) directory. 217 | 218 | 219 | 220 | ## Github Setup 221 | 222 | This repository is configured to run tests upon every commit, and when 223 | pull-requests are created/updated. The testing is carried out via 224 | [.github/run-tests.sh](.github/run-tests.sh) which is used by the 225 | [github-action-tester](https://github.com/skx/github-action-tester) action. 226 | 227 | Releases are automated in a similar fashion via [.github/build](.github/build), 228 | and the [github-action-publish-binaries](https://github.com/skx/github-action-publish-binaries) action. 229 | 230 | 231 | Steve 232 | -- 233 | -------------------------------------------------------------------------------- /SCREENSHOTS.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | # List of Nodes 7 | 8 | There is a summary that shows the list of known-nodes, and their state: 9 | 10 | ![Screenshot](_media/node-summary.png) 11 | 12 | 13 | 14 | 15 | 16 | # List of runs 17 | 18 | When you select a node you'll details of the last time puppet ran on that node, grouped by state: 19 | 20 | ![Screenshot](_media/node-runs.png) 21 | 22 | 23 | 24 | 25 | # Single Run 26 | 27 | Finally if you click upon a run from the section above you'll see details of _what_ happened: 28 | 29 | ![Screenshot](_media/single-run.png) 30 | 31 | 32 | (Sections here can be toggled, via a click.) 33 | -------------------------------------------------------------------------------- /_media/node-runs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skx/puppet-summary/7851132a898b174d31f0059b4e629008a676ca3e/_media/node-runs.png -------------------------------------------------------------------------------- /_media/node-summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skx/puppet-summary/7851132a898b174d31f0059b4e629008a676ca3e/_media/node-summary.png -------------------------------------------------------------------------------- /_media/single-run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skx/puppet-summary/7851132a898b174d31f0059b4e629008a676ca3e/_media/single-run.png -------------------------------------------------------------------------------- /cmd_metrics.go: -------------------------------------------------------------------------------- 1 | // 2 | // Submit metrics to a graphite host. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "fmt" 11 | "os" 12 | 13 | "github.com/google/subcommands" 14 | graphite "github.com/marpaia/graphite-golang" 15 | ) 16 | 17 | // 18 | // Get all the metrics 19 | // 20 | func getMetrics() map[string]string { 21 | 22 | // A map to store the names & values which should be sent. 23 | metrics := make(map[string]string) 24 | 25 | // Get the node-states. 26 | data, err := getStates("") 27 | if err != nil { 28 | fmt.Printf("Error getting node states: %s\n", err.Error()) 29 | os.Exit(1) 30 | } 31 | 32 | // Now record the metrics we would send. 33 | for i := range data { 34 | // 35 | // The name + value 36 | // 37 | metric := fmt.Sprintf("state.%s", data[i].State) 38 | value := fmt.Sprintf("%d", data[i].Count) 39 | 40 | metrics[metric] = value 41 | } 42 | 43 | // And return them 44 | return metrics 45 | } 46 | 47 | // 48 | // SendMetrics submits the metrics discovered to the specified carbon 49 | // server - unless `nop` is in-use, in which case they are dumped to 50 | // STDOUT. 51 | // 52 | func SendMetrics(host string, port int, prefix string, nop bool) { 53 | 54 | // Get the metrics. 55 | metrics := getMetrics() 56 | 57 | // Create the helper. 58 | g, err := graphite.NewGraphite(host, port) 59 | 60 | // 61 | // If there was an error in the helper we're OK, 62 | // providing we are running in `-nop`-mode. 63 | // 64 | if (err != nil) && (!nop) { 65 | fmt.Printf("Error creating metrics-helper: %s\n", err.Error()) 66 | return 67 | } 68 | 69 | // 70 | // For each one .. 71 | // 72 | for name, value := range metrics { 73 | 74 | // 75 | // Add the prefix. 76 | // 77 | name = fmt.Sprintf("%s.%s", prefix, name) 78 | 79 | // 80 | // Show/Send. 81 | // 82 | if nop { 83 | fmt.Fprintf(out, "%s %s\n", name, value) 84 | } else { 85 | g.SimpleSend(name, value) 86 | } 87 | 88 | } 89 | } 90 | 91 | // 92 | // The options set by our command-line flags. 93 | // 94 | type metricsCmd struct { 95 | dbFile string 96 | host string 97 | port int 98 | prefix string 99 | nop bool 100 | } 101 | 102 | // 103 | // Glue 104 | // 105 | func (*metricsCmd) Name() string { return "metrics" } 106 | func (*metricsCmd) Synopsis() string { return "Submit metrics to a central carbon server." } 107 | func (*metricsCmd) Usage() string { 108 | return `metrics [options]: 109 | Submit metrics to a central carbon server. 110 | ` 111 | } 112 | 113 | // 114 | // Flag setup 115 | // 116 | func (p *metricsCmd) SetFlags(f *flag.FlagSet) { 117 | f.StringVar(&p.dbFile, "db-file", "ps.db", "The SQLite database to use.") 118 | f.StringVar(&p.host, "host", "localhost", "The carbon host to send metrics to.") 119 | f.IntVar(&p.port, "port", 2003, "The carbon port to use, when submitting metrics.") 120 | f.StringVar(&p.prefix, "prefix", "puppet", "The prefix to use when submitting metrics.") 121 | f.BoolVar(&p.nop, "nop", false, "Print metrics rather than submitting them.") 122 | } 123 | 124 | // 125 | // Entry-point. 126 | // 127 | func (p *metricsCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { 128 | 129 | // 130 | // Setup the database, by opening a handle, and creating it if 131 | // missing. 132 | // 133 | SetupDB(p.dbFile) 134 | 135 | // 136 | // Run metrics 137 | // 138 | SendMetrics(p.host, p.port, p.prefix, p.nop) 139 | 140 | // 141 | // All done. 142 | // 143 | return subcommands.ExitSuccess 144 | } 145 | -------------------------------------------------------------------------------- /cmd_metrics_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "os" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestMetrics(t *testing.T) { 12 | 13 | // Create a fake database 14 | FakeDB() 15 | 16 | // Add some hosts. 17 | addFakeNodes() 18 | 19 | // Get the metrics 20 | metrics := getMetrics() 21 | 22 | // Now test we can find things. 23 | if len(metrics) != 4 { 24 | t.Errorf("Unexpected metrics-size: %v", len(metrics)) 25 | } 26 | 27 | // Some values 28 | if metrics["state.changed"] != "1" { 29 | t.Errorf("Unexpected metrics value") 30 | } 31 | if metrics["state.unchanged"] != "0" { 32 | t.Errorf("Unexpected metrics value") 33 | } 34 | if metrics["state.failed"] != "1" { 35 | t.Errorf("Unexpected metrics value") 36 | } 37 | if metrics["state.orphaned"] != "0" { 38 | t.Errorf("Unexpected metrics value") 39 | } 40 | 41 | // 42 | // Cleanup here because otherwise later tests will 43 | // see an active/valid DB-handle. 44 | // 45 | db.Close() 46 | db = nil 47 | os.RemoveAll(path) 48 | } 49 | 50 | // 51 | // Actually attempt to send the metrics to stdout. 52 | // 53 | func TestMetricNop(t *testing.T) { 54 | 55 | // 56 | // Fake out the STDOUT 57 | // 58 | bak := out 59 | out = new(bytes.Buffer) 60 | defer func() { out = bak }() 61 | 62 | // Create a fake database 63 | FakeDB() 64 | 65 | // Add some hosts. 66 | addFakeNodes() 67 | 68 | // 69 | // Dump our metrics to STDOUT, due to `nop`, which will end up 70 | // in our faux buffer. 71 | s := metricsCmd{nop: true} 72 | s.Execute(context.TODO(), nil) 73 | 74 | // 75 | // Now see what we got. 76 | // 77 | read := out.(*bytes.Buffer).String() 78 | 79 | // 80 | // And test it against each of the things we 81 | // expect. 82 | // 83 | // NOTE: We have to do this as the output is ordered 84 | // randomly. 85 | // 86 | desired := []string{".state.changed 0", 87 | ".state.failed 0", 88 | ".state.orphaned 0", 89 | ".state.unchanged 0"} 90 | 91 | for _, str := range desired { 92 | if !strings.Contains(read, str) { 93 | t.Errorf("Unexpected metric-output - %s", read) 94 | } 95 | } 96 | 97 | // 98 | // Cleanup here because otherwise later tests will 99 | // see an active/valid DB-handle. 100 | // 101 | db.Close() 102 | db = nil 103 | os.RemoveAll(path) 104 | } 105 | -------------------------------------------------------------------------------- /cmd_prune.go: -------------------------------------------------------------------------------- 1 | // 2 | // Prune history by removing old reports. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "fmt" 11 | 12 | "github.com/google/subcommands" 13 | ) 14 | 15 | // 16 | // The options set by our command-line flags. 17 | // 18 | type pruneCmd struct { 19 | dbFile string 20 | days int 21 | environment string 22 | unchanged bool 23 | orphaned bool 24 | prefix string 25 | dangling bool 26 | noop bool 27 | verbose bool 28 | } 29 | 30 | // 31 | // Run a prune 32 | // 33 | func runPrune(x pruneCmd) error { 34 | 35 | // 36 | // Remove yaml files that are not referenced in the database 37 | // 38 | if x.dangling { 39 | if x.verbose { 40 | fmt.Printf("Pruning yaml report files that are not referenced in the database from beneath %s\n", ReportPrefix) 41 | } 42 | return (pruneDangling(x.prefix, x.noop, x.verbose)) 43 | } 44 | 45 | // 46 | // Removing orphaned nodes? 47 | // 48 | if x.orphaned { 49 | if x.verbose { 50 | fmt.Printf("Pruning 'orphaned' reports from beneath %s\n", ReportPrefix) 51 | } 52 | return (pruneOrphaned(x.environment, x.prefix, x.verbose)) 53 | } 54 | 55 | // 56 | // Removing unchanged reports? 57 | // 58 | if x.unchanged { 59 | if x.verbose { 60 | fmt.Printf("Pruning 'unchanged' reports from beneath %s\n", ReportPrefix) 61 | } 62 | return (pruneUnchanged(x.environment, x.prefix, x.verbose)) 63 | } 64 | 65 | // 66 | // Otherwise just removing reports older than the given 67 | // number of days. 68 | // 69 | if x.verbose { 70 | fmt.Printf("Pruning reports older than %d days from beneath %s\n", x.days, ReportPrefix) 71 | } 72 | 73 | err := pruneReports(x.environment, x.prefix, x.days, x.verbose) 74 | return err 75 | } 76 | 77 | // 78 | // Glue 79 | // 80 | func (*pruneCmd) Name() string { return "prune" } 81 | func (*pruneCmd) Synopsis() string { return "Prune/delete old reports." } 82 | func (*pruneCmd) Usage() string { 83 | return `prune [options]: 84 | Remove old report-files from disk, and our database. 85 | ` 86 | } 87 | 88 | // 89 | // Flag setup 90 | // 91 | func (p *pruneCmd) SetFlags(f *flag.FlagSet) { 92 | f.BoolVar(&p.verbose, "verbose", false, "Be verbose in reporting output") 93 | f.IntVar(&p.days, "days", 7, "Remove reports older than this many days.") 94 | f.BoolVar(&p.unchanged, "unchanged", false, "Remove reports from hosts which had no changes.") 95 | f.BoolVar(&p.orphaned, "orphaned", false, "Remove reports from hosts which are orphaned.") 96 | f.StringVar(&p.dbFile, "db-file", "ps.db", "The SQLite database to use.") 97 | f.StringVar(&p.prefix, "prefix", "./reports/", "The prefix to the local YAML hierarchy.") 98 | f.BoolVar(&p.dangling, "dangling", false, "Remove yaml reports that are not referenced in the database.") 99 | f.BoolVar(&p.noop, "noop", false, "Do not remove dangling yaml files, just pretend.") 100 | f.StringVar(&p.environment, "environment", "", "If specified only prune this environment.") 101 | } 102 | 103 | // 104 | // Entry-point. 105 | // 106 | func (p *pruneCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { 107 | 108 | // 109 | // Setup the database, by opening a handle, and creating it if 110 | // missing. 111 | // 112 | SetupDB(p.dbFile) 113 | 114 | // 115 | // Invoke the prune 116 | // 117 | err := runPrune(*p) 118 | 119 | if err == nil { 120 | return subcommands.ExitSuccess 121 | } 122 | 123 | fmt.Printf("Error pruning: %s\n", err.Error()) 124 | return subcommands.ExitFailure 125 | } 126 | -------------------------------------------------------------------------------- /cmd_prune_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestPruneCommand(t *testing.T) { 9 | 10 | // Create a fake database 11 | FakeDB() 12 | 13 | // With some reports. 14 | addFakeReports() 15 | 16 | // 17 | // Count records and assume we have some. 18 | // 19 | old, err := countReports() 20 | 21 | if err != nil { 22 | t.Errorf("Error counting reports") 23 | } 24 | if old != 30 { 25 | t.Errorf("We have %d reports, not 30", old) 26 | } 27 | 28 | tmp := pruneCmd{days: 5, verbose: false} 29 | runPrune(tmp) 30 | 31 | // 32 | // Count them again 33 | // 34 | new, err := countReports() 35 | if err != nil { 36 | t.Errorf("Error counting reports") 37 | } 38 | 39 | if new != 6 { 40 | t.Errorf("We have %d reports, not 5", new) 41 | } 42 | 43 | // 44 | // Cleanup here because otherwise later tests will 45 | // see an active/valid DB-handle. 46 | // 47 | db.Close() 48 | db = nil 49 | os.RemoveAll(path) 50 | } 51 | -------------------------------------------------------------------------------- /cmd_serve.go: -------------------------------------------------------------------------------- 1 | // 2 | // Launch our HTTP-server for both consuming reports, and viewing them. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "encoding/json" 11 | "encoding/xml" 12 | "errors" 13 | "flag" 14 | "fmt" 15 | "html/template" 16 | "io/ioutil" 17 | "mime" 18 | "net/http" 19 | "os" 20 | "path/filepath" 21 | "regexp" 22 | "strconv" 23 | "strings" 24 | "time" 25 | 26 | "github.com/google/subcommands" 27 | "github.com/gorilla/handlers" 28 | "github.com/gorilla/mux" 29 | "github.com/robfig/cron" 30 | _ "github.com/skx/golang-metrics" 31 | ) 32 | 33 | // 34 | // ReportPrefix is the path beneath which reports are stored. 35 | // 36 | var ReportPrefix = "reports" 37 | 38 | // 39 | // Exists is a utility method to determine whether a file/directory exists. 40 | // 41 | func Exists(name string) bool { 42 | _, err := os.Stat(name) 43 | return !os.IsNotExist(err) 44 | } 45 | 46 | // 47 | // APIState is the handler for the HTTP end-point 48 | // 49 | // GET /api/state/$state 50 | // 51 | // This only will return plain-text by default, but JSON and XML are both 52 | // possible via the `Accept:` header or `?accept=XX` parameter. 53 | // 54 | func APIState(res http.ResponseWriter, req *http.Request) { 55 | 56 | var ( 57 | status int 58 | err error 59 | ) 60 | defer func() { 61 | if nil != err { 62 | http.Error(res, err.Error(), status) 63 | } 64 | }() 65 | 66 | // 67 | // Get the state the user is interested in. 68 | // 69 | vars := mux.Vars(req) 70 | state := vars["state"] 71 | 72 | // 73 | // Ensure we received a parameter. 74 | // 75 | if len(state) < 1 { 76 | status = http.StatusNotFound 77 | err = errors.New("missing 'state' parameter") 78 | return 79 | } 80 | 81 | // 82 | // Test the supplied state is valid. 83 | // 84 | switch state { 85 | case "changed": 86 | case "unchanged": 87 | case "failed": 88 | case "orphaned": 89 | default: 90 | err = errors.New("invalid state supplied") 91 | status = http.StatusInternalServerError 92 | return 93 | } 94 | 95 | // 96 | // Get the nodes. 97 | // 98 | NodeList, err := getIndexNodes("") 99 | if err != nil { 100 | status = http.StatusInternalServerError 101 | return 102 | } 103 | 104 | // 105 | // The result 106 | // 107 | var result []string 108 | 109 | // 110 | // Add the hosts in the correct users' preferred state. 111 | // 112 | for _, o := range NodeList { 113 | if o.State == state { 114 | result = append(result, o.Fqdn) 115 | } 116 | } 117 | 118 | // 119 | // What kind of reply should we send? 120 | // 121 | // Accept either a "?accept=XXX" URL-parameter, or 122 | // the Accept HEADER in the HTTP request 123 | // 124 | accept := req.FormValue("accept") 125 | if len(accept) < 1 { 126 | accept = req.Header.Get("Accept") 127 | } 128 | 129 | switch accept { 130 | case "text/plain": 131 | res.Header().Set("Content-Type", "text/plain") 132 | 133 | for _, o := range result { 134 | fmt.Fprintf(res, "%s\n", o) 135 | } 136 | case "application/xml": 137 | x, err := xml.MarshalIndent(result, "", " ") 138 | if err != nil { 139 | status = http.StatusInternalServerError 140 | return 141 | } 142 | 143 | res.Header().Set("Content-Type", "application/xml") 144 | res.Write(x) 145 | default: 146 | 147 | // 148 | // Convert the string-array to JSON, and return it. 149 | // 150 | res.Header().Set("Content-Type", "application/json") 151 | 152 | if len(result) > 0 { 153 | out, _ := json.Marshal(result) 154 | fmt.Fprintf(res, "%s", out) 155 | } else { 156 | fmt.Fprintf(res, "[]") 157 | } 158 | 159 | } 160 | 161 | } 162 | 163 | // 164 | // RadiatorView is the handler for the HTTP end-point 165 | // 166 | // GET /radiator/ 167 | // 168 | // It will respond in either HTML, JSON, or XML depending on the 169 | // Accepts-header which is received. 170 | // 171 | func RadiatorView(res http.ResponseWriter, req *http.Request) { 172 | 173 | var ( 174 | status int 175 | err error 176 | ) 177 | defer func() { 178 | if nil != err { 179 | http.Error(res, err.Error(), status) 180 | } 181 | }() 182 | 183 | // anonymous struct 184 | type Pagedata struct { 185 | States []PuppetState 186 | Urlprefix string 187 | } 188 | 189 | // 190 | // Get the state of the nodes. 191 | // 192 | data, err := getStates("") 193 | if err != nil { 194 | status = http.StatusInternalServerError 195 | return 196 | } 197 | 198 | // 199 | // Sum up our known-nodes. 200 | // 201 | total := 0 202 | for i := range data { 203 | total += data[i].Count 204 | } 205 | 206 | // 207 | // Add in the total count of nodes. 208 | // 209 | var tmp PuppetState 210 | tmp.State = "Total" 211 | tmp.Count = total 212 | tmp.Percentage = 0 213 | data = append(data, tmp) 214 | 215 | // genereic template args 216 | var x Pagedata 217 | x.States = data 218 | x.Urlprefix = templateArgs.urlprefix 219 | 220 | // 221 | // What kind of reply should we send? 222 | // 223 | // Accept either a "?accept=XXX" URL-parameter, or 224 | // the Accept HEADER in the HTTP request 225 | // 226 | accept := req.FormValue("accept") 227 | if len(accept) < 1 { 228 | accept = req.Header.Get("Accept") 229 | } 230 | 231 | switch accept { 232 | case "application/json": 233 | js, err := json.Marshal(data) 234 | 235 | if err != nil { 236 | status = http.StatusInternalServerError 237 | return 238 | } 239 | res.Header().Set("Content-Type", "application/json") 240 | res.Write(js) 241 | 242 | case "application/xml": 243 | x, err := xml.MarshalIndent(data, "", " ") 244 | if err != nil { 245 | status = http.StatusInternalServerError 246 | return 247 | } 248 | 249 | res.Header().Set("Content-Type", "application/xml") 250 | res.Write(x) 251 | default: 252 | // 253 | // Load our template resource. 254 | // 255 | tmpl, err := getResource("data/radiator.template") 256 | if err != nil { 257 | fmt.Fprint(res, err.Error()) 258 | return 259 | } 260 | 261 | // 262 | // Load our template, from the resource. 263 | // 264 | src := string(tmpl) 265 | t := template.Must(template.New("tmpl").Parse(src)) 266 | 267 | // 268 | // Execute the template into our buffer. 269 | // 270 | buf := &bytes.Buffer{} 271 | err = t.Execute(buf, x) 272 | 273 | // 274 | // If there were errors, then show them. 275 | if err != nil { 276 | fmt.Fprint(res, err.Error()) 277 | return 278 | } 279 | 280 | // 281 | // Otherwise write the result. 282 | // 283 | buf.WriteTo(res) 284 | } 285 | } 286 | 287 | // 288 | // ReportSubmissionHandler is the handler for the HTTP end-point: 289 | // 290 | // POST /upload 291 | // 292 | // The input is read, and parsed as Yaml, and assuming that succeeds 293 | // then the data is written beneath ./reports/$hostname/$timestamp 294 | // and a summary-record is inserted into our SQLite database. 295 | // 296 | // 297 | func ReportSubmissionHandler(res http.ResponseWriter, req *http.Request) { 298 | var ( 299 | status int 300 | err error 301 | ) 302 | defer func() { 303 | if nil != err { 304 | http.Error(res, err.Error(), status) 305 | 306 | // Don't spam stdout when running test-cases. 307 | if flag.Lookup("test.v") == nil { 308 | fmt.Printf("Error: %s\n", err.Error()) 309 | } 310 | } 311 | }() 312 | 313 | // 314 | // Ensure this was a POST-request 315 | // 316 | if req.Method != "POST" { 317 | err = errors.New("must be called via HTTP-POST") 318 | status = http.StatusInternalServerError 319 | return 320 | } 321 | 322 | // 323 | // Read the body of the request. 324 | // 325 | content, err := ioutil.ReadAll(req.Body) 326 | if err != nil { 327 | status = http.StatusInternalServerError 328 | return 329 | } 330 | 331 | // 332 | // Parse the YAML into something we can work with. 333 | // 334 | report, err := ParsePuppetReport(content) 335 | if err != nil { 336 | status = http.StatusInternalServerError 337 | return 338 | } 339 | 340 | // 341 | // Create a report directory for this host, unless it already exists. 342 | // 343 | dir := filepath.Join(ReportPrefix, report.Fqdn) 344 | if !Exists(dir) { 345 | err = os.MkdirAll(dir, 0755) 346 | if err != nil { 347 | status = http.StatusInternalServerError 348 | return 349 | } 350 | } 351 | 352 | // 353 | // Does this report already exist? This shouldn't happen 354 | // in a usual setup, but will happen if you're repeatedly 355 | // importing reports manually from a puppet-server. 356 | // 357 | // (Which is something you might do when testing the dashboard.) 358 | // 359 | path := filepath.Join(dir, report.Hash) 360 | 361 | if Exists(path) { 362 | fmt.Fprintf(res, "Ignoring duplicate submission") 363 | return 364 | } 365 | 366 | // 367 | // Create the new report-file, on-disk. 368 | // 369 | err = ioutil.WriteFile(path, content, 0644) 370 | if err != nil { 371 | status = http.StatusInternalServerError 372 | return 373 | } 374 | 375 | // 376 | // Record that report in our SQLite database 377 | // 378 | relativePath := filepath.Join(report.Fqdn, report.Hash) 379 | 380 | addDB(report, relativePath) 381 | 382 | // 383 | // Show something to the caller. 384 | // 385 | out := fmt.Sprintf("{\"host\":\"%s\"}", report.Fqdn) 386 | fmt.Fprint(res, string(out)) 387 | 388 | } 389 | 390 | // 391 | // SearchHandler is the handler for the HTTP end-point: 392 | // 393 | // POST /search 394 | // 395 | // We perform a search for nodes matching a given pattern. The comparison 396 | // is a regular substring-match, rather than a regular expression. 397 | // 398 | func SearchHandler(res http.ResponseWriter, req *http.Request) { 399 | var ( 400 | status int 401 | err error 402 | ) 403 | defer func() { 404 | if nil != err { 405 | http.Error(res, err.Error(), status) 406 | 407 | // Don't spam stdout when running test-cases. 408 | if flag.Lookup("test.v") == nil { 409 | fmt.Printf("Error: %s\n", err.Error()) 410 | } 411 | } 412 | }() 413 | 414 | // 415 | // Ensure this was a POST-request 416 | // 417 | if req.Method != "POST" { 418 | err = errors.New("must be called via HTTP-POST") 419 | status = http.StatusInternalServerError 420 | return 421 | } 422 | 423 | // 424 | // Get the term from the form. 425 | // 426 | req.ParseForm() 427 | term := req.FormValue("term") 428 | 429 | // 430 | // Ensure we have a term. 431 | // 432 | if len(term) < 1 { 433 | err = errors.New("missing search term") 434 | status = http.StatusInternalServerError 435 | return 436 | } 437 | 438 | // 439 | // Annoying struct to allow us to populate our template 440 | // with both the matching nodes, and the term used for the search 441 | // 442 | type Pagedata struct { 443 | Nodes []PuppetRuns 444 | Term string 445 | Urlprefix string 446 | } 447 | 448 | // 449 | // Get all known nodes. 450 | // 451 | NodeList, err := getIndexNodes("") 452 | if err != nil { 453 | status = http.StatusInternalServerError 454 | return 455 | } 456 | 457 | // 458 | // Populate this structure with the search-term 459 | // 460 | var x Pagedata 461 | x.Term = term 462 | x.Urlprefix = templateArgs.urlprefix 463 | 464 | // 465 | // Add in any nodes which match our term. 466 | // 467 | for _, o := range NodeList { 468 | if strings.Contains(o.Fqdn, term) { 469 | x.Nodes = append(x.Nodes, o) 470 | } 471 | } 472 | 473 | // 474 | // Load our template source. 475 | // 476 | tmpl, err := getResource("data/results.template") 477 | if err != nil { 478 | fmt.Fprint(res, err.Error()) 479 | return 480 | } 481 | 482 | // 483 | // Load our template, from the resource. 484 | // 485 | src := string(tmpl) 486 | t := template.Must(template.New("tmpl").Parse(src)) 487 | 488 | // 489 | // Execute the template into our buffer. 490 | // 491 | buf := &bytes.Buffer{} 492 | err = t.Execute(buf, x) 493 | 494 | // 495 | // If there were errors, then show them. 496 | if err != nil { 497 | fmt.Fprint(res, err.Error()) 498 | return 499 | } 500 | 501 | // 502 | // Otherwise write the result. 503 | // 504 | buf.WriteTo(res) 505 | } 506 | 507 | // 508 | // ReportHandler is the handler for the HTTP end-point 509 | // 510 | // GET /report/NN 511 | // 512 | // It will respond in either HTML, JSON, or XML depending on the 513 | // Accepts-header which is received. 514 | // 515 | func ReportHandler(res http.ResponseWriter, req *http.Request) { 516 | var ( 517 | status int 518 | err error 519 | ) 520 | defer func() { 521 | if nil != err { 522 | http.Error(res, err.Error(), status) 523 | 524 | // Don't spam stdout when running test-cases. 525 | if flag.Lookup("test.v") == nil { 526 | fmt.Printf("Error: %s\n", err.Error()) 527 | } 528 | } 529 | }() 530 | 531 | // 532 | // Get the node name we're going to show. 533 | // 534 | vars := mux.Vars(req) 535 | id := vars["id"] 536 | 537 | // 538 | // Ensure we received a parameter. 539 | // 540 | if len(id) < 1 { 541 | status = http.StatusNotFound 542 | err = errors.New("missing 'id' parameter") 543 | return 544 | } 545 | 546 | // 547 | // If the ID is non-numeric we're in trouble. 548 | // 549 | reg, _ := regexp.Compile("^([0-9]+)$") 550 | if !reg.MatchString(id) { 551 | status = http.StatusInternalServerError 552 | err = errors.New("the report ID must be numeric") 553 | return 554 | } 555 | 556 | // 557 | // Get the content. 558 | // 559 | content, err := getYAML(ReportPrefix, id) 560 | if err != nil { 561 | status = http.StatusInternalServerError 562 | return 563 | } 564 | 565 | // need generic struct 566 | type Pagedata struct { 567 | Report PuppetReport 568 | Urlprefix string 569 | } 570 | 571 | // 572 | // Parse it 573 | // 574 | report, err := ParsePuppetReport(content) 575 | if err != nil { 576 | status = http.StatusInternalServerError 577 | return 578 | } 579 | 580 | var x Pagedata 581 | x.Report = report 582 | x.Urlprefix = templateArgs.urlprefix 583 | 584 | // 585 | // Accept either a "?accept=XXX" URL-parameter, or 586 | // the Accept HEADER in the HTTP request 587 | // 588 | accept := req.FormValue("accept") 589 | if len(accept) < 1 { 590 | accept = req.Header.Get("Accept") 591 | } 592 | 593 | switch accept { 594 | case "application/json": 595 | js, err := json.Marshal(report) 596 | 597 | if err != nil { 598 | status = http.StatusInternalServerError 599 | return 600 | } 601 | res.Header().Set("Content-Type", "application/json") 602 | res.Write(js) 603 | 604 | case "application/xml": 605 | x, err := xml.MarshalIndent(report, "", " ") 606 | if err != nil { 607 | status = http.StatusInternalServerError 608 | return 609 | } 610 | 611 | res.Header().Set("Content-Type", "application/xml") 612 | res.Write(x) 613 | default: 614 | 615 | // 616 | // Load our template resource. 617 | // 618 | tmpl, err := getResource("data/report.template") 619 | if err != nil { 620 | fmt.Fprint(res, err.Error()) 621 | return 622 | } 623 | 624 | // 625 | // Helper to allow a float to be truncated 626 | // to two/three digits. 627 | // 628 | funcMap := template.FuncMap{ 629 | 630 | "truncate": func(s string) string { 631 | 632 | // 633 | // Parse as a float. 634 | // 635 | f, _ := strconv.ParseFloat(s, 64) 636 | 637 | // 638 | // Output to a truncated string 639 | // 640 | s = fmt.Sprintf("%.2f", f) 641 | return s 642 | }, 643 | } 644 | 645 | // 646 | // Load our template, from the resource. 647 | // 648 | src := string(tmpl) 649 | t := template.Must(template.New("tmpl").Funcs(funcMap).Parse(src)) 650 | 651 | // 652 | // Execute the template into our buffer. 653 | // 654 | buf := &bytes.Buffer{} 655 | err = t.Execute(buf, x) 656 | 657 | // 658 | // If there were errors, then show them. 659 | if err != nil { 660 | fmt.Fprint(res, err.Error()) 661 | return 662 | } 663 | 664 | // 665 | // Otherwise write the result. 666 | // 667 | buf.WriteTo(res) 668 | } 669 | } 670 | 671 | // 672 | // NodeHandler is the handler for the HTTP end-point 673 | // 674 | // GET /node/$FQDN 675 | // 676 | // It will respond in either HTML, JSON, or XML depending on the 677 | // Accepts-header which is received. 678 | // 679 | func NodeHandler(res http.ResponseWriter, req *http.Request) { 680 | var ( 681 | status int 682 | err error 683 | ) 684 | defer func() { 685 | if nil != err { 686 | http.Error(res, err.Error(), status) 687 | 688 | // Don't spam stdout when running test-cases. 689 | if flag.Lookup("test.v") == nil { 690 | fmt.Printf("Error: %s\n", err.Error()) 691 | } 692 | } 693 | }() 694 | 695 | // 696 | // Get the node name we're going to show. 697 | // 698 | vars := mux.Vars(req) 699 | fqdn := vars["fqdn"] 700 | 701 | // 702 | // Ensure we received a parameter. 703 | // 704 | if len(fqdn) < 1 { 705 | status = http.StatusNotFound 706 | err = errors.New("missing 'fqdn' parameter") 707 | return 708 | } 709 | 710 | // 711 | // Get the reports 712 | // 713 | reports, err := getReports(fqdn) 714 | 715 | // 716 | // Ensure that something was present. 717 | // 718 | if (reports == nil) || (len(reports) < 1) { 719 | status = http.StatusNotFound 720 | return 721 | } 722 | 723 | // 724 | // Handle error(s) 725 | // 726 | if err != nil { 727 | status = http.StatusInternalServerError 728 | return 729 | } 730 | 731 | // 732 | // Annoying struct to allow us to populate our template 733 | // with both the reports and the fqdn of the host. 734 | // 735 | type Pagedata struct { 736 | Fqdn string 737 | Nodes []PuppetReportSummary 738 | Urlprefix string 739 | } 740 | 741 | // 742 | // Populate this structure. 743 | // 744 | var x Pagedata 745 | x.Nodes = reports 746 | x.Fqdn = fqdn 747 | x.Urlprefix = templateArgs.urlprefix 748 | 749 | // 750 | // Accept either a "?accept=XXX" URL-parameter, or 751 | // the Accept HEADER in the HTTP request 752 | // 753 | accept := req.FormValue("accept") 754 | if len(accept) < 1 { 755 | accept = req.Header.Get("Accept") 756 | } 757 | 758 | switch accept { 759 | case "application/json": 760 | js, err := json.Marshal(reports) 761 | 762 | if err != nil { 763 | status = http.StatusInternalServerError 764 | return 765 | } 766 | res.Header().Set("Content-Type", "application/json") 767 | res.Write(js) 768 | 769 | case "application/xml": 770 | x, err := xml.MarshalIndent(reports, "", " ") 771 | if err != nil { 772 | status = http.StatusInternalServerError 773 | return 774 | } 775 | 776 | res.Header().Set("Content-Type", "application/xml") 777 | res.Write(x) 778 | default: 779 | 780 | // 781 | // Load our template resource. 782 | // 783 | tmpl, err := getResource("data/node.template") 784 | if err != nil { 785 | fmt.Fprint(res, err.Error()) 786 | return 787 | } 788 | 789 | funcMap := template.FuncMap{ 790 | 791 | "incr": func(d int) string { 792 | 793 | // 794 | // Return the incremented string. 795 | // 796 | s := fmt.Sprintf("%d", (d + 1)) 797 | return s 798 | }, 799 | } 800 | 801 | // 802 | // Load our template, from the resource. 803 | // 804 | src := string(tmpl) 805 | t := template.Must(template.New("tmpl").Funcs(funcMap).Parse(src)) 806 | 807 | // 808 | // Execute the template into our buffer. 809 | // 810 | buf := &bytes.Buffer{} 811 | err = t.Execute(buf, x) 812 | 813 | // 814 | // If there were errors, then show them. 815 | if err != nil { 816 | fmt.Fprint(res, err.Error()) 817 | return 818 | } 819 | 820 | // 821 | // Otherwise write the result. 822 | // 823 | buf.WriteTo(res) 824 | } 825 | } 826 | 827 | // StaticHandler is responsible for returning the contents of 828 | // all our embedded resources to HTTP-clients. 829 | // 830 | // It is configured as 404-handler, and can look for resources, 831 | // serving those that are present, and returning genuine 404 832 | // responses for requests that are entirely unknown. 833 | func StaticHandler(res http.ResponseWriter, req *http.Request) { 834 | 835 | // 836 | // Get the path we're going to serve. 837 | // 838 | path := req.URL.Path 839 | 840 | // 841 | // Is this a static-resource we know about? 842 | // 843 | data, err := getResource("data" + path) 844 | if err != nil { 845 | res.WriteHeader(http.StatusNotFound) 846 | fmt.Fprintf(res, "Error loading the resource you requested: %s : %s", path, err.Error()) 847 | return 848 | } 849 | 850 | // 851 | // OK at this point we're handling a valid static-resource, 852 | // so we just need to get the content-type setup appropriately. 853 | // 854 | suffix := filepath.Ext(path) 855 | mType := mime.TypeByExtension(suffix) 856 | if mType != "" { 857 | res.Header().Set("Content-Type", mType) 858 | } 859 | res.Write(data) 860 | 861 | } 862 | 863 | // 864 | // IndexHandler is the handler for the HTTP end-point 865 | // 866 | // GET / 867 | // 868 | // It will respond in either HTML, JSON, or XML depending on the 869 | // Accepts-header which is received. 870 | // 871 | func IndexHandler(res http.ResponseWriter, req *http.Request) { 872 | var ( 873 | status int 874 | err error 875 | ) 876 | defer func() { 877 | if nil != err { 878 | http.Error(res, err.Error(), status) 879 | 880 | // Don't spam stdout when running test-cases. 881 | if flag.Lookup("test.v") == nil { 882 | fmt.Printf("Error: %s\n", err.Error()) 883 | } 884 | } 885 | }() 886 | 887 | // 888 | // Check if we are filtering by environment 889 | // 890 | vars := mux.Vars(req) 891 | environment := vars["environment"] 892 | 893 | // 894 | // Annoying struct to allow us to populate our template 895 | // with both the nodes in the list, and the graph-data 896 | // 897 | type Pagedata struct { 898 | Graph []PuppetHistory 899 | Nodes []PuppetRuns 900 | Environment string 901 | Environments []string 902 | Urlprefix string 903 | } 904 | 905 | // 906 | // Get the nodes to show on our front-page 907 | // 908 | NodeList, err := getIndexNodes(environment) 909 | if err != nil { 910 | status = http.StatusInternalServerError 911 | return 912 | } 913 | 914 | // 915 | // Get the graph-data 916 | // 917 | graphs, err := getHistory(environment, 30) 918 | if err != nil { 919 | status = http.StatusInternalServerError 920 | return 921 | } 922 | 923 | // 924 | // Get all environments 925 | environments, err := getEnvironments() 926 | 927 | // 928 | // Populate this structure. 929 | // 930 | var x Pagedata 931 | x.Graph = graphs 932 | x.Nodes = NodeList 933 | x.Environment = environment 934 | x.Environments = environments 935 | x.Urlprefix = templateArgs.urlprefix 936 | 937 | // 938 | // Accept either a "?accept=XXX" URL-parameter, or 939 | // the Accept HEADER in the HTTP request 940 | // 941 | accept := req.FormValue("accept") 942 | if len(accept) < 1 { 943 | accept = req.Header.Get("Accept") 944 | } 945 | 946 | switch accept { 947 | case "application/json": 948 | js, err := json.Marshal(NodeList) 949 | 950 | if err != nil { 951 | status = http.StatusInternalServerError 952 | return 953 | } 954 | res.Header().Set("Content-Type", "application/json") 955 | res.Write(js) 956 | 957 | case "application/xml": 958 | x, err := xml.MarshalIndent(NodeList, "", " ") 959 | if err != nil { 960 | status = http.StatusInternalServerError 961 | return 962 | } 963 | 964 | res.Header().Set("Content-Type", "application/xml") 965 | res.Write(x) 966 | default: 967 | 968 | // 969 | // Load our template source. 970 | // 971 | tmpl, err := getResource("data/index.template") 972 | if err != nil { 973 | fmt.Fprint(res, err.Error()) 974 | return 975 | } 976 | 977 | // 978 | // Load our template, from the resource. 979 | // 980 | src := string(tmpl) 981 | t := template.Must(template.New("tmpl").Parse(src)) 982 | 983 | // 984 | // Execute the template into our buffer. 985 | // 986 | buf := &bytes.Buffer{} 987 | err = t.Execute(buf, x) 988 | 989 | // 990 | // If there were errors, then show them. 991 | if err != nil { 992 | fmt.Fprint(res, err.Error()) 993 | return 994 | } 995 | 996 | // 997 | // Otherwise write the result. 998 | // 999 | buf.WriteTo(res) 1000 | } 1001 | } 1002 | 1003 | // 1004 | // Entry-point. 1005 | // 1006 | func serve(settings serveCmd) { 1007 | templateArgs.urlprefix = settings.urlprefix 1008 | 1009 | // 1010 | // Preserve our prefix 1011 | // 1012 | ReportPrefix = settings.prefix 1013 | 1014 | // 1015 | // Create a new router and our route-mappings. 1016 | // 1017 | router := mux.NewRouter() 1018 | 1019 | // 1020 | // Static-Files are handled via the 404-handler, 1021 | // as that is invoked when other routes don't match. 1022 | // 1023 | router.NotFoundHandler = http.HandlerFunc(StaticHandler) 1024 | 1025 | // 1026 | // API end-points 1027 | // 1028 | router.HandleFunc("/api/state/{state}/", APIState).Methods("GET") 1029 | router.HandleFunc("/api/state/{state}", APIState).Methods("GET") 1030 | 1031 | // 1032 | // 1033 | // 1034 | router.HandleFunc("/radiator/", RadiatorView).Methods("GET") 1035 | router.HandleFunc("/radiator", RadiatorView).Methods("GET") 1036 | 1037 | // 1038 | // Upload a new report. 1039 | // 1040 | router.HandleFunc("/upload/", ReportSubmissionHandler).Methods("POST") 1041 | router.HandleFunc("/upload", ReportSubmissionHandler).Methods("POST") 1042 | 1043 | // 1044 | // Search nodes. 1045 | // 1046 | router.HandleFunc("/search/", SearchHandler).Methods("POST") 1047 | router.HandleFunc("/search", SearchHandler).Methods("POST") 1048 | 1049 | // 1050 | // Show the recent state of a node. 1051 | // 1052 | router.HandleFunc("/node/{fqdn}/", NodeHandler).Methods("GET") 1053 | router.HandleFunc("/node/{fqdn}", NodeHandler).Methods("GET") 1054 | 1055 | // 1056 | // Show "everything" about a given run. 1057 | // 1058 | router.HandleFunc("/report/{id}/", ReportHandler).Methods("GET") 1059 | router.HandleFunc("/report/{id}", ReportHandler).Methods("GET") 1060 | 1061 | // 1062 | // Handle a display of all known nodes, and their last state. 1063 | // 1064 | router.HandleFunc("/", IndexHandler).Methods("GET") 1065 | // also do it for environments 1066 | router.HandleFunc("/environment/{environment}/", IndexHandler).Methods("GET") 1067 | router.HandleFunc("/environment/{environment}", IndexHandler).Methods("GET") 1068 | 1069 | // 1070 | // Bind the router. 1071 | // 1072 | http.Handle("/", router) 1073 | 1074 | // 1075 | // Show where we'll bind 1076 | // 1077 | bind := fmt.Sprintf("%s:%d", settings.bindHost, settings.bindPort) 1078 | fmt.Printf("Launching the server on http://%s\n", bind) 1079 | 1080 | // 1081 | // Wire up logging. 1082 | // 1083 | loggedRouter := handlers.LoggingHandler(os.Stdout, router) 1084 | 1085 | // 1086 | // We want to make sure we handle timeouts effectively by using 1087 | // a non-default http-server 1088 | // 1089 | srv := &http.Server{ 1090 | Addr: bind, 1091 | Handler: loggedRouter, 1092 | ReadTimeout: 300 * time.Second, 1093 | WriteTimeout: 300 * time.Second, 1094 | } 1095 | 1096 | // 1097 | // Launch the server. 1098 | // 1099 | err := srv.ListenAndServe() 1100 | if err != nil { 1101 | fmt.Printf("\nError: %s\n", err.Error()) 1102 | } 1103 | } 1104 | 1105 | // 1106 | // The options set by our command-line flags. 1107 | // 1108 | type serveCmd struct { 1109 | autoPrune bool 1110 | bindHost string 1111 | bindPort int 1112 | dbFile string 1113 | prefix string 1114 | urlprefix string 1115 | } 1116 | 1117 | type templateOptions struct { 1118 | urlprefix string 1119 | } 1120 | 1121 | var templateArgs templateOptions 1122 | 1123 | // 1124 | // Glue 1125 | // 1126 | func (*serveCmd) Name() string { return "serve" } 1127 | func (*serveCmd) Synopsis() string { return "Launch the HTTP server." } 1128 | func (*serveCmd) Usage() string { 1129 | return `serve [options]: 1130 | Launch the HTTP server for receiving reports & viewing them 1131 | ` 1132 | } 1133 | 1134 | // 1135 | // Flag setup 1136 | // 1137 | func (p *serveCmd) SetFlags(f *flag.FlagSet) { 1138 | f.IntVar(&p.bindPort, "port", 3001, "The port to bind upon.") 1139 | f.BoolVar(&p.autoPrune, "auto-prune", false, "Prune reports automatically, once per week.") 1140 | f.StringVar(&p.bindHost, "host", "127.0.0.1", "The IP to listen upon.") 1141 | f.StringVar(&p.dbFile, "db-file", "ps.db", "The SQLite database to use.") 1142 | f.StringVar(&p.prefix, "prefix", "./reports/", "The prefix to the local YAML hierarchy.") 1143 | f.StringVar(&p.urlprefix, "urlprefix", "", "The URL prefix for serving behind a proxy.") 1144 | } 1145 | 1146 | // 1147 | // Entry-point. 1148 | // 1149 | func (p *serveCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { 1150 | 1151 | // 1152 | // Setup the database, by opening a handle, and creating it if 1153 | // missing. 1154 | // 1155 | SetupDB(p.dbFile) 1156 | 1157 | // 1158 | // Check for entries with no environment 1159 | // 1160 | populateEnvironment(p.prefix) 1161 | 1162 | // 1163 | // If autoprune 1164 | // 1165 | if p.autoPrune { 1166 | 1167 | // 1168 | // Create a cron scheduler 1169 | // 1170 | c := cron.New() 1171 | 1172 | // 1173 | // Every seven days prune the reports. 1174 | // 1175 | c.AddFunc("@weekly", func() { 1176 | fmt.Printf("Automatically pruning old reports") 1177 | pruneReports("", p.prefix, 7, false) 1178 | }) 1179 | 1180 | // 1181 | // Launch the cron-scheduler. 1182 | // 1183 | c.Start() 1184 | } 1185 | 1186 | // 1187 | // Start the server 1188 | // 1189 | serve(*p) 1190 | 1191 | // 1192 | // All done. 1193 | // 1194 | return subcommands.ExitSuccess 1195 | } 1196 | -------------------------------------------------------------------------------- /cmd_serve_test.go: -------------------------------------------------------------------------------- 1 | // Simple testing of the HTTP-server 2 | package main 3 | 4 | import ( 5 | "bytes" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httptest" 10 | "net/url" 11 | "os" 12 | "strconv" 13 | "strings" 14 | "testing" 15 | "unicode" 16 | 17 | "github.com/gorilla/mux" 18 | ) 19 | 20 | // Report IDs must be numeric. Submit some bogus requests to 21 | // ensure they fail with a suitable error-message. 22 | func TestNonNumericReport(t *testing.T) { 23 | router := mux.NewRouter() 24 | router.HandleFunc("/report/{id}/", ReportHandler).Methods("GET") 25 | router.HandleFunc("/report/{id}", ReportHandler).Methods("GET") 26 | 27 | // Table driven test 28 | ids := []string{"/report/1a", "/report/steve", "/report/bob/", "/report/3a.3/"} 29 | 30 | for _, id := range ids { 31 | req, err := http.NewRequest("GET", id, nil) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | 36 | rr := httptest.NewRecorder() 37 | router.ServeHTTP(rr, req) 38 | 39 | // Check the status code is what we expect. 40 | if status := rr.Code; status != http.StatusInternalServerError { 41 | t.Errorf("Unexpected status-code: %v", status) 42 | } 43 | 44 | // Check the response body is what we expect. 45 | expected := "the report ID must be numeric\n" 46 | if rr.Body.String() != expected { 47 | t.Errorf("handler returned unexpected body: got '%v' want '%v'", 48 | rr.Body.String(), expected) 49 | } 50 | } 51 | } 52 | 53 | // API-state must use known values. Submit some bogus values to ensure 54 | // a suitable error is returned. 55 | func TestUknownAPIState(t *testing.T) { 56 | 57 | // Wire up the route 58 | r := mux.NewRouter() 59 | r.HandleFunc("/api/state/{state}", APIState).Methods("GET") 60 | r.HandleFunc("/api/state/{state}/", APIState).Methods("GET") 61 | 62 | // Get the test-server 63 | ts := httptest.NewServer(r) 64 | defer ts.Close() 65 | 66 | // These are all bogus 67 | states := []string{"foo", "bart", "liza", "moi kissa", "steve/"} 68 | 69 | for _, state := range states { 70 | url := ts.URL + "/api/state/" + state 71 | 72 | resp, err := http.Get(url) 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | 77 | // 78 | // Get the body 79 | // 80 | defer resp.Body.Close() 81 | body, err := ioutil.ReadAll(resp.Body) 82 | 83 | if err != nil { 84 | t.Errorf("Failed to read response-body %v\n", err) 85 | } 86 | 87 | content := string(body) 88 | if status := resp.StatusCode; status != http.StatusInternalServerError { 89 | t.Errorf("Unexpected status-code: %v", status) 90 | } 91 | if content != "invalid state supplied\n" { 92 | t.Fatalf("Unexpected body: '%s'", body) 93 | } 94 | } 95 | 96 | } 97 | 98 | // Test that our report-view returns content that seems reasonable, 99 | // in all three cases: 100 | // 101 | // - text/html 102 | // - application/json 103 | // - application/xml 104 | func TestReportView(t *testing.T) { 105 | 106 | // Create a fake database 107 | FakeDB() 108 | 109 | // Add some data. 110 | addFakeReports() 111 | 112 | // 113 | // We'll make one test for each supported content-type 114 | // 115 | type TestCase struct { 116 | Type string 117 | Response string 118 | } 119 | 120 | // 121 | // The tests 122 | // 123 | tests := []TestCase{ 124 | {"text/html", "Report of execution against www.steve.org.uk in production, at 2017-07-29 23:17:01"}, 125 | {"application/json", "\"State\":\"unchanged\","}, 126 | {"application/xml", "unchanged"}} 127 | 128 | // 129 | // Run each one. 130 | // 131 | for _, test := range tests { 132 | 133 | // 134 | // Create a router. 135 | // 136 | router := mux.NewRouter() 137 | router.HandleFunc("/report/{id}/", ReportHandler).Methods("GET") 138 | router.HandleFunc("/report/{id}", ReportHandler).Methods("GET") 139 | 140 | // 141 | // Get a valid report ID, and use it to build a URL. 142 | // 143 | id, _ := validReportID() 144 | url := fmt.Sprintf("/report/%d", id) 145 | 146 | // 147 | // Make the request, with the appropriate Accept: header 148 | // 149 | req, err := http.NewRequest("GET", url, nil) 150 | if err != nil { 151 | t.Fatal(err) 152 | } 153 | req.Header.Add("Accept", test.Type) 154 | 155 | // 156 | // Fake out the request 157 | // 158 | rr := httptest.NewRecorder() 159 | router.ServeHTTP(rr, req) 160 | 161 | // 162 | // Test the status-code is OK 163 | // 164 | if status := rr.Code; status != http.StatusOK { 165 | t.Errorf("Unexpected status-code: %v", status) 166 | } 167 | 168 | // 169 | // Test that the body contained our expected content. 170 | // 171 | if !strings.Contains(rr.Body.String(), test.Response) { 172 | t.Fatalf("Unexpected body: '%s'", rr.Body.String()) 173 | } 174 | } 175 | 176 | // 177 | // Cleanup here because otherwise later tests will 178 | // see an active/valid DB-handle. 179 | // 180 | db.Close() 181 | db = nil 182 | os.RemoveAll(path) 183 | } 184 | 185 | // API state must be known. 186 | func TestKnownAPIState(t *testing.T) { 187 | 188 | // Create a fake database 189 | FakeDB() 190 | 191 | // Add some data. 192 | addFakeNodes() 193 | 194 | // Wire up the router. 195 | r := mux.NewRouter() 196 | r.HandleFunc("/api/state/{state}", APIState).Methods("GET") 197 | r.HandleFunc("/api/state/{state}/", APIState).Methods("GET") 198 | 199 | // Get the test-server 200 | ts := httptest.NewServer(r) 201 | defer ts.Close() 202 | 203 | // 204 | // We'll make one test for each known-state 205 | // 206 | type TestCase struct { 207 | State string 208 | Response string 209 | } 210 | 211 | tests := []TestCase{ 212 | {"changed", "[\"foo.example.com\"]"}, 213 | {"unchanged", "[]"}, 214 | {"failed", "[\"bar.example.com\"]"}, 215 | {"orphaned", "[]"}} 216 | 217 | // 218 | // Run each one. 219 | // 220 | for _, test := range tests { 221 | 222 | // 223 | // Make the request 224 | // 225 | url := ts.URL + "/api/state/" + test.State 226 | 227 | resp, err := http.Get(url) 228 | if err != nil { 229 | t.Fatal(err) 230 | } 231 | 232 | // 233 | // Get the body 234 | // 235 | defer resp.Body.Close() 236 | body, err := ioutil.ReadAll(resp.Body) 237 | 238 | if err != nil { 239 | t.Errorf("Failed to read response-body %v\n", err) 240 | } 241 | 242 | content := string(body) 243 | 244 | if status := resp.StatusCode; status != http.StatusOK { 245 | t.Errorf("Unexpected status-code: %v", status) 246 | } 247 | if content != test.Response { 248 | t.Fatalf("Unexpected body: '%s'", body) 249 | } 250 | } 251 | 252 | // 253 | // Cleanup here because otherwise later tests will 254 | // see an active/valid DB-handle. 255 | // 256 | db.Close() 257 | db = nil 258 | os.RemoveAll(path) 259 | 260 | } 261 | 262 | // API state should accept XML, JSON, and plain-text 263 | func TestAPITypes(t *testing.T) { 264 | 265 | // Create a fake database 266 | FakeDB() 267 | 268 | // Add some data. 269 | addFakeNodes() 270 | 271 | // Wire up the router. 272 | r := mux.NewRouter() 273 | r.HandleFunc("/api/state/{state}", APIState).Methods("GET") 274 | r.HandleFunc("/api/state/{state}/", APIState).Methods("GET") 275 | 276 | // Get the test-server 277 | ts := httptest.NewServer(r) 278 | defer ts.Close() 279 | 280 | // 281 | // We'll make one test for each known-state 282 | // 283 | type TestCase struct { 284 | Type string 285 | Response string 286 | } 287 | 288 | tests := []TestCase{ 289 | {"application/json", "[\"foo.example.com\"]"}, 290 | {"application/xml", "foo.example.com"}, 291 | {"text/plain", "foo.example.com\n"}, 292 | {"", "[\"foo.example.com\"]"}, 293 | } 294 | 295 | // 296 | // Run each one. 297 | // 298 | for _, test := range tests { 299 | 300 | // 301 | // Make the request 302 | // 303 | url := ts.URL + "/api/state/changed?accept=" + test.Type 304 | 305 | resp, err := http.Get(url) 306 | if err != nil { 307 | t.Fatal(err) 308 | } 309 | 310 | // 311 | // Get the body 312 | // 313 | defer resp.Body.Close() 314 | body, err := ioutil.ReadAll(resp.Body) 315 | 316 | if err != nil { 317 | t.Errorf("Failed to read response-body %v\n", err) 318 | } 319 | 320 | content := string(body) 321 | 322 | if status := resp.StatusCode; status != http.StatusOK { 323 | t.Errorf("Unexpected status-code: %v", status) 324 | } 325 | if content != test.Response { 326 | t.Fatalf("Unexpected body for %s: '%s'", test.Type, body) 327 | } 328 | } 329 | 330 | // 331 | // Cleanup here because otherwise later tests will 332 | // see an active/valid DB-handle. 333 | // 334 | db.Close() 335 | db = nil 336 | os.RemoveAll(path) 337 | 338 | } 339 | 340 | // Searching must be done via a POST. 341 | func TestSearchMethod(t *testing.T) { 342 | 343 | req, err := http.NewRequest("GET", "/search", nil) 344 | if err != nil { 345 | t.Fatal(err) 346 | } 347 | 348 | rr := httptest.NewRecorder() 349 | handler := http.HandlerFunc(SearchHandler) 350 | 351 | handler.ServeHTTP(rr, req) 352 | 353 | // Check the status code is what we expect. 354 | if status := rr.Code; status != http.StatusInternalServerError { 355 | t.Errorf("Unexpected status-code: %v", status) 356 | } 357 | 358 | // Check the response body is what we expect. 359 | expected := "must be called via HTTP-POST\n" 360 | if rr.Body.String() != expected { 361 | t.Errorf("handler returned unexpected body: got '%v' want '%v'", 362 | rr.Body.String(), expected) 363 | } 364 | 365 | } 366 | 367 | // The search handler must have a term-parameter. 368 | func TestSearchEmpty(t *testing.T) { 369 | 370 | req, err := http.NewRequest("POST", "/search", bytes.NewReader(nil)) 371 | if err != nil { 372 | t.Fatal(err) 373 | } 374 | 375 | rr := httptest.NewRecorder() 376 | handler := http.HandlerFunc(SearchHandler) 377 | 378 | // Our handlers satisfy http.Handler, so we can call 379 | // their ServeHTTP method directly and pass in our 380 | // Request and ResponseRecorder. 381 | handler.ServeHTTP(rr, req) 382 | 383 | // Check the status code is what we expect. 384 | if status := rr.Code; status != http.StatusInternalServerError { 385 | t.Errorf("Unexpected status-code: %v", status) 386 | } 387 | 388 | // Check the response body is what we expect. 389 | expected := "missing search term\n" 390 | if rr.Body.String() != expected { 391 | t.Errorf("handler returned unexpected body: got '%v' want '%v'", 392 | rr.Body.String(), expected) 393 | } 394 | } 395 | 396 | // The search handler should run a search 397 | func TestSearch(t *testing.T) { 398 | 399 | // Create a fake database 400 | FakeDB() 401 | 402 | // Add some data. 403 | addFakeNodes() 404 | 405 | // The term we're going to search for: "example" 406 | data := url.Values{} 407 | data.Set("term", "example") 408 | 409 | req, err := http.NewRequest("POST", "/search", bytes.NewBufferString(data.Encode())) 410 | if err != nil { 411 | t.Fatal(err) 412 | } 413 | 414 | // 415 | // Ensure we're POSTing a FORM 416 | // 417 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 418 | req.Header.Add("Content-Length", strconv.Itoa(len(data.Encode()))) 419 | 420 | rr := httptest.NewRecorder() 421 | handler := http.HandlerFunc(SearchHandler) 422 | 423 | // Our handlers satisfy http.Handler, so we can call 424 | // their ServeHTTP method directly and pass in our 425 | // Request and ResponseRecorder. 426 | handler.ServeHTTP(rr, req) 427 | 428 | // Check the status code is what we expect. 429 | if status := rr.Code; status != http.StatusOK { 430 | t.Errorf("Unexpected status-code: %v", status) 431 | } 432 | 433 | // Check the response body is what we expect. 434 | expected := "/node/bar.example.com" 435 | if !strings.Contains(rr.Body.String(), expected) { 436 | t.Fatalf("Unexpected body: '%s'", rr.Body.String()) 437 | } 438 | 439 | // 440 | // Cleanup here because otherwise later tests will 441 | // see an active/valid DB-handle. 442 | // 443 | db.Close() 444 | db = nil 445 | os.RemoveAll(path) 446 | } 447 | 448 | // Submitting reports must be done via a POST. 449 | func TestUploadReportMethod(t *testing.T) { 450 | 451 | req, err := http.NewRequest("GET", "/upload", nil) 452 | if err != nil { 453 | t.Fatal(err) 454 | } 455 | 456 | rr := httptest.NewRecorder() 457 | handler := http.HandlerFunc(ReportSubmissionHandler) 458 | 459 | handler.ServeHTTP(rr, req) 460 | 461 | // Check the status code is what we expect. 462 | if status := rr.Code; status != http.StatusInternalServerError { 463 | t.Errorf("Unexpected status-code: %v", status) 464 | } 465 | 466 | // Check the response body is what we expect. 467 | expected := "must be called via HTTP-POST\n" 468 | if rr.Body.String() != expected { 469 | t.Errorf("handler returned unexpected body: got '%v' want '%v'", 470 | rr.Body.String(), expected) 471 | } 472 | 473 | } 474 | 475 | // Submitting a pre-cooked report should succeed. 476 | func TestUploadReport(t *testing.T) { 477 | 478 | // Create a fake database 479 | FakeDB() 480 | 481 | // Ensure we point our report-upload directory at 482 | // our temporary location. 483 | ReportPrefix = path 484 | 485 | // 486 | // Read the YAML file. 487 | // 488 | tmpl, err := getResource("data/valid.yaml") 489 | if err != nil { 490 | t.Fatal(err) 491 | } 492 | 493 | // 494 | // Call this two times. 495 | // 496 | count := 0 497 | 498 | // 499 | // The two expected results 500 | // 501 | expected := []string{"{\"host\":\"www.steve.org.uk\"}", "Ignoring duplicate submission"} 502 | 503 | for count < 2 { 504 | req, err := http.NewRequest("POST", "/upload", bytes.NewReader(tmpl)) 505 | if err != nil { 506 | t.Fatal(err) 507 | } 508 | 509 | rr := httptest.NewRecorder() 510 | handler := http.HandlerFunc(ReportSubmissionHandler) 511 | 512 | // Our handlers satisfy http.Handler, so we can call their ServeHTTP method 513 | // directly and pass in our Request and ResponseRecorder. 514 | handler.ServeHTTP(rr, req) 515 | 516 | // Check the status code is what we expect. 517 | if status := rr.Code; status != http.StatusOK { 518 | t.Errorf("Unexpected status-code: %v", status) 519 | } 520 | 521 | if rr.Body.String() != expected[count] { 522 | t.Errorf("Body was '%v' we wanted '%v'", 523 | rr.Body.String(), expected[count]) 524 | } 525 | 526 | count++ 527 | } 528 | 529 | // 530 | // Cleanup here because otherwise later tests will 531 | // see an active/valid DB-handle. 532 | // 533 | db.Close() 534 | db = nil 535 | os.RemoveAll(path) 536 | } 537 | 538 | // Submitting a pre-cooked report which is bogus should fail. 539 | func TestUploadBogusReport(t *testing.T) { 540 | 541 | // Create a fake database 542 | FakeDB() 543 | 544 | // Ensure we point our report-upload directory at 545 | // our temporary location. 546 | ReportPrefix = path 547 | 548 | // 549 | // Read the YAML file. 550 | // 551 | tmpl, err := getResource("data/valid.yaml") 552 | if err != nil { 553 | t.Fatal(err) 554 | } 555 | 556 | // 557 | // Upper-case the YAML 558 | // 559 | for i := range tmpl { 560 | tmpl[i] = byte(unicode.ToUpper(rune(tmpl[i]))) 561 | } 562 | 563 | req, err := http.NewRequest("POST", "/upload", bytes.NewReader(tmpl)) 564 | if err != nil { 565 | t.Fatal(err) 566 | } 567 | 568 | rr := httptest.NewRecorder() 569 | handler := http.HandlerFunc(ReportSubmissionHandler) 570 | 571 | // Our handlers satisfy http.Handler, so we can call their ServeHTTP method 572 | // directly and pass in our Request and ResponseRecorder. 573 | handler.ServeHTTP(rr, req) 574 | 575 | // Check the status code is what we expect. 576 | if status := rr.Code; status != http.StatusInternalServerError { 577 | t.Errorf("Unexpected status-code: %v", status) 578 | } 579 | 580 | // Check the response body is what we expect. 581 | expected := "failed to get 'host' from YAML\n" 582 | if rr.Body.String() != expected { 583 | t.Errorf("handler returned unexpected body: got '%v' want '%v'", 584 | rr.Body.String(), expected) 585 | } 586 | 587 | // 588 | // Cleanup here because otherwise later tests will 589 | // see an active/valid DB-handle. 590 | // 591 | db.Close() 592 | db = nil 593 | os.RemoveAll(path) 594 | } 595 | 596 | // Unknown-nodes are handled. 597 | func TestUnknownNode(t *testing.T) { 598 | 599 | // Create a fake database 600 | FakeDB() 601 | 602 | // Add some data. 603 | addFakeNodes() 604 | 605 | // Wire up the router. 606 | r := mux.NewRouter() 607 | r.HandleFunc("/node/{fqdn}", NodeHandler).Methods("GET") 608 | 609 | // Get the test-server 610 | ts := httptest.NewServer(r) 611 | defer ts.Close() 612 | 613 | // 614 | // Test a bogus name. 615 | // 616 | url := ts.URL + "/node/missing.invalid.tld" 617 | 618 | resp, err := http.Get(url) 619 | if err != nil { 620 | t.Fatal(err) 621 | } 622 | 623 | // 624 | // Get the body 625 | // 626 | defer resp.Body.Close() 627 | body, err := ioutil.ReadAll(resp.Body) 628 | 629 | if err != nil { 630 | t.Errorf("Failed to read response-body %v\n", err) 631 | } 632 | 633 | content := string(body) 634 | 635 | if status := resp.StatusCode; status != http.StatusNotFound { 636 | t.Errorf("Unexpected status-code: %v", status) 637 | } 638 | if content != "Failed to find reports for missing.invalid.tld\n" { 639 | t.Fatalf("Unexpected body: '%s'", body) 640 | } 641 | 642 | // 643 | // Cleanup here because otherwise later tests will 644 | // see an active/valid DB-handle. 645 | // 646 | db.Close() 647 | db = nil 648 | os.RemoveAll(path) 649 | 650 | } 651 | 652 | // Test that our node-view returns content that seems reasonable, 653 | // in all three cases: 654 | // 655 | // - text/html 656 | // - application/json 657 | // - application/xml 658 | func TestKnownNode(t *testing.T) { 659 | 660 | // Create a fake database 661 | FakeDB() 662 | 663 | // Add some data. 664 | addFakeNodes() 665 | 666 | // 667 | // We'll make one test for each supported content-type 668 | // 669 | type TestCase struct { 670 | Type string 671 | Response string 672 | } 673 | 674 | // 675 | // The tests 676 | // 677 | tests := []TestCase{ 678 | {"text/html", "3.134"}, 679 | {"application/json", "\"State\":\"unchanged\","}, 680 | {"application/xml", ""}} 681 | 682 | // 683 | // Run each one. 684 | // 685 | for _, test := range tests { 686 | 687 | // 688 | // Create a router. 689 | // 690 | router := mux.NewRouter() 691 | router.HandleFunc("/node/{fqdn}/", NodeHandler).Methods("GET") 692 | router.HandleFunc("/node/{fqdn}", NodeHandler).Methods("GET") 693 | 694 | // 695 | // Make the request, with the appropriate Accept: header 696 | // 697 | req, err := http.NewRequest("GET", "/node/foo.example.com", nil) 698 | if err != nil { 699 | t.Fatal(err) 700 | } 701 | req.Header.Add("Accept", test.Type) 702 | 703 | // 704 | // Fake out the request 705 | // 706 | rr := httptest.NewRecorder() 707 | router.ServeHTTP(rr, req) 708 | 709 | // 710 | // Test the status-code is OK 711 | // 712 | if status := rr.Code; status != http.StatusOK { 713 | t.Errorf("Unexpected status-code: %v", status) 714 | } 715 | 716 | // 717 | // Test that the body contained our expected content. 718 | // 719 | if !strings.Contains(rr.Body.String(), test.Response) { 720 | t.Fatalf("Unexpected body: '%s'", rr.Body.String()) 721 | } 722 | } 723 | 724 | // 725 | // Cleanup here because otherwise later tests will 726 | // see an active/valid DB-handle. 727 | // 728 | db.Close() 729 | db = nil 730 | os.RemoveAll(path) 731 | 732 | } 733 | 734 | // Test that our index-view returns content that seems reasonable, 735 | // in all three cases: 736 | // 737 | // - text/html 738 | // - application/json 739 | // - application/xml 740 | func TestIndexView(t *testing.T) { 741 | 742 | // Create a fake database 743 | FakeDB() 744 | 745 | // Add some data. 746 | addFakeNodes() 747 | 748 | // 749 | // We'll make one test for each supported content-type 750 | // 751 | type TestCase struct { 752 | Type string 753 | Response string 754 | } 755 | 756 | // 757 | // The tests 758 | // 759 | tests := []TestCase{ 760 | {"text/html", "foo.example.com"}, 761 | {"application/json", "\"State\":\"failed\","}, 762 | {"application/xml", ""}} 763 | 764 | // 765 | // Run each one. 766 | // 767 | for _, test := range tests { 768 | 769 | // 770 | // Make the request, with the appropriate Accept: header 771 | // 772 | req, err := http.NewRequest("GET", "/", nil) 773 | if err != nil { 774 | t.Fatal(err) 775 | } 776 | req.Header.Add("Accept", test.Type) 777 | 778 | // 779 | // Fake it out 780 | // 781 | rr := httptest.NewRecorder() 782 | handler := http.HandlerFunc(IndexHandler) 783 | handler.ServeHTTP(rr, req) 784 | 785 | // 786 | // Test the status-code is OK 787 | // 788 | if status := rr.Code; status != http.StatusOK { 789 | t.Errorf("Unexpected status-code: %v", status) 790 | } 791 | 792 | // 793 | // Test that the body contained our expected content. 794 | // 795 | if !strings.Contains(rr.Body.String(), test.Response) { 796 | t.Fatalf("Unexpected body: '%s'", rr.Body.String()) 797 | } 798 | } 799 | 800 | // 801 | // Cleanup here because otherwise later tests will 802 | // see an active/valid DB-handle. 803 | // 804 | db.Close() 805 | db = nil 806 | os.RemoveAll(path) 807 | 808 | } 809 | 810 | // Test that static-resources work: 811 | // 812 | // 1. They produce content 813 | // 2. They have sensible MIME-types 814 | func TestStaticResources(t *testing.T) { 815 | 816 | // Test-cases 817 | type TestCase struct { 818 | path string 819 | mime string 820 | } 821 | 822 | tests := []TestCase{ 823 | TestCase{path: "/favicon.ico", mime: "image/vnd.microsoft.icon"}, 824 | TestCase{path: "/robots.txt", mime: "text/plain"}, 825 | TestCase{path: "/js/jquery.tablesorter.min.js", mime: "text/javascript"}, 826 | TestCase{path: "/css/bootstrap.min.css", mime: "text/css"}, 827 | TestCase{path: "/fonts/glyphicons-halflings-regular.woff2", mime: "font/woff2"}, 828 | } 829 | 830 | // Wire up the router. 831 | r := mux.NewRouter() 832 | r.NotFoundHandler = http.HandlerFunc(StaticHandler) 833 | 834 | for _, test := range tests { 835 | 836 | // Get the test-server 837 | ts := httptest.NewServer(r) 838 | defer ts.Close() 839 | 840 | // Make a request 841 | url := ts.URL + test.path 842 | 843 | resp, err := http.Get(url) 844 | if err != nil { 845 | t.Fatal(err) 846 | } 847 | 848 | defer resp.Body.Close() 849 | body, err := ioutil.ReadAll(resp.Body) 850 | 851 | if err != nil { 852 | t.Errorf("Failed to read response-body %v\n", err) 853 | } 854 | 855 | if len(body) < 10 { 856 | t.Errorf("too-short body reading %s: %d\n", test.path, len(body)) 857 | } 858 | 859 | // 860 | // Test that the content-type was what we expect. 861 | // 862 | headers := resp.Header 863 | ctype := headers["Content-Type"][0] 864 | 865 | // Content-type might have a character-set, so we can 866 | // expect either of these: 867 | // 868 | // Content-Type: text/plain 869 | // Content-Type: text/css; charset=utf-8 870 | // 871 | // Strip anything after the ";" to avoid caring about this 872 | if strings.Contains(ctype, ";") { 873 | pieces := strings.Split(ctype, ";") 874 | ctype = pieces[0] 875 | } 876 | 877 | if ctype != test.mime { 878 | t.Errorf("expected %s for %s - got %s", test.mime, test.path, ctype) 879 | } 880 | } 881 | } 882 | 883 | // Test that our radiator-view returns content that seems reasonable, 884 | // in all three cases: 885 | // 886 | // - text/html 887 | // - application/json 888 | // - application/xml 889 | func TestRadiatorView(t *testing.T) { 890 | 891 | // Create a fake database 892 | FakeDB() 893 | 894 | // Add some data. 895 | addFakeNodes() 896 | 897 | // 898 | // We'll make one test for each supported content-type 899 | // 900 | type TestCase struct { 901 | Type string 902 | Response string 903 | } 904 | 905 | // 906 | // The tests 907 | // 908 | tests := []TestCase{ 909 | {"text/html", "

"}, 910 | {"application/json", "\"State\":\"failed\","}, 911 | {"application/xml", ""}} 912 | 913 | // 914 | // Run each one. 915 | // 916 | for _, test := range tests { 917 | 918 | // 919 | // Make the request, with the appropriate Accept: header 920 | // 921 | req, err := http.NewRequest("GET", "/radiator/", nil) 922 | if err != nil { 923 | t.Fatal(err) 924 | } 925 | req.Header.Add("Accept", test.Type) 926 | 927 | // 928 | // Fake it out 929 | // 930 | rr := httptest.NewRecorder() 931 | handler := http.HandlerFunc(RadiatorView) 932 | handler.ServeHTTP(rr, req) 933 | 934 | // 935 | // Test the status-code is OK 936 | // 937 | if status := rr.Code; status != http.StatusOK { 938 | t.Errorf("Unexpected status-code: %v", status) 939 | } 940 | 941 | // 942 | // Test that the body contained our expected content. 943 | // 944 | if !strings.Contains(rr.Body.String(), test.Response) { 945 | t.Fatalf("Unexpected body: '%s'", rr.Body.String()) 946 | } 947 | } 948 | 949 | // 950 | // Cleanup here because otherwise later tests will 951 | // see an active/valid DB-handle. 952 | // 953 | db.Close() 954 | db = nil 955 | os.RemoveAll(path) 956 | 957 | } 958 | -------------------------------------------------------------------------------- /cmd_version.go: -------------------------------------------------------------------------------- 1 | // 2 | // Show our version - This uses a level of indirection for our test-case 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "fmt" 11 | "io" 12 | "os" 13 | "runtime" 14 | 15 | "github.com/google/subcommands" 16 | ) 17 | 18 | // 19 | // modified during testing 20 | // 21 | var out io.Writer = os.Stdout 22 | 23 | var ( 24 | version = "unreleased" 25 | ) 26 | 27 | type versionCmd struct { 28 | verbose bool 29 | } 30 | 31 | // 32 | // Glue 33 | // 34 | func (*versionCmd) Name() string { return "version" } 35 | func (*versionCmd) Synopsis() string { return "Show our version." } 36 | func (*versionCmd) Usage() string { 37 | return `version : 38 | Report upon our version, and exit. 39 | ` 40 | } 41 | 42 | // 43 | // Flag setup 44 | // 45 | func (p *versionCmd) SetFlags(f *flag.FlagSet) { 46 | f.BoolVar(&p.verbose, "verbose", false, "Show go version the binary was generated with.") 47 | } 48 | 49 | // 50 | // Show the version - using the "out"-writer. 51 | // 52 | func showVersion(verbose bool) { 53 | fmt.Fprintf(out, "%s\n", version) 54 | if verbose { 55 | fmt.Fprintf(out, "Built with %s\n", runtime.Version()) 56 | } 57 | } 58 | 59 | // 60 | // Entry-point. 61 | // 62 | func (p *versionCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { 63 | 64 | showVersion(p.verbose) 65 | return subcommands.ExitSuccess 66 | } 67 | -------------------------------------------------------------------------------- /cmd_version_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "runtime" 7 | "testing" 8 | ) 9 | 10 | func TestVersion(t *testing.T) { 11 | bak := out 12 | out = new(bytes.Buffer) 13 | defer func() { out = bak }() 14 | 15 | // 16 | // Expected 17 | // 18 | expected := "unreleased\n" 19 | 20 | s := versionCmd{} 21 | s.Execute(context.TODO(), nil) 22 | if out.(*bytes.Buffer).String() != expected { 23 | t.Errorf("Expected '%s' received '%s'", expected, out) 24 | } 25 | } 26 | 27 | func TestVersionVerbose(t *testing.T) { 28 | bak := out 29 | out = new(bytes.Buffer) 30 | defer func() { out = bak }() 31 | 32 | // 33 | // Expected 34 | // 35 | expected := "unreleased\nBuilt with " + runtime.Version() + "\n" 36 | 37 | s := versionCmd{verbose: true} 38 | s.Execute(context.TODO(), nil) 39 | if out.(*bytes.Buffer).String() != expected { 40 | t.Errorf("Expected '%s' received '%s'", expected, out) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /cmd_yaml.go: -------------------------------------------------------------------------------- 1 | // 2 | // Show a YAML file, interactively 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "fmt" 11 | "io/ioutil" 12 | 13 | "github.com/google/subcommands" 14 | ) 15 | 16 | // 17 | // The options set by our command-line flags. 18 | // 19 | type yamlCmd struct { 20 | } 21 | 22 | // 23 | // Glue 24 | // 25 | func (*yamlCmd) Name() string { return "yaml" } 26 | func (*yamlCmd) Synopsis() string { return "Show a summary of a YAML report." } 27 | func (*yamlCmd) Usage() string { 28 | return `yaml file1 file2 .. fileN: 29 | Show a summary of the specified YAML reports. 30 | ` 31 | } 32 | 33 | // 34 | // Flag setup: NOP 35 | // 36 | func (p *yamlCmd) SetFlags(f *flag.FlagSet) { 37 | } 38 | 39 | // 40 | // YamlDump parses the given file, and then dumps appropriate information 41 | // from the give report. 42 | // 43 | func YamlDump(file string) { 44 | content, _ := ioutil.ReadFile(file) 45 | node, err := ParsePuppetReport(content) 46 | if err != nil { 47 | fmt.Printf("Failed to read %s, %v\n", file, err) 48 | return 49 | } 50 | 51 | fmt.Printf("Hostname: %s\n", node.Fqdn) 52 | fmt.Printf("Reported: %s\n", node.At) 53 | fmt.Printf("State : %s\n", node.State) 54 | fmt.Printf("Runtime : %s\n", node.Runtime) 55 | 56 | fmt.Printf("\nResources\n") 57 | fmt.Printf("\tFailed : %s\n", node.Failed) 58 | fmt.Printf("\tChanged: %s\n", node.Changed) 59 | fmt.Printf("\tSkipped: %s\n", node.Skipped) 60 | fmt.Printf("\tTotal : %s\n", node.Total) 61 | 62 | if node.Failed != "0" { 63 | fmt.Printf("\nFailed:\n") 64 | for i := range node.ResourcesFailed { 65 | fmt.Printf("\t%s\n", node.ResourcesFailed[i].Name) 66 | fmt.Printf("\t\t%s:%s\n", node.ResourcesFailed[i].File, node.ResourcesFailed[i].Line) 67 | } 68 | } 69 | 70 | if node.Changed != "0" { 71 | fmt.Printf("\nChanged:\n") 72 | for i := range node.ResourcesChanged { 73 | fmt.Printf("\t%s\n", node.ResourcesChanged[i].Name) 74 | fmt.Printf("\t\t%s:%s\n", node.ResourcesChanged[i].File, node.ResourcesChanged[i].Line) 75 | } 76 | } 77 | 78 | if node.Skipped != "0" { 79 | fmt.Printf("\nSkipped:\n") 80 | for i := range node.ResourcesSkipped { 81 | fmt.Printf("\t%s\n", node.ResourcesSkipped[i].Name) 82 | fmt.Printf("\t\t%s:%s\n", node.ResourcesSkipped[i].File, node.ResourcesSkipped[i].Line) 83 | } 84 | } 85 | 86 | } 87 | 88 | // 89 | // Entry-point. 90 | // 91 | func (p *yamlCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { 92 | 93 | // 94 | // Show each file. 95 | // 96 | for _, arg := range f.Args() { 97 | YamlDump(arg) 98 | } 99 | 100 | // 101 | // All done. 102 | // 103 | return subcommands.ExitSuccess 104 | } 105 | -------------------------------------------------------------------------------- /data/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skx/puppet-summary/7851132a898b174d31f0059b4e629008a676ca3e/data/favicon.ico -------------------------------------------------------------------------------- /data/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skx/puppet-summary/7851132a898b174d31f0059b4e629008a676ca3e/data/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /data/index.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Node List 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 99 | 100 | 101 |

141 |
142 | 143 |

Node Summary{{if ne .Environment "" }} for environment: {{.Environment}}{{ end }}

144 | 145 |
146 |

 

147 | 148 | 155 | 156 | 157 |
158 | 159 |
160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | {{range .Nodes }} 170 | 175 | 176 | 177 | 178 | 179 | 180 | {{end}} 181 |
NodeEnvironmentStateSeen
{{.Fqdn}}{{.Environment}}{{.State}}{{.Ago}}
182 |
183 | 184 | 185 |
186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | {{range .Nodes }} 196 | {{if eq .State "failed" }} 197 | 198 | 199 | 200 | 201 | 202 | 203 | {{end}} 204 | {{end}} 205 |
NodeEnvironmentStateSeen
{{.Fqdn}}{{.Environment}}{{.State}}{{.Ago}}
206 |
207 | 208 | 209 |
210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | {{range .Nodes }} 220 | {{if eq .State "changed" }} 221 | 222 | 223 | 224 | 225 | 226 | 227 | {{end}} 228 | {{end}} 229 |
NodeEnvironmentStateSeen
{{.Fqdn}}{{.Environment}}{{.State}}{{.Ago}}
230 |
231 | 232 | 233 |
234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | {{range .Nodes }} 244 | {{if eq .State "unchanged" }} 245 | 246 | 247 | 248 | 249 | 250 | 251 | {{end}} 252 | {{end}} 253 |
NodeEnvironmentStateSeen
{{.Fqdn}}{{.Environment}}{{.State}}{{.Ago}}
254 |
255 | 256 | 257 |
258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | {{range .Nodes }} 268 | {{if eq .State "orphaned" }} 269 | 270 | 271 | 272 | 273 | 274 | 275 | {{end}} 276 | {{end}} 277 |
NodeEnvironmentStateSeen
{{.Fqdn}}{{.Environment}}{{.State}}{{.Ago}}
278 |

 

279 |

Orphaned nodes are those which have not submitted a report to the puppet-master "recently", they will gradually fall off the display as reports are pruned.

280 |
281 | 282 |
283 |
284 |

 

285 |

 

286 |
287 | 306 | 346 | 347 | 348 | -------------------------------------------------------------------------------- /data/node.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {{.Fqdn}} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 69 | 70 | 71 | 100 |
101 |

{{.Fqdn}}

102 | 103 |

 

104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | {{range $i, $e := .Nodes}} 116 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | {{end}} 130 |
IDNodeEnvironmentStatusSeenFailedChangedTotal
{{incr $i}}{{.Fqdn}}{{.Environment}}{{.State}}{{.Ago}}{{.Failed}}{{.Changed}}{{.Total}}
131 |
132 |

 

133 |

 

134 |
135 | 154 | 225 | 226 | 227 | -------------------------------------------------------------------------------- /data/radiator.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Radiator View 5 | 6 | 7 | 8 | 9 | 10 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | {{range .States }} 183 | 184 | 185 | 195 | 196 | {{end}} 197 |
Puppet Summary ✓

{{.Count}}

 186 |
187 |

188 | 189 | {{.State}} 190 | 191 |

192 |

{{.State}}

193 |
194 |
198 | 226 | 227 | 228 | 229 | -------------------------------------------------------------------------------- /data/report.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Puppet Report {{ .Report.Fqdn }} 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 42 |
43 |

Overview

44 |
45 |
46 |
47 |
48 |
49 |

Report of execution against {{ .Report.Fqdn }} in {{ .Report.Environment }}, at {{ .Report.At }}:

50 | 51 | 52 | 53 | 54 | 55 |
Changed {{ .Report.Changed }}
Skipped {{ .Report.Skipped }}
Failed {{ .Report.Failed }}
Total {{ .Report.Total }}
56 |

This run took {{truncate .Report.Runtime }} seconds to complete.

57 | 58 |
59 |
60 |
61 | 62 |

Logs

63 |
64 |
65 |
66 |
67 |
68 | {{range .Report.LogMessages}} 69 | 
{{.}}
70 | {{else}} 71 |

Nothing reported.

72 | {{end}} 73 |
74 |
75 |
76 | 77 | {{if .Report.ResourcesFailed }} 78 |

Failed

79 |
80 |
81 |
82 |
83 |
84 |
    85 | {{range .Report.ResourcesFailed}} 86 |
  • {{.Type}}: {{.Name}} 87 |
      88 |
    • {{.File}}:{{.Line}}
      • 89 |
    • 90 | {{end}} 91 |
92 |
93 |
94 |
95 | {{end}} 96 | 97 | {{if .Report.ResourcesChanged }} 98 |

Changed

99 |
100 |
101 |
102 |
103 |
104 |
    105 | {{range .Report.ResourcesChanged}} 106 |
  • {{.Type}}: {{.Name}} 107 |
      108 |
    • {{.File}}:{{.Line}}
      • 109 |
    • 110 | {{end}} 111 |
112 |
113 |
114 |
115 | {{end}} 116 | 117 | {{if .Report.ResourcesSkipped}} 118 |

Skipped

119 |
120 |
121 |
122 |
123 |
124 |
    125 | {{range .Report.ResourcesSkipped}} 126 |
  • {{.Type}}: {{.Name}} 127 |
      128 |
    • {{.File}}:{{.Line}}
      • 129 |
    • 130 | {{end}} 131 |
132 |
133 |
134 |
135 | {{end}} 136 | 137 | 138 | {{if .Report.ResourcesOK }} 139 |

Unchanged

140 | 156 | {{end}} 157 | 158 |
159 |

 

160 |

 

161 |
162 | 181 | 194 | 195 | 196 | -------------------------------------------------------------------------------- /data/results.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Search Results 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 43 |
44 | 45 |

Search Results

46 |

 

47 | 48 | {{if .Nodes }} 49 | 52 | 53 |
54 | 55 |
56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | {{range .Nodes }} 65 | 70 | 71 | 72 | 73 | 74 | {{end}} 75 |
NodeStateSeen
{{.Fqdn}}{{.State}}{{.Ago}}
76 |
77 |
78 | {{else}} 79 |

No nodes were found, matching the pattern {{.Term}}.

80 | {{end}} 81 |
82 |

 

83 |

 

84 |
85 | 104 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /data/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / 3 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | // 2 | // This package contains our SQLite DB interface. It is a little ropy. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "database/sql" 9 | "errors" 10 | "fmt" 11 | "io/ioutil" 12 | "os" 13 | "path/filepath" 14 | "regexp" 15 | "sort" 16 | "strconv" 17 | "time" 18 | 19 | _ "github.com/mattn/go-sqlite3" 20 | ) 21 | 22 | // 23 | // The global DB handle. 24 | // 25 | var db *sql.DB 26 | 27 | // 28 | // PuppetRuns is the structure which is used to list a summary of puppet 29 | // runs on the front-page. 30 | // 31 | type PuppetRuns struct { 32 | Fqdn string 33 | Environment string 34 | State string 35 | At string 36 | Epoch string 37 | Ago string 38 | Runtime string 39 | } 40 | 41 | // 42 | // PuppetReportSummary is the structure used to represent a series 43 | // of puppet-runs against a particular node. 44 | // 45 | type PuppetReportSummary struct { 46 | ID string 47 | Fqdn string 48 | Environment string 49 | State string 50 | At string 51 | Ago string 52 | Runtime string 53 | Failed int 54 | Changed int 55 | Total int 56 | YamlFile string 57 | } 58 | 59 | // 60 | // PuppetHistory is a simple structure used solely for the stacked-graph 61 | // on the front-page of our site. 62 | // 63 | type PuppetHistory struct { 64 | Date string 65 | Failed string 66 | Changed string 67 | Unchanged string 68 | } 69 | 70 | // 71 | // PuppetState is used to return the number of nodes in a given state, 72 | // and is used for the submission of metrics. 73 | // 74 | type PuppetState struct { 75 | State string 76 | Count int 77 | Percentage float64 78 | } 79 | 80 | // 81 | // SetupDB opens our SQLite database, creating it if necessary. 82 | // 83 | func SetupDB(path string) error { 84 | 85 | var err error 86 | 87 | // 88 | // Return if the database already exists. 89 | // 90 | db, err = sql.Open("sqlite3", "file:"+path+"?_journal_mode=WAL") 91 | if err != nil { 92 | return err 93 | } 94 | 95 | // 96 | // Create the table. 97 | // 98 | sqlStmt := ` 99 | 100 | PRAGMA automatic_index = ON; 101 | PRAGMA cache_size = 32768; 102 | PRAGMA journal_size_limit = 67110000; 103 | PRAGMA locking_mode = NORMAL; 104 | PRAGMA synchronous = NORMAL; 105 | PRAGMA temp_store = MEMORY; 106 | PRAGMA journal_mode = WAL; 107 | PRAGMA wal_autocheckpoint = 16384; 108 | 109 | CREATE TABLE IF NOT EXISTS reports ( 110 | id INTEGER PRIMARY KEY AUTOINCREMENT, 111 | fqdn text, 112 | environment text, 113 | state text, 114 | yaml_file text, 115 | runtime integer, 116 | executed_at integer(4), 117 | total integer, 118 | skipped integer, 119 | failed integer, 120 | changed integer 121 | ) 122 | ` 123 | 124 | // 125 | // Create the table, if missing. 126 | // 127 | // Errors here are pretty unlikely. 128 | // 129 | _, err = db.Exec(sqlStmt) 130 | if err != nil { 131 | return err 132 | } 133 | 134 | // 135 | // Check if the table has environment column 136 | // 137 | var name string 138 | row := db.QueryRow("SELECT name FROM pragma_table_info('reports') WHERE name='environment'") 139 | err = row.Scan(&name) 140 | if err != nil { 141 | if err == sql.ErrNoRows { 142 | fmt.Println("Did not find environment column, adding") 143 | _, err = db.Exec("ALTER TABLE reports ADD environment text") 144 | if err != nil { 145 | return err 146 | } 147 | } else { 148 | return err 149 | } 150 | } 151 | return nil 152 | } 153 | 154 | // 155 | // Populate environment column after adding it 156 | // 157 | func populateEnvironment(prefix string) error { 158 | 159 | // 160 | // Ensure we have a DB-handle 161 | // 162 | if db == nil { 163 | return errors.New("SetupDB not called") 164 | } 165 | 166 | ids := make(map[int]string) 167 | rows, err := db.Query("SELECT id,yaml_file FROM reports WHERE environment IS NULL") 168 | if err != nil { 169 | return err 170 | } 171 | defer rows.Close() 172 | for rows.Next() { 173 | var id int 174 | var yamlfile string 175 | err = rows.Scan(&id, &yamlfile) 176 | if err != nil { 177 | return err 178 | } 179 | ids[id] = yamlfile 180 | } 181 | 182 | for id, yamlfile := range ids { 183 | if len(yamlfile) > 0 { 184 | var content []byte 185 | path := filepath.Join(prefix, yamlfile) 186 | content, err = ioutil.ReadFile(path) 187 | if err == nil { 188 | var report PuppetReport 189 | report, err = ParsePuppetReport(content) 190 | if err == nil { 191 | fmt.Println("Updating id:", id, "with environment:", report.Environment) 192 | _, _ = db.Exec("UPDATE reports SET environment = ? WHERE id = ?", report.Environment, id) 193 | } 194 | } 195 | } 196 | } 197 | return err 198 | } 199 | 200 | // 201 | // Add an entry to the database. 202 | // 203 | // The entry contains most of the interesting data from the parsed YAML. 204 | // 205 | // But note that it doesn't contain changed resources, etc. 206 | // 207 | // 208 | func addDB(data PuppetReport, path string) error { 209 | 210 | // 211 | // Ensure we have a DB-handle 212 | // 213 | if db == nil { 214 | return errors.New("SetupDB not called") 215 | } 216 | 217 | tx, err := db.Begin() 218 | if err != nil { 219 | return err 220 | } 221 | stmt, err := tx.Prepare("INSERT INTO reports(fqdn,environment,state,yaml_file,executed_at,runtime, failed, changed, total, skipped) values(?,?,?,?,?,?,?,?,?,?)") 222 | if err != nil { 223 | return err 224 | } 225 | defer stmt.Close() 226 | 227 | stmt.Exec(data.Fqdn, 228 | data.Environment, 229 | data.State, 230 | path, 231 | time.Now().Unix(), 232 | data.Runtime, 233 | data.Failed, 234 | data.Changed, 235 | data.Total, 236 | data.Skipped) 237 | tx.Commit() 238 | 239 | return nil 240 | } 241 | 242 | // 243 | // Count the number of reports we have. 244 | // 245 | func countReports() (int, error) { 246 | 247 | // 248 | // Ensure we have a DB-handle 249 | // 250 | if db == nil { 251 | return 0, errors.New("SetupDB not called") 252 | } 253 | 254 | var count int 255 | row := db.QueryRow("SELECT COUNT(*) FROM reports") 256 | err := row.Scan(&count) 257 | return count, err 258 | } 259 | 260 | // 261 | // Count the number of reports we have reaped. 262 | // 263 | func countUnchangedAndReapedReports() (int, error) { 264 | 265 | // 266 | // Ensure we have a DB-handle 267 | // 268 | if db == nil { 269 | return 0, errors.New("SetupDB not called") 270 | } 271 | 272 | var count int 273 | row := db.QueryRow("SELECT COUNT(*) FROM reports WHERE yaml_file='pruned'") 274 | err := row.Scan(&count) 275 | return count, err 276 | } 277 | 278 | // 279 | // Get a list of all environments 280 | // 281 | func getEnvironments() ([]string, error) { 282 | // 283 | // Ensure we have a DB-handle 284 | // 285 | if db == nil { 286 | return nil, errors.New("SetupDB not called") 287 | } 288 | 289 | var environments []string 290 | rows, err := db.Query("SELECT DISTINCT environment FROM reports ORDER BY environment") 291 | if err != nil { 292 | return nil, err 293 | } 294 | defer rows.Close() 295 | 296 | for rows.Next() { 297 | var env string 298 | err := rows.Scan(&env) 299 | if err != nil { 300 | return nil, err 301 | } 302 | environments = append(environments, env) 303 | } 304 | return environments, nil 305 | } 306 | 307 | // 308 | // Return the contents of the YAML file which was associated 309 | // with the given report-ID. 310 | // 311 | func getYAML(prefix string, id string) ([]byte, error) { 312 | 313 | // 314 | // Ensure we have a DB-handle 315 | // 316 | if db == nil { 317 | return nil, errors.New("SetupDB not called") 318 | } 319 | 320 | var path string 321 | row := db.QueryRow("SELECT yaml_file FROM reports WHERE id=?", id) 322 | err := row.Scan(&path) 323 | 324 | switch { 325 | case err == sql.ErrNoRows: 326 | case err != nil: 327 | return nil, errors.New("report not found") 328 | default: 329 | } 330 | 331 | // 332 | // Read the file content, first of all adding in the 333 | // prefix. 334 | // 335 | // (Because our reports are stored as relative paths 336 | // such as "$host/$time", rather than absolute paths 337 | // such as "reports/$host/$time".) 338 | // 339 | if len(path) > 0 { 340 | path = filepath.Join(prefix, path) 341 | content, err := ioutil.ReadFile(path) 342 | return content, err 343 | } 344 | return nil, errors.New("failed to find report with specified ID") 345 | } 346 | 347 | // 348 | // Get the data which is shown on our index page 349 | // 350 | // * The node-name. 351 | // * The status. 352 | // * The last-seen time. 353 | // 354 | func getIndexNodes(environment string) ([]PuppetRuns, error) { 355 | 356 | // 357 | // Our return-result. 358 | // 359 | var NodeList []PuppetRuns 360 | 361 | // 362 | // The threshold which marks the difference between 363 | // "current" and "orphaned" 364 | // 365 | // Here we set it to 4.5 days, which should be long 366 | // enough to cover any hosts that were powered-off over 367 | // a weekend. (Friday + Saturday + Sunday + slack). 368 | // 369 | threshold := 3.5 * (24 * 60 * 60) 370 | 371 | // 372 | // Ensure we have a DB-handle 373 | // 374 | if db == nil { 375 | return nil, errors.New("SetupDB not called") 376 | } 377 | 378 | // 379 | // Shared query piece 380 | // 381 | queryStart := "SELECT fqdn, state, runtime, max(executed_at) FROM reports WHERE " 382 | 383 | // 384 | // If environment is specified add a filter 385 | // 386 | if len(environment) > 0 { 387 | queryStart += " environment = '" + environment + "' AND " 388 | } 389 | 390 | // 391 | // Select the status - for nodes seen in the past 24 hours. 392 | // 393 | rows, err := db.Query(queryStart+" ( ( strftime('%s','now') - executed_at ) < ? ) GROUP by fqdn;", threshold) 394 | if err != nil { 395 | return nil, err 396 | } 397 | defer rows.Close() 398 | 399 | // 400 | // We'll keep track of which nodes we've seen recently. 401 | // 402 | seen := make(map[string]int) 403 | 404 | // 405 | // For each row in the result-set 406 | // 407 | // Parse into a structure and add to the list. 408 | // 409 | for rows.Next() { 410 | var tmp PuppetRuns 411 | var at string 412 | err = rows.Scan(&tmp.Fqdn, &tmp.State, &tmp.Runtime, &at) 413 | if err != nil { 414 | return nil, err 415 | } 416 | 417 | // 418 | // At this point `at` is a string containing seconds past 419 | // the epoch. 420 | // 421 | // We want to parse that into a string `At` which will 422 | // contain the literal time, and also the relative 423 | // time "Ago" 424 | // 425 | tmp.Epoch = at 426 | tmp.Ago = timeRelative(at) 427 | 428 | // 429 | i, _ := strconv.ParseInt(at, 10, 64) 430 | tmp.At = time.Unix(i, 0).Format("2006-01-02 15:04:05") 431 | 432 | // 433 | // Mark this node as non-orphaned. 434 | // 435 | seen[tmp.Fqdn] = 1 436 | 437 | // 438 | // Add the new record. 439 | // 440 | NodeList = append(NodeList, tmp) 441 | 442 | } 443 | err = rows.Err() 444 | if err != nil { 445 | return nil, err 446 | } 447 | 448 | // 449 | // Now look for orphaned nodes. 450 | // 451 | rows2, err2 := db.Query(queryStart+" ( ( strftime('%s','now') - executed_at ) > ? ) GROUP by fqdn;", threshold) 452 | if err2 != nil { 453 | return nil, err 454 | } 455 | defer rows2.Close() 456 | 457 | // 458 | // For each row in the result-set 459 | // 460 | // Parse into a structure and add to the list. 461 | // 462 | for rows2.Next() { 463 | var tmp PuppetRuns 464 | var at string 465 | err = rows2.Scan(&tmp.Fqdn, &tmp.State, &tmp.Runtime, &at) 466 | if err != nil { 467 | return nil, err 468 | } 469 | 470 | // 471 | // At this point `at` is a string containing 472 | // seconds-past-the-epoch. 473 | // 474 | // We want that to contain a human-readable 475 | // string so we first convert to an integer, then 476 | // parse as a Unix-timestamp 477 | // 478 | tmp.Epoch = at 479 | tmp.Ago = timeRelative(at) 480 | 481 | // 482 | i, _ := strconv.ParseInt(at, 10, 64) 483 | tmp.At = time.Unix(i, 0).Format("2006-01-02 15:04:05") 484 | 485 | // 486 | // Force the state to be `orphaned`. 487 | // 488 | tmp.State = "orphaned" 489 | 490 | // 491 | // If we've NOT already seen this node then 492 | // we can add it to our result set. 493 | // 494 | if seen[tmp.Fqdn] != 1 { 495 | NodeList = append(NodeList, tmp) 496 | } 497 | } 498 | err = rows2.Err() 499 | if err != nil { 500 | return nil, err 501 | } 502 | 503 | return NodeList, nil 504 | } 505 | 506 | // 507 | // Return the state of our nodes. 508 | // 509 | func getStates(environment string) ([]PuppetState, error) { 510 | 511 | // 512 | // Get the nodes. 513 | // 514 | NodeList, err := getIndexNodes(environment) 515 | if err != nil { 516 | return nil, err 517 | } 518 | 519 | // 520 | // Create a map to hold state. 521 | // 522 | states := make(map[string]int) 523 | 524 | // 525 | // Each known-state will default to being empty. 526 | // 527 | states["changed"] = 0 528 | states["unchanged"] = 0 529 | states["failed"] = 0 530 | states["orphaned"] = 0 531 | 532 | // 533 | // Count the nodes we encounter, such that we can 534 | // create a %-figure for each distinct-state. 535 | // 536 | var total int 537 | 538 | // 539 | // Count the states. 540 | // 541 | for _, o := range NodeList { 542 | states[o.State]++ 543 | total++ 544 | } 545 | 546 | // 547 | // Our return-result 548 | // 549 | var data []PuppetState 550 | 551 | // 552 | // Get the distinct keys/states in a sorted order. 553 | // 554 | var keys []string 555 | for name := range states { 556 | keys = append(keys, name) 557 | } 558 | sort.Strings(keys) 559 | 560 | // 561 | // Now for each key .. 562 | // 563 | for _, name := range keys { 564 | 565 | var tmp PuppetState 566 | tmp.State = name 567 | tmp.Count = states[name] 568 | tmp.Percentage = 0 569 | 570 | // Percentage has to be capped :) 571 | if total != 0 { 572 | c := float64(states[name]) 573 | tmp.Percentage = (c / float64(total)) * 100 574 | } 575 | data = append(data, tmp) 576 | } 577 | 578 | return data, nil 579 | } 580 | 581 | // 582 | // Get the summary-details of the runs against a given host 583 | // 584 | func getReports(fqdn string) ([]PuppetReportSummary, error) { 585 | 586 | // 587 | // Ensure we have a DB-handle 588 | // 589 | if db == nil { 590 | return nil, errors.New("SetupDB not called") 591 | } 592 | 593 | // 594 | // Select the status. 595 | // 596 | stmt, err := db.Prepare("SELECT id, fqdn, environment, state, executed_at, runtime, failed, changed, total, yaml_file FROM reports WHERE fqdn=? ORDER by executed_at DESC") 597 | if err != nil { 598 | return nil, err 599 | } 600 | rows, err := stmt.Query(fqdn) 601 | if err != nil { 602 | return nil, err 603 | } 604 | defer stmt.Close() 605 | defer rows.Close() 606 | 607 | // 608 | // We'll return a list of these hosts. 609 | // 610 | var NodeList []PuppetReportSummary 611 | 612 | // 613 | // For each row in the result-set 614 | // 615 | // Parse into a structure and add to the list. 616 | // 617 | for rows.Next() { 618 | var tmp PuppetReportSummary 619 | var at string 620 | err = rows.Scan(&tmp.ID, &tmp.Fqdn, &tmp.Environment, &tmp.State, &at, &tmp.Runtime, &tmp.Failed, &tmp.Changed, &tmp.Total, &tmp.YamlFile) 621 | if err != nil { 622 | return nil, err 623 | } 624 | 625 | // 626 | // At this point `at` is a string containing seconds past 627 | // the epoch. 628 | // 629 | // We want to parse that into a string `At` which will 630 | // contain the literal time, and also the relative 631 | // time "Ago" 632 | // 633 | tmp.Ago = timeRelative(at) 634 | 635 | i, _ := strconv.ParseInt(at, 10, 64) 636 | tmp.At = time.Unix(i, 0).Format("2006-01-02 15:04:05") 637 | 638 | // Add the result of this fetch to our list. 639 | NodeList = append(NodeList, tmp) 640 | } 641 | err = rows.Err() 642 | if err != nil { 643 | return nil, err 644 | } 645 | 646 | if len(NodeList) < 1 { 647 | return nil, errors.New("Failed to find reports for " + fqdn) 648 | 649 | } 650 | return NodeList, nil 651 | } 652 | 653 | // 654 | // Get data for our stacked bar-graph 655 | // 656 | func getHistory(environment string, limit int) ([]PuppetHistory, error) { 657 | 658 | // 659 | // Ensure we have a DB-handle 660 | // 661 | if db == nil { 662 | return nil, errors.New("SetupDB not called") 663 | } 664 | 665 | // 666 | // Our result. 667 | // 668 | var res []PuppetHistory 669 | 670 | if limit < 2 { 671 | limit = 60 672 | } 673 | // 674 | // An array to hold the unique dates we've seen. 675 | // 676 | var dates []string 677 | 678 | sel := "SELECT DISTINCT(strftime('%d/%m/%Y', DATE(executed_at, 'unixepoch', 'localtime'))) FROM reports" 679 | if len(environment) > 0 { 680 | sel = sel + " WHERE environment = '" + environment + "'" 681 | } 682 | sel = sel + " ORDER BY executed_at DESC" 683 | // 684 | // Get all the distinct dates we have data for. 685 | // 686 | stmt, err := db.Prepare(sel) 687 | if err != nil { 688 | return nil, err 689 | } 690 | 691 | rows, err := stmt.Query() 692 | if err != nil { 693 | return nil, err 694 | } 695 | defer stmt.Close() 696 | defer rows.Close() 697 | 698 | // 699 | // For each row in the result-set 700 | // 701 | for rows.Next() { 702 | var d string 703 | err = rows.Scan(&d) 704 | if err != nil { 705 | return nil, errors.New("failed to scan SQL") 706 | } 707 | 708 | dates = append(dates, d) 709 | } 710 | err = rows.Err() 711 | if err != nil { 712 | return nil, err 713 | } 714 | if ( len(dates) < limit ){ 715 | limit = len(dates) 716 | } 717 | 718 | // 719 | // Now we have all the unique dates in `dates`. 720 | // 721 | loc, _ := time.LoadLocation("Local") 722 | for _, known := range dates[:limit] { // but we only get the first limit days PuppetHistory. 723 | 724 | // 725 | // The result for this date. 726 | // 727 | var x PuppetHistory 728 | x.Changed = "0" 729 | x.Unchanged = "0" 730 | x.Failed = "0" 731 | x.Date = known 732 | formatTime, _ := time.ParseInLocation("02/01/2006 15:04:05", known+" 00:00:00", loc) 733 | ts1 := formatTime.Unix() 734 | ts2 := ts1 + 3600*24 - 1 735 | 736 | query := "SELECT distinct state, COUNT(state) AS CountOf FROM reports WHERE executed_at between ? and ?" 737 | if len(environment) > 0 { 738 | query += " AND environment = '" + environment + "' " 739 | } 740 | query += " GROUP by state" 741 | stmt, err = db.Prepare(query) 742 | if err != nil { 743 | return nil, err 744 | } 745 | 746 | rows, err = stmt.Query(ts1, ts2) 747 | if err != nil { 748 | return nil, err 749 | } 750 | defer stmt.Close() 751 | defer rows.Close() 752 | 753 | // 754 | // For each row in the result-set 755 | // 756 | for rows.Next() { 757 | var name string 758 | var count string 759 | 760 | err = rows.Scan(&name, &count) 761 | if err != nil { 762 | return nil, errors.New("failed to scan SQL") 763 | } 764 | if name == "changed" { 765 | x.Changed = count 766 | } 767 | if name == "unchanged" { 768 | x.Unchanged = count 769 | } 770 | if name == "failed" { 771 | x.Failed = count 772 | } 773 | } 774 | err = rows.Err() 775 | if err != nil { 776 | return nil, err 777 | } 778 | 779 | // 780 | // Add this days result. 781 | // 782 | res = append(res, x) 783 | 784 | } 785 | 786 | return res, err 787 | 788 | } 789 | 790 | // 791 | // Prune dangling reports 792 | // 793 | // Walk the reports directory and remove all files that are not referenced 794 | // in the database. 795 | // 796 | func pruneDangling(prefix string, noop bool, verbose bool) error { 797 | 798 | // 799 | // Ensure we have a DB-handle 800 | // 801 | if db == nil { 802 | return errors.New("SetupDB not called") 803 | } 804 | 805 | // 806 | // Find all yaml files 807 | // 808 | find, err := db.Query("SELECT yaml_file FROM reports") 809 | if err != nil { 810 | return err 811 | } 812 | 813 | // 814 | // Copy them for easy access 815 | // 816 | reports := make(map[string]int) 817 | for find.Next() { 818 | var fname string 819 | find.Scan(&fname) 820 | reports[fname] = 1 821 | } 822 | 823 | // 824 | // We have to be real careful so we will match filenames to this regexp 825 | // 826 | r, _ := regexp.Compile("^[0-9a-f]{40}$") 827 | 828 | // 829 | // Walk reports directory 830 | // 831 | err = filepath.Walk(prefix, func(path string, info os.FileInfo, err error) error { 832 | if err != nil { 833 | fmt.Printf("Error accessing path %q: %v\n", path, err) 834 | return err 835 | } 836 | if !info.IsDir() { 837 | rel, lerr := filepath.Rel(prefix, path) 838 | if r.MatchString(info.Name()) && lerr == nil { 839 | _, found := reports[rel] 840 | if found { 841 | // can be used to find db entries with no file reports 842 | reports[rel] = 2 843 | } else { 844 | if noop { 845 | fmt.Printf("Would remove file %q\n", path) 846 | } else { 847 | if verbose { 848 | fmt.Printf("Removing file %q\n", path) 849 | } 850 | os.Remove(path) 851 | } 852 | } 853 | } else { 854 | fmt.Printf("Warning - unexpected file or error parsing: %q\n", path) 855 | } 856 | } 857 | return nil 858 | }) 859 | 860 | // 861 | // Check for database entries with missing yaml file reports 862 | // 863 | if verbose { 864 | for k, v := range reports { 865 | if v != 2 { 866 | fmt.Printf("Missing file: %q\n", k) 867 | } 868 | } 869 | } 870 | 871 | return err 872 | } 873 | 874 | // 875 | // Prune old reports 876 | // 877 | // We have to find the old reports, individually, so we can unlink the 878 | // copy of the on-disk YAML, but once we've done that we can delete them 879 | // as a group. 880 | // 881 | func pruneReports(environment string, prefix string, days int, verbose bool) error { 882 | 883 | // 884 | // Ensure we have a DB-handle 885 | // 886 | if db == nil { 887 | return errors.New("SetupDB not called") 888 | } 889 | 890 | // 891 | // Select appropriate environment, if specified 892 | // 893 | envCondition := "" 894 | if len(environment) > 0 { 895 | envCondition = " AND environment = '" + environment + "'" 896 | } 897 | 898 | // 899 | // Convert our query into something useful. 900 | // 901 | time := days * (24 * 60 * 60) 902 | 903 | // 904 | // Find things that are old. 905 | // 906 | find, err := db.Prepare("SELECT id,yaml_file FROM reports WHERE ( ( strftime('%s','now') - executed_at ) > ? )" + envCondition) 907 | if err != nil { 908 | return err 909 | } 910 | 911 | // 912 | // Remove old reports, en mass. 913 | // 914 | clean, err := db.Prepare("DELETE FROM reports WHERE ( ( strftime('%s','now') - executed_at ) > ? )" + envCondition) 915 | if err != nil { 916 | return err 917 | } 918 | 919 | // 920 | // Find the old reports. 921 | // 922 | rows, err := find.Query(time) 923 | if err != nil { 924 | return err 925 | } 926 | defer find.Close() 927 | defer clean.Close() 928 | defer rows.Close() 929 | 930 | // 931 | // For each row in the result-set 932 | // 933 | // Parse into "id" + "path", then remove the path from disk. 934 | // 935 | for rows.Next() { 936 | var id string 937 | var path string 938 | 939 | err = rows.Scan(&id, &path) 940 | if err == nil { 941 | 942 | // 943 | // Convert the path to a qualified one, 944 | // rather than one relative to our report-dir. 945 | // 946 | path = filepath.Join(prefix, path) 947 | if verbose { 948 | fmt.Printf("Removing ID:%s - %s\n", id, path) 949 | } 950 | 951 | // 952 | // Remove the file from-disk 953 | // 954 | // We won't care if this fails, it might have 955 | // been removed behind our back or failed to 956 | // be uploaded in the first place. 957 | // 958 | os.Remove(path) 959 | } 960 | } 961 | err = rows.Err() 962 | if err != nil { 963 | return err 964 | } 965 | 966 | // 967 | // Now cleanup the old records 968 | // 969 | _, err = clean.Exec(time) 970 | if err != nil { 971 | return err 972 | } 973 | 974 | return nil 975 | } 976 | 977 | // 978 | // Prune reports from nodes which are unchanged. 979 | // 980 | // We have to find the old reports, individually, so we can unlink the 981 | // copy of the on-disk YAML, but once we've done that we can delete them 982 | // as a group. 983 | // 984 | func pruneUnchanged(environment string, prefix string, verbose bool) error { 985 | 986 | // 987 | // Ensure we have a DB-handle 988 | // 989 | if db == nil { 990 | return errors.New("SetupDB not called") 991 | } 992 | 993 | // 994 | // Select appropriate environment, if specified 995 | // 996 | envCondition := "" 997 | if len(environment) > 0 { 998 | envCondition = " AND environment = '" + environment + "'" 999 | } 1000 | 1001 | // 1002 | // Find unchanged reports. 1003 | // 1004 | find, err := db.Prepare("SELECT id,yaml_file FROM reports WHERE state='unchanged'" + envCondition) 1005 | if err != nil { 1006 | return err 1007 | } 1008 | 1009 | // 1010 | // Prepare to update them all. 1011 | // 1012 | clean, err := db.Prepare("UPDATE reports SET yaml_file='pruned' WHERE state='unchanged'" + envCondition) 1013 | if err != nil { 1014 | return err 1015 | } 1016 | 1017 | // 1018 | // Find the reports. 1019 | // 1020 | rows, err := find.Query() 1021 | if err != nil { 1022 | return err 1023 | } 1024 | defer find.Close() 1025 | defer clean.Close() 1026 | defer rows.Close() 1027 | 1028 | // 1029 | // For each row in the result-set 1030 | // 1031 | // Parse into "id" + "path", then remove the path from disk. 1032 | // 1033 | for rows.Next() { 1034 | var id string 1035 | var path string 1036 | 1037 | err = rows.Scan(&id, &path) 1038 | if err == nil { 1039 | 1040 | // 1041 | // Convert the path to a qualified one, 1042 | // rather than one relative to our report-dir. 1043 | // 1044 | path = filepath.Join(prefix, path) 1045 | if verbose { 1046 | fmt.Printf("Removing ID:%s - %s\n", id, path) 1047 | } 1048 | 1049 | // 1050 | // Remove the file from-disk 1051 | // 1052 | // We won't care if this fails, it might have 1053 | // been removed behind our back or failed to 1054 | // be uploaded in the first place. 1055 | // 1056 | os.Remove(path) 1057 | } 1058 | } 1059 | err = rows.Err() 1060 | if err != nil { 1061 | return err 1062 | } 1063 | 1064 | // 1065 | // Now cleanup the old records 1066 | // 1067 | _, err = clean.Exec() 1068 | if err != nil { 1069 | return err 1070 | } 1071 | 1072 | return nil 1073 | } 1074 | 1075 | func pruneOrphaned(environment string, prefix string, verbose bool) error { 1076 | 1077 | NodeList, err := getIndexNodes(environment) 1078 | if err != nil { 1079 | return err 1080 | } 1081 | 1082 | for _, entry := range NodeList { 1083 | 1084 | if entry.State == "orphaned" { 1085 | if verbose { 1086 | fmt.Printf("Orphaned host: %s\n", entry.Fqdn) 1087 | } 1088 | 1089 | // 1090 | // Find all reports that refer to this host. 1091 | // 1092 | rows, err := db.Query("SELECT yaml_file FROM reports WHERE fqdn=?", entry.Fqdn) 1093 | if err != nil { 1094 | return err 1095 | } 1096 | defer rows.Close() 1097 | 1098 | for rows.Next() { 1099 | var tmp string 1100 | err = rows.Scan(&tmp) 1101 | if err != nil { 1102 | return err 1103 | } 1104 | 1105 | // 1106 | // Convert the path to a qualified one, 1107 | // rather than one relative to our report-dir. 1108 | // 1109 | path := filepath.Join(prefix, tmp) 1110 | if verbose { 1111 | fmt.Printf("\tRemoving: %s\n", path) 1112 | } 1113 | 1114 | // 1115 | // Remove the file from-disk 1116 | // 1117 | // We won't care if this fails, it might have 1118 | // been removed behind our back or failed to 1119 | // be uploaded in the first place. 1120 | // 1121 | os.Remove(path) 1122 | } 1123 | 1124 | // 1125 | // Now remove the report-entries 1126 | // 1127 | clean, err := db.Prepare("DELETE FROM reports WHERE fqdn=?") 1128 | if err != nil { 1129 | return err 1130 | } 1131 | defer clean.Close() 1132 | _, err = clean.Exec(entry.Fqdn) 1133 | if err != nil { 1134 | return err 1135 | } 1136 | 1137 | } 1138 | 1139 | } 1140 | 1141 | return nil 1142 | } 1143 | -------------------------------------------------------------------------------- /db_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Basic testing of our DB primitives 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "database/sql" 9 | "fmt" 10 | "io/ioutil" 11 | "os" 12 | "regexp" 13 | "testing" 14 | "time" 15 | ) 16 | 17 | // 18 | // Temporary location for database 19 | // 20 | var path string 21 | 22 | // 23 | // Create a temporary database 24 | // 25 | func FakeDB() { 26 | p, err := ioutil.TempDir(os.TempDir(), "prefix") 27 | if err == nil { 28 | path = p 29 | } 30 | 31 | // 32 | // Setup the tables. 33 | // 34 | SetupDB(p + "/db.sql") 35 | 36 | } 37 | 38 | // 39 | // Add some fake reports 40 | // 41 | func addFakeReports() { 42 | tx, err := db.Begin() 43 | if err != nil { 44 | panic(err) 45 | } 46 | 47 | // 48 | // Add some records 49 | stmt, err := tx.Prepare("INSERT INTO reports(fqdn,environment,yaml_file,executed_at) values(?,?,?,?)") 50 | if err != nil { 51 | panic(err) 52 | } 53 | defer stmt.Close() 54 | 55 | count := 0 56 | 57 | for count < 30 { 58 | now := time.Now().Unix() 59 | days := int64(60 * 60 * 24 * count) 60 | env := "production" 61 | if count > 2 { 62 | env = "test" 63 | } 64 | 65 | fqdn := fmt.Sprintf("node%d.example.com", count) 66 | now -= days 67 | stmt.Exec(fqdn, env, "/../data/valid.yaml", now) 68 | count++ 69 | } 70 | tx.Commit() 71 | } 72 | 73 | // 74 | // Add some (repeated) nodes in various states 75 | // 76 | func addFakeNodes() { 77 | 78 | var n PuppetReport 79 | n.Fqdn = "foo.example.com" 80 | n.State = "changed" 81 | n.Runtime = "3.134" 82 | n.Failed = "0" 83 | n.Total = "1" 84 | n.Changed = "2" 85 | n.Skipped = "3" 86 | addDB(n, "") 87 | 88 | n.Fqdn = "bar.example.com" 89 | n.State = "failed" 90 | n.Runtime = "2.718" 91 | n.Failed = "0" 92 | n.Total = "1" 93 | n.Changed = "2" 94 | n.Skipped = "3" 95 | addDB(n, "") 96 | 97 | n.Fqdn = "foo.example.com" 98 | n.State = "unchanged" 99 | n.Runtime = "2.718" 100 | n.Failed = "0" 101 | n.Total = "1" 102 | n.Changed = "2" 103 | n.Skipped = "3" 104 | addDB(n, "") 105 | 106 | // 107 | // Here we're trying to fake an orphaned node. 108 | // 109 | // When a report is added the exected_at field is set to 110 | // "time.Now().Unix()". To make an orphaned record we need 111 | // to change that to some time >24 days ago. 112 | // 113 | // We do that by finding the last report-ID, and then editing 114 | // the field. 115 | // 116 | var maxID string 117 | row := db.QueryRow("SELECT MAX(id) FROM reports") 118 | err := row.Scan(&maxID) 119 | 120 | switch { 121 | case err == sql.ErrNoRows: 122 | case err != nil: 123 | panic("failed to find max report ID") 124 | default: 125 | } 126 | 127 | // 128 | // Now we can change the executed_at field of that last 129 | // addition 130 | // 131 | sqlStmt := fmt.Sprintf("UPDATE reports SET executed_at=300 WHERE id=%s", 132 | maxID) 133 | _, err = db.Exec(sqlStmt) 134 | if err != nil { 135 | panic("Failed to change report ") 136 | } 137 | 138 | } 139 | 140 | // 141 | // Get a valid report ID. 142 | // 143 | func validReportID() (int, error) { 144 | var count int 145 | row := db.QueryRow("SELECT MAX(id) FROM reports") 146 | err := row.Scan(&count) 147 | return count, err 148 | } 149 | 150 | // 151 | // Test that functions return errors if setup hasn't been called. 152 | // 153 | func TestMissingInit(t *testing.T) { 154 | 155 | // 156 | // Regexp to match the error we expect to receive. 157 | // 158 | reg, _ := regexp.Compile("SetupDB not called") 159 | 160 | var x PuppetReport 161 | err := addDB(x, "") 162 | if !reg.MatchString(err.Error()) { 163 | t.Errorf("Got wrong error: %v", err) 164 | } 165 | 166 | _, err = countReports() 167 | if !reg.MatchString(err.Error()) { 168 | t.Errorf("Got wrong error: %v", err) 169 | } 170 | 171 | _, err = getYAML("", "") 172 | if !reg.MatchString(err.Error()) { 173 | t.Errorf("Got wrong error: %v", err) 174 | } 175 | 176 | _, err = getIndexNodes("") 177 | if !reg.MatchString(err.Error()) { 178 | t.Errorf("Got wrong error: %v", err) 179 | } 180 | 181 | _, err = getReports("example.com") 182 | if !reg.MatchString(err.Error()) { 183 | t.Errorf("Got wrong error: %v", err) 184 | } 185 | 186 | _, err = getHistory("", 60) 187 | if !reg.MatchString(err.Error()) { 188 | t.Errorf("Got wrong error: %v", err) 189 | } 190 | 191 | err = pruneReports("", "", 3, false) 192 | if !reg.MatchString(err.Error()) { 193 | t.Errorf("Got wrong error: %v", err) 194 | } 195 | 196 | } 197 | 198 | // 199 | // Test creating a new DB fails when given a directory. 200 | // 201 | func TestBogusInit(t *testing.T) { 202 | 203 | // Create a fake database 204 | FakeDB() 205 | 206 | err := SetupDB(path) 207 | 208 | if err == nil { 209 | t.Errorf("We should have seen a create-error") 210 | } 211 | 212 | // 213 | // Cleanup here because otherwise later tests will 214 | // see an active/valid DB-handle. 215 | // 216 | db.Close() 217 | db = nil 218 | os.RemoveAll(path) 219 | } 220 | 221 | // 222 | // Add some nodes and verify they are reaped. 223 | // 224 | func TestPrune(t *testing.T) { 225 | 226 | // Create a fake database 227 | FakeDB() 228 | 229 | // With some reports. 230 | addFakeReports() 231 | 232 | // 233 | // Count records and assume we have some. 234 | // 235 | old, err := countReports() 236 | 237 | if err != nil { 238 | t.Errorf("Error counting reports") 239 | } 240 | if old != 30 { 241 | t.Errorf("We have %d reports, not 30", old) 242 | } 243 | 244 | // 245 | // Run the prune 246 | // 247 | pruneReports("", "", 5, false) 248 | 249 | // 250 | // Count them again 251 | // 252 | new, err := countReports() 253 | if err != nil { 254 | t.Errorf("Error counting reports") 255 | } 256 | 257 | if new != 6 { 258 | t.Errorf("We have %d reports, not 6", new) 259 | } 260 | 261 | // 262 | // Test pruning of specific environments by pruning all test envs 263 | // 264 | pruneReports("test", "", 0, false) 265 | 266 | // 267 | // Final count 268 | // 269 | fnl, err := countReports() 270 | 271 | if err != nil { 272 | t.Errorf("Error counting reports") 273 | } 274 | if fnl != 3 { 275 | t.Errorf("We have %d production environment reports, not 3", fnl) 276 | } 277 | 278 | // 279 | // Cleanup here because otherwise later tests will 280 | // see an active/valid DB-handle. 281 | // 282 | db.Close() 283 | db = nil 284 | os.RemoveAll(path) 285 | } 286 | 287 | // 288 | // Add some nodes and verify they are reaped, if unchanged. 289 | // 290 | func TestPruneUnchanged(t *testing.T) { 291 | 292 | // Create a fake database 293 | FakeDB() 294 | 295 | // With some reports. 296 | addFakeNodes() 297 | 298 | // 299 | // Count records and assume we have some. 300 | // 301 | old, err := countReports() 302 | 303 | if err != nil { 304 | t.Errorf("Error counting reports") 305 | } 306 | if old != 3 { 307 | t.Errorf("We have %d reports, not 3", old) 308 | } 309 | 310 | // 311 | // Run the prune 312 | // 313 | pruneUnchanged("", "", false) 314 | 315 | // 316 | // Count them again 317 | // 318 | new, err := countReports() 319 | if err != nil { 320 | t.Errorf("Error counting reports") 321 | } 322 | 323 | // 324 | // The value won't have changed. 325 | // 326 | if new != old { 327 | t.Errorf("We have %d reports, not %d", new, old) 328 | } 329 | 330 | // 331 | // But we'll expect that several will have updated 332 | // to show that their paths have been changed to 'reaped' 333 | // 334 | pruned, err := countUnchangedAndReapedReports() 335 | if err != nil { 336 | t.Errorf("Error counting reaped reports") 337 | } 338 | 339 | if pruned != 1 { 340 | t.Errorf("We have %d pruned reports, not 1", pruned) 341 | } 342 | 343 | // 344 | // Cleanup here because otherwise later tests will 345 | // see an active/valid DB-handle. 346 | // 347 | db.Close() 348 | db = nil 349 | os.RemoveAll(path) 350 | } 351 | 352 | // 353 | // Test the index nodes are valid 354 | // 355 | func TestIndex(t *testing.T) { 356 | 357 | // 358 | // Create a fake database. 359 | // 360 | FakeDB() 361 | 362 | // Add some fake nodes. 363 | addFakeNodes() 364 | 365 | // 366 | // We have three fake nodes now, two of which have the 367 | // same hostname. 368 | // 369 | runs, err := getIndexNodes("") 370 | if err != nil { 371 | t.Errorf("getIndexNodes failed: %v", err) 372 | } 373 | 374 | // 375 | // Should have two side 376 | // 377 | if len(runs) != 2 { 378 | t.Errorf("getIndexNodes returned wrong number of results: %d", len(runs)) 379 | } 380 | 381 | // 382 | // But three reports 383 | // 384 | total, err := countReports() 385 | if err != nil { 386 | t.Errorf("Failed to count reports") 387 | } 388 | 389 | if total != 3 { 390 | t.Errorf("We found the wrong number of reports, %d", total) 391 | } 392 | 393 | // 394 | // Cleanup here because otherwise later tests will 395 | // see an active/valid DB-handle. 396 | // 397 | db.Close() 398 | db = nil 399 | os.RemoveAll(path) 400 | } 401 | 402 | // 403 | // Test the report-run are valid 404 | // 405 | func TestMissiongReport(t *testing.T) { 406 | FakeDB() 407 | 408 | _, err := getYAML("", "") 409 | 410 | reg, _ := regexp.Compile("failed to find report with specified ID") 411 | if !reg.MatchString(err.Error()) { 412 | t.Errorf("Got wrong error: %v", err) 413 | } 414 | 415 | // 416 | // Cleanup here because otherwise later tests will 417 | // see an active/valid DB-handle. 418 | // 419 | db.Close() 420 | db = nil 421 | os.RemoveAll(path) 422 | } 423 | 424 | // 425 | // Test the report-run are valid 426 | // 427 | func TestReports(t *testing.T) { 428 | 429 | // 430 | // Add fake reports. 431 | // 432 | FakeDB() 433 | addFakeNodes() 434 | 435 | // 436 | // We have three fake nodes now, two of which have the 437 | // same hostname. 438 | // 439 | runs, err := getReports("foo.example.com") 440 | if err != nil { 441 | t.Errorf("getReports failed: %v", err) 442 | } 443 | 444 | // 445 | // Should have two runs against the host 446 | // 447 | if len(runs) != 2 { 448 | t.Errorf("getReports returned wrong number of results: %d", len(runs)) 449 | } 450 | 451 | // 452 | // Cleanup here because otherwise later tests will 453 | // see an active/valid DB-handle. 454 | // 455 | db.Close() 456 | db = nil 457 | os.RemoveAll(path) 458 | } 459 | 460 | // 461 | // Test the report-run are valid 462 | // 463 | func TestHistory(t *testing.T) { 464 | 465 | // 466 | // Add fake reports. 467 | // 468 | FakeDB() 469 | addFakeNodes() 470 | 471 | // 472 | // We have three fake nodes now, two of which have the same hostname. 473 | // 474 | runs, err := getHistory("", 60) 475 | if err != nil { 476 | t.Errorf("getHistory failed: %v", err) 477 | } 478 | 479 | // 480 | // Should have 2 runs, becase we have only one unique date.. 481 | // 482 | if len(runs) != 2 { 483 | t.Errorf("getReports returned wrong number of results: %d", len(runs)) 484 | } 485 | 486 | // 487 | // Cleanup here because otherwise later tests will 488 | // see an active/valid DB-handle. 489 | // 490 | db.Close() 491 | db = nil 492 | os.RemoveAll(path) 493 | } 494 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/skx/puppet-summary 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/felixge/httpsnoop v1.0.4 // indirect 7 | github.com/google/subcommands v1.2.0 8 | github.com/gorilla/handlers v1.5.2 9 | github.com/gorilla/mux v1.8.1 10 | github.com/marpaia/graphite-golang v0.0.0-20190519024811-caf161d2c2b1 11 | github.com/mattn/go-sqlite3 v1.14.22 12 | github.com/robfig/cron v1.2.0 13 | github.com/skx/golang-metrics v0.0.0-20190325085214-453332cf54e8 14 | github.com/smallfish/simpleyaml v0.1.0 15 | ) 16 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 4 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 5 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 6 | github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= 7 | github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= 8 | github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= 9 | github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= 10 | github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= 11 | github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= 12 | github.com/marpaia/graphite-golang v0.0.0-20171231172105-134b9af18cf3/go.mod h1:llZw8JbFm5CvdRrtgdjaQNlZR1bQhAWsBKtb0HTX+sw= 13 | github.com/marpaia/graphite-golang v0.0.0-20190519024811-caf161d2c2b1 h1:lODGHy+2Namopi4v7AeiqW106eo4QMXqj9aE8jVXcO4= 14 | github.com/marpaia/graphite-golang v0.0.0-20190519024811-caf161d2c2b1/go.mod h1:llZw8JbFm5CvdRrtgdjaQNlZR1bQhAWsBKtb0HTX+sw= 15 | github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= 16 | github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 17 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 18 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 19 | github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= 20 | github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= 21 | github.com/skx/golang-metrics v0.0.0-20190325085214-453332cf54e8 h1:NVwRIqHO7J7vnKGbTz5dBwWjl5Wr6mR1U8JQ32tw7vk= 22 | github.com/skx/golang-metrics v0.0.0-20190325085214-453332cf54e8/go.mod h1:P+OUoQPrBQUZg9lbHEu7iJsZYTC5Na4qghTSs5ZmTA4= 23 | github.com/smallfish/simpleyaml v0.1.0 h1:5uAZdLAiHxS9cmzkOxg7lH0dILXKTD7uRZbAhyHmyU0= 24 | github.com/smallfish/simpleyaml v0.1.0/go.mod h1:gU3WdNn44dQVAbVHD2SrSqKKCvmzFApWD2UURhgEj1M= 25 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 26 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 27 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 28 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 29 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 30 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 31 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 32 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 33 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 34 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // 2 | // Entry-point to the puppet-summary service. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "flag" 10 | "fmt" 11 | "os" 12 | "runtime/debug" 13 | 14 | "github.com/google/subcommands" 15 | ) 16 | 17 | // 18 | // Setup our sub-commands and use them. 19 | // 20 | func main() { 21 | defer func() { 22 | if r := recover(); r != nil { 23 | fmt.Println("Panic at the disco: \n" + string(debug.Stack())) 24 | } 25 | }() 26 | 27 | subcommands.Register(subcommands.HelpCommand(), "") 28 | subcommands.Register(subcommands.FlagsCommand(), "") 29 | subcommands.Register(subcommands.CommandsCommand(), "") 30 | subcommands.Register(&metricsCmd{}, "") 31 | subcommands.Register(&pruneCmd{}, "") 32 | subcommands.Register(&serveCmd{}, "") 33 | subcommands.Register(&versionCmd{}, "") 34 | subcommands.Register(&yamlCmd{}, "") 35 | 36 | flag.Parse() 37 | ctx := context.Background() 38 | os.Exit(int(subcommands.Execute(ctx))) 39 | } 40 | -------------------------------------------------------------------------------- /samples/systemd_service.txt: -------------------------------------------------------------------------------- 1 | To have puppet-summary start as a daemon: 2 | 3 | cd /etc/systemd/system 4 | 5 | cat < puppet-summary.service 6 | [Unit] 7 | Description=Web interface providing reporting features for Puppet 8 | [Service] 9 | Type=simple 10 | WorkingDirectory=/opt/puppet-summary 11 | ExecStart=/opt/puppet-summary/puppet-summary serve -auto-prune -host 0.0.0.0 12 | [Install] 13 | WantedBy=multi-user.target 14 | EOF 15 | 16 | systemctl daemon-reload && \ 17 | systemctl enable --now puppet-summary.service && \ 18 | systemctl status puppet-summary.service 19 | -------------------------------------------------------------------------------- /static_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Simple testing of our embedded resource. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | "testing" 13 | ) 14 | 15 | // 16 | // Test that we have one embedded resource. 17 | // 18 | func TestResourceCount(t *testing.T) { 19 | 20 | expected := 0 21 | 22 | // We're going to compare what is embedded with 23 | // what is on-disk. 24 | // 25 | // We could just hard-wire the count, but that 26 | // would require updating the count every time 27 | // we add/remove a new resource 28 | err := filepath.Walk("data", 29 | func(path string, info os.FileInfo, err error) error { 30 | if err != nil { 31 | return err 32 | } 33 | if !info.IsDir() { 34 | expected++ 35 | } 36 | return nil 37 | }) 38 | if err != nil { 39 | t.Errorf("failed to find resources beneath data/ %s", err.Error()) 40 | } 41 | 42 | // ARBITRARY! 43 | if expected < 10 { 44 | t.Fatalf("we expected more than 10 files beneath data/") 45 | } 46 | 47 | out := getResources() 48 | 49 | if len(out) != expected { 50 | t.Errorf("We expected %d resources but found %d.", expected, len(out)) 51 | } 52 | } 53 | 54 | // 55 | // Test that each of our resources is identical to the master 56 | // version. 57 | // 58 | func TestResourceMatches(t *testing.T) { 59 | 60 | // 61 | // For each resource 62 | // 63 | all := getResources() 64 | 65 | for _, entry := range all { 66 | 67 | // 68 | // Get the data from our embedded copy 69 | // 70 | data, err := getResource(entry.Filename) 71 | if err != nil { 72 | t.Errorf("Loading our resource failed:%s", entry.Filename) 73 | } 74 | 75 | // 76 | // Get the data from our master-copy. 77 | // 78 | master, err := ioutil.ReadFile(entry.Filename) 79 | if err != nil { 80 | t.Errorf("Loading our master-resource failed:%s", entry.Filename) 81 | } 82 | 83 | // 84 | // Test the lengths match 85 | // 86 | if len(master) != len(data) { 87 | t.Errorf("Embedded and real resources have different sizes.") 88 | } 89 | 90 | // 91 | // Test the data-matches 92 | // 93 | if string(master) != string(data) { 94 | t.Errorf("Embedded and real resources have different content.") 95 | } 96 | } 97 | } 98 | 99 | // 100 | // Test that a missing resource is handled. 101 | // 102 | func TestMissingResource(t *testing.T) { 103 | 104 | // 105 | // Get the data from our embedded copy 106 | // 107 | data, err := getResource("moi/kissa") 108 | if data != nil { 109 | t.Errorf("We expected to find no data, but got some.") 110 | } 111 | if err == nil { 112 | t.Errorf("We expected an error loading a missing resource, but got none.") 113 | } 114 | if !strings.Contains(err.Error(), "failed to find resource") { 115 | t.Errorf("Error message differed from expectations.") 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /timespan.go: -------------------------------------------------------------------------------- 1 | // 2 | // Utility function to create relative time-spans. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "strconv" 10 | "time" 11 | ) 12 | 13 | // 14 | // Describe the given number of seconds. Negative values are treated 15 | // identically to positive ones. 16 | // 17 | func timeDescr(seconds int64) string { 18 | 19 | // 20 | // We don't deal with future-time 21 | // 22 | if seconds < 0 { 23 | seconds *= -1 24 | } 25 | 26 | // 27 | // Divide up, from most recent to most distant. 28 | // 29 | switch { 30 | case seconds < 1: 31 | return "just now" 32 | case seconds < 2: 33 | return "1 second ago" 34 | case seconds < 60: 35 | return fmt.Sprintf("%d seconds ago", seconds) 36 | case seconds < 120: 37 | return "1 minute ago" 38 | case seconds < 60*60: 39 | return fmt.Sprintf("%d minutes ago", seconds/(60)) 40 | case seconds < 2*60*60: 41 | return "1 hour ago" 42 | case seconds < 48*60*60: 43 | return fmt.Sprintf("%d hours ago", seconds/(60*60)) 44 | default: 45 | return fmt.Sprintf("%d days ago", seconds/(60*60*24)) 46 | } 47 | } 48 | 49 | // 50 | // Given a string containing the seconds past the epoch return 51 | // a human-friendly description of how long ago that was. 52 | // 53 | // (Using a string is weird. I blame SQLite :) 54 | // 55 | func timeRelative(epoch string) string { 56 | 57 | // 58 | // Get the current time. 59 | // 60 | var now = time.Now().Unix() 61 | 62 | // 63 | // Convert the given string to an int 64 | // 65 | var unix, _ = strconv.ParseInt(epoch, 10, 64) 66 | 67 | // 68 | // How long ago was that, in an absolute number of seconds? 69 | // 70 | ago := now - unix 71 | if ago < 0 { 72 | ago *= -1 73 | } 74 | 75 | return (timeDescr(ago)) 76 | } 77 | -------------------------------------------------------------------------------- /timespan_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestDescriptions(t *testing.T) { 10 | 11 | type TestCase struct { 12 | Seconds int64 13 | Result string 14 | } 15 | 16 | cases := []TestCase{{20, "20 seconds ago"}, {13, "13 seconds ago"}, 17 | {-43, "43 seconds ago"}, 18 | {64, "1 minute ago"}, 19 | {300, "5 minutes ago"}, 20 | {7000, "1 hour ago"}, 21 | {60 * 60 * 2.4, "2 hours ago"}, 22 | {60 * 60 * 24, "24 hours ago"}, 23 | {60 * 60 * 24 * 3, "3 days ago"}, 24 | } 25 | 26 | for _, o := range cases { 27 | 28 | out := timeDescr(o.Seconds) 29 | 30 | if out != o.Result { 31 | t.Errorf("Expected '%s' received '%s' for %d", o.Result, out, o.Seconds) 32 | } 33 | } 34 | } 35 | 36 | // 37 | // Test the wrapping method accepts sane values. 38 | // 39 | func TestString(t *testing.T) { 40 | 41 | // 42 | // Test "just now". 43 | // 44 | str := fmt.Sprintf("%d", time.Now().Unix()) 45 | out := timeRelative(str) 46 | 47 | if out != "just now" { 48 | t.Errorf("Invalid time-value - got %s", out) 49 | } 50 | 51 | // 52 | // Test again with a negative time. 53 | // (Since "now + 1" will become negative when the test is run.) 54 | // 55 | str = fmt.Sprintf("%d", time.Now().Unix()+1) 56 | out = timeRelative(str) 57 | 58 | if out != "1 second ago" { 59 | t.Errorf("Invalid time-value - got %s", out) 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /yaml_parser.go: -------------------------------------------------------------------------------- 1 | // 2 | // This package is the sole place that we extract data from the YAML 3 | // that Puppet submits to us. 4 | // 5 | // Here is where we're going to extract: 6 | // 7 | // * Logged messages 8 | // * Runtime 9 | // * etc. 10 | // 11 | 12 | package main 13 | 14 | import ( 15 | "crypto/sha1" 16 | "errors" 17 | "fmt" 18 | "reflect" 19 | "regexp" 20 | "strings" 21 | 22 | "github.com/smallfish/simpleyaml" 23 | ) 24 | 25 | // 26 | // Resource refers to a resource in your puppet modules, a resource has 27 | // a name, along with the file & line-number it was defined in within your 28 | // manifest 29 | // 30 | type Resource struct { 31 | Name string 32 | Type string 33 | File string 34 | Line string 35 | } 36 | 37 | // 38 | // PuppetReport stores the details of a single run of puppet. 39 | // 40 | type PuppetReport struct { 41 | 42 | // 43 | // FQDN of the node. 44 | // 45 | Fqdn string 46 | 47 | // 48 | // Environment of the node. 49 | // 50 | Environment string 51 | 52 | // 53 | // State of the run: changed unchanged, etc. 54 | // 55 | State string 56 | 57 | // 58 | // The time the puppet-run was completed. 59 | // 60 | // This is self-reported by the node, and copied almost literally. 61 | // 62 | At string 63 | 64 | // 65 | // The time puppet took to run, in seconds. 66 | // 67 | Runtime string 68 | 69 | // 70 | // A count of resources that failed, changed, were unchanged, 71 | // etc. Strings for simplicity even though they are clearly 72 | // integers. 73 | // 74 | Failed string 75 | Changed string 76 | Total string 77 | Skipped string 78 | 79 | // 80 | // Log messages. 81 | // 82 | LogMessages []string 83 | 84 | // 85 | // Resources which have failed/changed/been skipped. 86 | // 87 | // These include the file/line in which they were defined 88 | // in the puppet manifest(s), due to their use of the Resource 89 | // structure 90 | // 91 | ResourcesFailed []Resource 92 | ResourcesChanged []Resource 93 | ResourcesSkipped []Resource 94 | ResourcesOK []Resource 95 | 96 | // 97 | // Hash of the report-body. 98 | // 99 | // This is used to create the file to store the report in on-disk, 100 | // and as a means of detecting duplication submissions. 101 | // 102 | Hash string 103 | } 104 | 105 | // 106 | // Here we have some simple methods that each parse a part of the 107 | // YAML file, updating the structure they are passed. 108 | // 109 | // These snippets are broken down to avoid an uber-complex 110 | // set of code in the ParsePuppetReport method. 111 | // 112 | 113 | // 114 | // parseHost reads the `host` parameter from the YAML and populates 115 | // the given report-structure with suitable values. 116 | // 117 | func parseHost(y *simpleyaml.Yaml, out *PuppetReport) error { 118 | // 119 | // Get the hostname. 120 | // 121 | host, err := y.Get("host").String() 122 | if err != nil { 123 | return errors.New("failed to get 'host' from YAML") 124 | } 125 | 126 | // 127 | // Ensure the hostname passes a simple regexp 128 | // 129 | reg, _ := regexp.Compile("^([a-z0-9._-]+)$") 130 | if !reg.MatchString(host) { 131 | return errors.New("the submitted 'host' field failed our security check") 132 | } 133 | 134 | out.Fqdn = host 135 | return nil 136 | } 137 | 138 | // 139 | // parseEnvironment reads the `environment` parameter from the YAML and populates 140 | // the given report-structure with suitable values. 141 | // 142 | func parseEnvironment(y *simpleyaml.Yaml, out *PuppetReport) error { 143 | // 144 | // Get the hostname. 145 | // 146 | env, err := y.Get("environment").String() 147 | if err != nil { 148 | return errors.New("failed to get 'environment' from YAML") 149 | } 150 | 151 | // 152 | // Ensure the hostname passes a simple regexp 153 | // 154 | reg, _ := regexp.Compile("^([A-Za-z0-9_]+)$") 155 | if !reg.MatchString(env) { 156 | return errors.New("the submitted 'environment' field failed our security check") 157 | } 158 | 159 | out.Environment = env 160 | return nil 161 | } 162 | 163 | // 164 | // parseTime reads the `time` parameter from the YAML and populates 165 | // the given report-structure with suitable values. 166 | // 167 | func parseTime(y *simpleyaml.Yaml, out *PuppetReport) error { 168 | 169 | // 170 | // Get the time puppet executed 171 | // 172 | at, err := y.Get("time").String() 173 | if err != nil { 174 | return errors.New("failed to get 'time' from YAML") 175 | } 176 | 177 | // Strip any quotes that might surround the time. 178 | at = strings.Replace(at, "'", "", -1) 179 | 180 | // Convert "T" -> " " 181 | at = strings.Replace(at, "T", " ", -1) 182 | 183 | // strip the time at the first period. 184 | parts := strings.Split(at, ".") 185 | at = parts[0] 186 | 187 | // update the struct 188 | out.At = at 189 | 190 | return nil 191 | } 192 | 193 | // 194 | // parseStatus reads the `status` parameter from the YAML and populates 195 | // the given report-structure with suitable values. 196 | // 197 | func parseStatus(y *simpleyaml.Yaml, out *PuppetReport) error { 198 | // 199 | // Get the status 200 | // 201 | state, err := y.Get("status").String() 202 | if err != nil { 203 | return errors.New("failed to get 'status' from YAML") 204 | } 205 | 206 | switch state { 207 | case "changed": 208 | case "unchanged": 209 | case "failed": 210 | default: 211 | return errors.New("unexpected 'status' - " + state) 212 | } 213 | 214 | out.State = state 215 | return nil 216 | } 217 | 218 | // 219 | // parseRuntime reads the `metrics.time.values` parameters from the YAML 220 | // and populates given report-structure with suitable values. 221 | // 222 | func parseRuntime(y *simpleyaml.Yaml, out *PuppetReport) error { 223 | 224 | // 225 | // Get the run-time this execution took. 226 | // 227 | times, err := y.Get("metrics").Get("time").Get("values").Array() 228 | if err != nil { 229 | return err 230 | } 231 | 232 | r, _ := regexp.Compile("Total ([0-9.]+)") 233 | 234 | runtime := "" 235 | 236 | // 237 | // HORRID: Help me, I'm in hell. 238 | // 239 | // TODO: Improve via reflection as per log-handling. 240 | // 241 | for _, value := range times { 242 | match := r.FindStringSubmatch(fmt.Sprint(value)) 243 | if len(match) == 2 { 244 | runtime = match[1] 245 | } 246 | } 247 | out.Runtime = runtime 248 | return nil 249 | } 250 | 251 | // 252 | // parseResources looks for the counts of resources which have been 253 | // failed, changed, skipped, etc, and updates the given report-structure 254 | // with those values. 255 | // 256 | func parseResources(y *simpleyaml.Yaml, out *PuppetReport) error { 257 | 258 | resources, err := y.Get("metrics").Get("resources").Get("values").Array() 259 | if err != nil { 260 | return err 261 | } 262 | 263 | tr, _ := regexp.Compile("Total ([0-9.]+)") 264 | fr, _ := regexp.Compile("Failed ([0-9.]+)") 265 | sr, _ := regexp.Compile("Skipped ([0-9.]+)") 266 | cr, _ := regexp.Compile("Changed ([0-9.]+)") 267 | 268 | total := "" 269 | changed := "" 270 | failed := "" 271 | skipped := "" 272 | 273 | // 274 | // HORRID: Help me, I'm in hell. 275 | // 276 | // TODO: Improve via reflection as per log-handling. 277 | // 278 | for _, value := range resources { 279 | mr := tr.FindStringSubmatch(fmt.Sprint(value)) 280 | if len(mr) == 2 { 281 | total = mr[1] 282 | } 283 | mf := fr.FindStringSubmatch(fmt.Sprint(value)) 284 | if len(mf) == 2 { 285 | failed = mf[1] 286 | } 287 | ms := sr.FindStringSubmatch(fmt.Sprint(value)) 288 | if len(ms) == 2 { 289 | skipped = ms[1] 290 | } 291 | mc := cr.FindStringSubmatch(fmt.Sprint(value)) 292 | if len(mc) == 2 { 293 | changed = mc[1] 294 | } 295 | } 296 | 297 | out.Total = total 298 | out.Changed = changed 299 | out.Failed = failed 300 | out.Skipped = skipped 301 | return nil 302 | } 303 | 304 | // 305 | // parseLogs updates the given report with any logged messages. 306 | // 307 | func parseLogs(y *simpleyaml.Yaml, out *PuppetReport) error { 308 | logs, err := y.Get("logs").Array() 309 | if err != nil { 310 | return errors.New("failed to get 'logs' from YAML") 311 | } 312 | 313 | var logged []string 314 | 315 | for _, v2 := range logs { 316 | 317 | // create a map 318 | m := make(map[string]string) 319 | 320 | v := reflect.ValueOf(v2) 321 | if v.Kind() == reflect.Map { 322 | for _, key := range v.MapKeys() { 323 | strct := v.MapIndex(key) 324 | 325 | // Store the key/val in the map. 326 | key, val := key.Interface(), strct.Interface() 327 | m[key.(string)] = fmt.Sprint(val) 328 | } 329 | } 330 | 331 | if len(m["message"]) > 0 { 332 | logged = append(logged, m["source"]+" : "+m["message"]) 333 | } 334 | } 335 | 336 | out.LogMessages = logged 337 | return nil 338 | } 339 | 340 | // 341 | // parseResults updates the given report with details of any resource 342 | // which was failed, changed, or skipped. 343 | // 344 | func parseResults(y *simpleyaml.Yaml, out *PuppetReport) error { 345 | rs, err := y.Get("resource_statuses").Map() 346 | if err != nil { 347 | return errors.New("failed to get 'resource_statuses' from YAML") 348 | } 349 | 350 | var failed []Resource 351 | var changed []Resource 352 | var skipped []Resource 353 | var ok []Resource 354 | 355 | for _, v2 := range rs { 356 | 357 | // create a map here. 358 | m := make(map[string]string) 359 | 360 | v := reflect.ValueOf(v2) 361 | if v.Kind() == reflect.Map { 362 | for _, key := range v.MapKeys() { 363 | strct := v.MapIndex(key) 364 | 365 | // Store the key/val in the map. 366 | key, val := key.Interface(), strct.Interface() 367 | m[key.(string)] = fmt.Sprint(val) 368 | } 369 | } 370 | 371 | // Now we should be able to look for skipped ones. 372 | if m["skipped"] == "true" { 373 | skipped = append(skipped, 374 | Resource{Name: m["title"], 375 | Type: m["resource_type"], 376 | File: m["file"], 377 | Line: m["line"]}) 378 | } 379 | 380 | // Now we should be able to look for skipped ones. 381 | if m["changed"] == "true" { 382 | changed = append(changed, 383 | Resource{Name: m["title"], 384 | Type: m["resource_type"], 385 | File: m["file"], 386 | Line: m["line"]}) 387 | } 388 | 389 | // Now we should be able to look for skipped ones. 390 | if m["failed"] == "true" { 391 | failed = append(failed, 392 | Resource{Name: m["title"], 393 | Type: m["resource_type"], 394 | File: m["file"], 395 | Line: m["line"]}) 396 | } 397 | 398 | if m["failed"] == "false" && 399 | m["skipped"] == "false" && 400 | m["changed"] == "false" { 401 | ok = append(ok, 402 | Resource{Name: m["title"], 403 | Type: m["resource_type"], 404 | File: m["file"], 405 | Line: m["line"]}) 406 | } 407 | 408 | } 409 | 410 | out.ResourcesSkipped = skipped 411 | out.ResourcesFailed = failed 412 | out.ResourcesChanged = changed 413 | out.ResourcesOK = ok 414 | 415 | return nil 416 | 417 | } 418 | 419 | // 420 | // ParsePuppetReport is our main function in this module. Given an 421 | // array of bytes we read the input and produce a PuppetReport structure. 422 | // 423 | // Various (simple) error conditions are handled to ensure that the result 424 | // is somewhat safe - for example we must have some fields such as 425 | // `hostname`, `time`, etc. 426 | // 427 | func ParsePuppetReport(content []byte) (PuppetReport, error) { 428 | // 429 | // The return-value. 430 | // 431 | var x PuppetReport 432 | 433 | // 434 | // Parse the YAML. 435 | // 436 | yaml, err := simpleyaml.NewYaml(content) 437 | if err != nil { 438 | return x, errors.New("failed to parse YAML") 439 | } 440 | 441 | // 442 | // Store the SHA1-hash of the report contents 443 | // 444 | helper := sha1.New() 445 | helper.Write(content) 446 | x.Hash = fmt.Sprintf("%x", helper.Sum(nil)) 447 | 448 | // 449 | // Parse the hostname 450 | // 451 | hostError := parseHost(yaml, &x) 452 | if hostError != nil { 453 | return x, hostError 454 | } 455 | 456 | // 457 | // Parse the environment 458 | // 459 | envError := parseEnvironment(yaml, &x) 460 | if envError != nil { 461 | return x, envError 462 | } 463 | 464 | // 465 | // Parse the time. 466 | // 467 | timeError := parseTime(yaml, &x) 468 | if timeError != nil { 469 | return x, timeError 470 | } 471 | 472 | // 473 | // Parse the status 474 | // 475 | stateError := parseStatus(yaml, &x) 476 | if stateError != nil { 477 | return x, stateError 478 | } 479 | 480 | // 481 | // Parse the runtime of this execution 482 | // 483 | runError := parseRuntime(yaml, &x) 484 | if runError != nil { 485 | return x, runError 486 | } 487 | 488 | // 489 | // Get the resource-data from this run 490 | // 491 | resourcesError := parseResources(yaml, &x) 492 | if resourcesError != nil { 493 | return x, resourcesError 494 | } 495 | 496 | // 497 | // Get the logs from this run 498 | // 499 | logsError := parseLogs(yaml, &x) 500 | if logsError != nil { 501 | return x, logsError 502 | } 503 | 504 | // 505 | // Finally the resources 506 | // 507 | resError := parseResults(yaml, &x) 508 | if resError != nil { 509 | return x, resError 510 | } 511 | 512 | return x, nil 513 | } 514 | -------------------------------------------------------------------------------- /yaml_parser_test.go: -------------------------------------------------------------------------------- 1 | // 2 | // Our YAML parser is our single biggest potential source of 3 | // failure - whether users give us bogus input, or puppet-versions 4 | // change what thye submit. 5 | // 6 | // We should have good, thorough, and real test-cases here. 7 | // 8 | // 9 | 10 | package main 11 | 12 | import ( 13 | "regexp" 14 | "strings" 15 | "testing" 16 | ) 17 | 18 | // 19 | // Ensure that bogus YAML is caught. 20 | // 21 | func TestBogusYaml(t *testing.T) { 22 | // 23 | // Parse the bogus YAML content "`\n3'" 24 | // 25 | _, err := ParsePuppetReport([]byte("`\n3'")) 26 | 27 | // 28 | // Ensure the error is what we expect. 29 | // 30 | reg, _ := regexp.Compile("failed to parse YAML") 31 | if !reg.MatchString(err.Error()) { 32 | t.Errorf("Got wrong error: %v", err) 33 | } 34 | } 35 | 36 | // 37 | // Test that we can handle dates of various forms. 38 | // 39 | func TestYamlDates(t *testing.T) { 40 | 41 | tests := []string{"---\ntime: '2017-03-10T10:22:33.659245699+00:00'\nhost: bart\nenvironment: production\n", 42 | "---\ntime: 2017-03-10 10:22:33.493526494 +00:00\nhost: foo\nenvironment: production\n"} 43 | 44 | for _, input := range tests { 45 | 46 | // 47 | // Error will be set here, since we only supply 48 | // `host` + `time` we'll expect something like 49 | // 50 | // "Failed to get `status' from YAML 51 | // 52 | node, _ := ParsePuppetReport([]byte(input)) 53 | 54 | if node.At != "2017-03-10 10:22:33" { 55 | t.Errorf("Invalid time result, got '%s'", node.At) 56 | } 57 | } 58 | 59 | } 60 | 61 | // 62 | // Test that we can handle filter out bogus hostnames. 63 | // 64 | // Here we look for an exception of the form "blah invalid|missing host" 65 | // to know whether we passed/failed. 66 | // 67 | func TestHostName(t *testing.T) { 68 | 69 | // 70 | // Test-cases 71 | // 72 | type HostTest struct { 73 | hostname string 74 | valid bool 75 | } 76 | 77 | // 78 | // Possible Hostnames 79 | // 80 | fail := []HostTest{ 81 | {"../../../etc/passwd%00", false}, 82 | {"node1.example.com../../../etc", false}, 83 | {"steve_example com", false}, 84 | {"node1./example.com", false}, 85 | {"steve1.example.com", true}, 86 | {"steve-example.com", true}, 87 | 88 | {"example3-3_2.com", true}} 89 | 90 | // 91 | // For each test-case 92 | // 93 | for _, input := range fail { 94 | 95 | // 96 | // Build up YAML 97 | // 98 | tmp := "---\n" + 99 | "host: " + input.hostname 100 | 101 | // 102 | // Parse it. 103 | // 104 | _, err := ParsePuppetReport([]byte(tmp)) 105 | 106 | // 107 | // Host-regexp. 108 | // 109 | reg, _ := regexp.Compile("host") 110 | 111 | // 112 | // Do we expect this to pass/fail? 113 | // 114 | if input.valid { 115 | 116 | if reg.MatchString(err.Error()) { 117 | t.Errorf("Expected no error relating to 'host', but got one: %v", err) 118 | } 119 | } else { 120 | 121 | // 122 | // We expect this to fail. Did it? 123 | // 124 | if !reg.MatchString(err.Error()) { 125 | t.Errorf("Expected an error relating to 'host', but didn't: %v", err) 126 | } 127 | } 128 | } 129 | } 130 | 131 | // 132 | // Test that we can detect unknown states. 133 | // 134 | func TestNodeStatus(t *testing.T) { 135 | 136 | // 137 | // Test-cases 138 | // 139 | type TestCase struct { 140 | state string 141 | valid bool 142 | } 143 | 144 | // 145 | // Possible states, and whether they are valid 146 | // 147 | fail := []TestCase{ 148 | {"changed", true}, 149 | {"unchanged", true}, 150 | {"failed", true}, 151 | {"blah", false}, 152 | {"forced", false}, 153 | {"unknown", false}} 154 | 155 | // 156 | // For each test-case 157 | // 158 | for _, input := range fail { 159 | 160 | // 161 | // Build up YAML 162 | // 163 | tmp := "---\n" + 164 | "host: foo.example.com\n" + 165 | "environment: production\n" + 166 | "time: '2017-08-07T16:37:42.659245699+00:00'\n" + 167 | "status: " + input.state 168 | 169 | // 170 | // Parse it. 171 | // 172 | _, err := ParsePuppetReport([]byte(tmp)) 173 | 174 | // 175 | // regexp for matching error-conditions 176 | // 177 | reg, _ := regexp.Compile("status") 178 | 179 | // 180 | // Do we expect this to pass/fail? 181 | // 182 | if input.valid { 183 | 184 | if reg.MatchString(err.Error()) { 185 | t.Errorf("Expected no error relating to 'status', but got one: %v", err) 186 | } 187 | } else { 188 | 189 | // 190 | // We expect this to fail. Did it? 191 | // 192 | if !reg.MatchString(err.Error()) { 193 | t.Errorf("Expected an error relating to 'status', but didn't: %v", err) 194 | } 195 | } 196 | } 197 | } 198 | 199 | // 200 | // Test importing a valid YAML file. 201 | // 202 | // TODO: Test bogus ones too. 203 | // 204 | func TestValidYaml(t *testing.T) { 205 | 206 | // 207 | // Read the YAML file. 208 | // 209 | tmpl, err := getResource("data/valid.yaml") 210 | if err != nil { 211 | t.Fatal("Failed to load YAML asset data/valid.yaml") 212 | } 213 | 214 | report, err := ParsePuppetReport(tmpl) 215 | 216 | if err != nil { 217 | t.Fatal("Failed to parse YAML file") 218 | } 219 | 220 | // 221 | // Test data from YAML 222 | // 223 | if report.Fqdn != "www.steve.org.uk" { 224 | t.Errorf("Incorrect hostname: %v", report.Fqdn) 225 | } 226 | if report.State != "unchanged" { 227 | t.Errorf("Incorrect state: %v", report.State) 228 | } 229 | if report.At != "2017-07-29 23:17:01" { 230 | t.Errorf("Incorrect at: %v", report.At) 231 | } 232 | if report.Failed != "0" { 233 | t.Errorf("Incorrect failed: %v", report.Failed) 234 | } 235 | if report.Changed != "0" { 236 | t.Errorf("Incorrect changed: %v", report.Changed) 237 | } 238 | if report.Skipped != "2" { 239 | t.Errorf("Incorrect skipped: %v", report.Skipped) 240 | } 241 | } 242 | 243 | // 244 | // Test a valid report which has been modified to remove fields of 245 | // interest raises errors as expected. 246 | // 247 | func TestMissingResources(t *testing.T) { 248 | 249 | // 250 | // Various fields we remove. 251 | // 252 | tests := []string{"resource_statuses", 253 | "logs", 254 | "metrics", 255 | "resources", 256 | "values"} 257 | 258 | // 259 | // Read the YAML file. 260 | // 261 | tmpl, err := getResource("data/valid.yaml") 262 | if err != nil { 263 | t.Fatal("Failed to load YAML asset data/valid.yaml") 264 | } 265 | 266 | // 267 | // For each field-test 268 | // 269 | for _, field := range tests { 270 | 271 | // 272 | // Conver the template to a string, and remove 273 | // the bit that we should. 274 | // 275 | str := string(tmpl) 276 | str = strings.Replace(str, field, "blah", -1) 277 | 278 | // 279 | // Now parse, which we expect to fail. 280 | // 281 | var b = []byte(str) 282 | _, err = ParsePuppetReport(b) 283 | 284 | // 285 | // We expect an error. 286 | // 287 | if err == nil { 288 | t.Fatal("We expected an error from the report!") 289 | } 290 | 291 | // 292 | // The error will relate to our string, or an interface 293 | // violation 294 | // 295 | if !strings.Contains(err.Error(), field) && !strings.Contains(err.Error(), "type assertion") { 296 | t.Fatal("No reference to field/type in our error") 297 | } 298 | } 299 | } 300 | --------------------------------------------------------------------------------