├── .gitattributes ├── .github └── workflows │ └── build_and_publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── arguments.go ├── go.mod ├── go.sum ├── main.go └── provider.json /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/workflows/build_and_publish.yml: -------------------------------------------------------------------------------- 1 | name: Build and Publish 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | workflow_dispatch: 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | os-arch: 15 | - name: Linux-amd64 16 | os: linux 17 | arch: amd64 18 | - name: macOS-amd64 19 | os: darwin 20 | arch: amd64 21 | - name: macOS-arm64 22 | os: darwin 23 | arch: arm64 24 | - name: Windows-amd64 25 | os: windows 26 | arch: amd64 27 | - name: Linux-arm64 28 | os: linux 29 | arch: arm64 30 | steps: 31 | - name: Checkout code 32 | uses: actions/checkout@v4 33 | 34 | - name: Setup Go 35 | uses: actions/setup-go@v5 36 | with: 37 | go-version-file: 'go.mod' # Use the Go version specified in go.mod 38 | - name: Set Ref Name Variable 39 | run: | 40 | if [ "$GITHUB_EVENT_NAME" != "release" ]; then 41 | # Use Git commit SHA as the reference when manually triggered 42 | ref_name=${GITHUB_SHA::7} 43 | else 44 | ref_name=${{ github.ref_name }} 45 | fi 46 | echo "REF_NAME=${ref_name}" >> "$GITHUB_ENV" 47 | - name: Build for ${{ matrix.os-arch.name }} 48 | run: | 49 | mkdir -p builds/${{ matrix.os-arch.name }}/usr/bin 50 | if [ "${{ matrix.os-arch.os }}" == "windows" ]; then 51 | # For Windows, add .exe to the binary name 52 | binary_name=nzbrefresh.exe 53 | else 54 | binary_name=nzbrefresh 55 | fi 56 | GOARCH=${{ matrix.os-arch.arch }} GOOS=${{ matrix.os-arch.os }} go build -ldflags="-s -w -X main.appVersion=${{ env.REF_NAME }}" -o builds/${{ matrix.os-arch.name }}/usr/bin/$binary_name 57 | zip -j "nzbrefresh_${{ env.REF_NAME }}-${{ matrix.os-arch.os }}-${{ matrix.os-arch.arch }}.zip" builds/${{ matrix.os-arch.name }}/usr/bin/$binary_name provider.json 58 | # Makeing deb packages for linux and darwin 59 | if [ "${{ matrix.os-arch.os }}" == "linux" ] || [ "${{ matrix.os-arch.os }}" == "darwin" ]; then 60 | mkdir -p builds/${{ matrix.os-arch.name }}/DEBIAN 61 | VERSION=$(echo "${{ github.ref_name }}" | sed 's/^v//') 62 | if [ "${{ matrix.os-arch.os }}" == "darwin" ]; then 63 | ARCH=${{ matrix.os-arch.os }}-${{ matrix.os-arch.arch }} 64 | else 65 | ARCH=${{ matrix.os-arch.arch }} 66 | fi 67 | echo "Package: nzbrefresh" >> builds/${{ matrix.os-arch.name }}/DEBIAN/control 68 | echo "Version: ${VERSION}" >> builds/${{ matrix.os-arch.name }}/DEBIAN/control 69 | echo "Maintainer: ${{ github.repository_owner }} <${{ github.repository_owner_email }}>" >> builds/${{ matrix.os-arch.name }}/DEBIAN/control 70 | echo "Architecture: ${ARCH}" >> builds/${{ matrix.os-arch.name }}/DEBIAN/control 71 | echo "Description: NZB Refresh - a cmd line tool to re-upload articles missing from low retention providers or after takedowns" >> builds/${{ matrix.os-arch.name }}/DEBIAN/control 72 | dpkg-deb --root-owner-group --build builds/${{ matrix.os-arch.name }} nzbrefresh_${{ env.REF_NAME }}-${{ matrix.os-arch.os }}-${{ matrix.os-arch.arch }}.deb 73 | fi 74 | - name: Upload Release Assets 75 | if: github.event_name == 'release' # Only on release 76 | uses: softprops/action-gh-release@v2 77 | with: 78 | files: | 79 | nzbrefresh_*.zip 80 | nzbrefresh_*.deb 81 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | 23 | # test files 24 | test*.* 25 | 26 | # github codespace 27 | oryxBuildBinary 28 | 29 | # dont ignore gh action yml 30 | !.github/workflows/build_and_publish.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Tensai 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Release Workflow](https://github.com/Tensai75/nzbrefresh/actions/workflows/build_and_publish.yml/badge.svg?event=release)](https://github.com/Tensai75/nzbrefresh/actions/workflows/build_and_publish.yml) 2 | [![Latest Release)](https://img.shields.io/github/v/release/Tensai75/nzbrefresh?logo=github)](https://github.com/Tensai75/nzbrefresh/releases/latest) 3 | 4 | # NZB Refresh 5 | Proof of concept for a cmd line tool to re-upload articles that are missing from providers with low retention. 6 | 7 | The cmd line tool analyses the NZB file specified as positional argument and checks the availability of the individual articles at all Usenet providers listed in the provider.json. 8 | If an article is missing from one or more providers, but is still available from at least another provider, the tool will download the article from one of the providers where the article is still available and attempt to re-upload the article using the POST command. It will first try to re-upload the article to one of the providers where the article is not available and, if this is not successful (e.g. due to missing POST capability), it will also try to re-upload the article to one of the other providers. However, the providers already haveing the article might refuse to accept the re-upload of the same article. So for best results, all usenet accounts used for this tool should have POST capability. 9 | 10 | The article is re-uploaded completely unchanged (same message ID, same subject), with the exception of the date header, which is updated to the current date. Once the article has been uploaded to one provider, it should then propagate to all other providers. 11 | 12 | As a result, the upload should become available again at *all* providers and be able to be downloaded with the *identical / original* NZB file that was used for the refresh. 13 | 14 | __PLEASE NOTE: This is a very early alpha version, intended for initial testing only.__ 15 | 16 | Bug reports are very welcome. Please open an issue for this. If possible, add a link to the NZB file that was used when the error occurred. 17 | 18 | ## Installation 19 | 1. Download the executable file for your system from the release page. 20 | 2. Extract the archive. 21 | 3. Edit the `provider.json` according to your requirements. 22 | 23 | ## Running the program 24 | Run the program in a cmd line with the following argument: 25 | 26 | `nzbrefresh [--check] [--provider PROVIDER] [--debug] NZBFILE` 27 | 28 | Positional arguments: 29 | 30 | NZBFILE path to the NZB file to be checked (required) 31 | 32 | Options: 33 | 34 | --check, -c only check availability - don't re-upload (optional) 35 | 36 | --provider PROVIDER, -p PROVIDER 37 | path to the provider JSON config file (optional / default is: './provider.json') 38 | 39 | --debug, -d logs additional output to log file (optional, log file will be named NZBFILENAME.log) 40 | 41 | --csv writes statistic about available segements to a csv file (optional, csv file will be named NZBFILENAME.csv) 42 | 43 | --help, -h display this help and exit 44 | 45 | --version display version and exit 46 | 47 | 48 | ## provider.json options 49 | `"Name": "Provider 1",` arbitrary name of the provider, used in the debug text/output 50 | 51 | `"Host": "",` usenet server host name or IP address 52 | 53 | `"Port": 119,` usenet server port number 54 | 55 | `"SSL": false,` if true, secure SSL connections will be used 56 | 57 | `"SkipSslCheck": true,` if true, certificate errors will be ignored 58 | 59 | `"Username": "",` usenet account username 60 | 61 | `"Password": "",` usenet account password 62 | 63 | `"ConnWaitTime": 10,` waiting time until reconnection after connection errors 64 | 65 | `"MaxConns": 50,` maximum number of connections to be used 66 | 67 | `"IdleTimeout": 30,` time after a connection is closed 68 | 69 | `"HealthCheck": false,` if true, will check health of connection before using it (will reduce speed) 70 | 71 | `"MaxTooManyConnsErrors": 3,` maximum number of consecutive "too manny connections error" after which MaxConns is automatically reduced 72 | 73 | `"MaxConnErrors": 3` maximum number of consecutive fatal connection errors after which the connection with the provider is deemed to have failed 74 | 75 | ## TODOs 76 | - option to set the priority for the providers to be used for re-uploading 77 | - option to use either the STAT, HEAD or BODY command for the check 78 | - option to use the IHAVE command for re-uploading (not implemented with most providers, however) 79 | - folder monitoring with automatic checking 80 | - ...? 81 | 82 | This is a Proof of Concept with the minimum necessary features. 83 | The TODOs is what I currently plan to implement but there is certainly also a lot of other things left to do. 84 | 85 | ## Version history 86 | ### alpha 3 87 | - fix for panic errors (should fix all errors in issue [#1](https://github.com/Tensai75/nzbrefresh/issues/1)) 88 | - added --csv switch for csv output of available segements per file per provider 89 | - use STAT instead of HEAD (should improve speed) 90 | - a lot of refactoring 91 | 92 | ### alpha 2 93 | - highly improved version with parallel processing 94 | 95 | ### alpha 1 96 | - first public version 97 | 98 | ## Credits 99 | This software is built using golang ([License](https://go.dev/LICENSE)). 100 | 101 | This software uses the following external libraries: 102 | - github.com/alexflint/go-arg ([License](https://github.com/alexflint/go-arg/blob/master/LICENSE)) 103 | - github.com/alexflint/go-scalar ([License](https://github.com/alexflint/go-scalar/blob/master/LICENSE)) 104 | - github.com/fatih/color ([License](https://github.com/fatih/color/blob/main/LICENSE.md)) 105 | - github.com/mattn/go-colorable ([License](https://github.com/mattn/go-colorable/blob/master/LICENSE)) 106 | - github.com/mattn/go-isatty ([License](https://github.com/mattn/go-isatty/blob/master/LICENSE)) 107 | - github.com/nu11ptr/cmpb ([License](https://github.com/nu11ptr/cmpb/blob/master/LICENSE)) 108 | -------------------------------------------------------------------------------- /arguments.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "log" 8 | "os" 9 | 10 | parser "github.com/alexflint/go-arg" 11 | ) 12 | 13 | // arguments structure 14 | type Args struct { 15 | NZBFile string `arg:"positional" help:"path to the NZB file to be checked"` 16 | CheckOnly bool `arg:"-c, --check" help:"only check availability - don't re-upload"` 17 | Provider string `arg:"-p, --provider" help:"path to the provider JSON config file (Default: './provider.json')"` 18 | Debug bool `arg:"-d, --debug" help:"logs additional output to log file"` 19 | Csv bool `arg:"--csv" help:"writes statistic about available segements to a csv file"` 20 | } 21 | 22 | // version information 23 | func (Args) Version() string { 24 | return fmt.Sprintf("%v %v", appName, appVersion) 25 | } 26 | 27 | // additional description 28 | func (Args) Epilogue() string { 29 | return "For more information visit github.com/Tensai75/nzbrefresh\n" 30 | } 31 | 32 | // global arguments variable 33 | var args struct { 34 | Args 35 | } 36 | 37 | // parser variable 38 | 39 | func parseArguments() { 40 | var argParser *parser.Parser 41 | 42 | parserConfig := parser.Config{ 43 | IgnoreEnv: true, 44 | } 45 | 46 | // parse flags 47 | argParser, _ = parser.NewParser(parserConfig, &args) 48 | if err := parser.Parse(&args); err != nil { 49 | if err.Error() == "help requested by user" { 50 | writeHelp(argParser) 51 | os.Exit(0) 52 | } else if err.Error() == "version requested by user" { 53 | fmt.Println(args.Version()) 54 | os.Exit(0) 55 | } 56 | writeUsage(argParser) 57 | log.Fatal(err) 58 | } 59 | 60 | checkArguments(argParser) 61 | 62 | } 63 | 64 | func checkArguments(argParser *parser.Parser) { 65 | if args.NZBFile == "" { 66 | writeUsage(argParser) 67 | exit(fmt.Errorf("no path to NZB file provided")) 68 | } 69 | 70 | if args.Provider == "" { 71 | args.Provider = "./provider.json" 72 | } 73 | } 74 | 75 | func writeUsage(parser *parser.Parser) { 76 | var buf bytes.Buffer 77 | parser.WriteUsage(&buf) 78 | scanner := bufio.NewScanner(&buf) 79 | for scanner.Scan() { 80 | fmt.Println(" " + scanner.Text()) 81 | } 82 | } 83 | 84 | func writeHelp(parser *parser.Parser) { 85 | var buf bytes.Buffer 86 | parser.WriteHelp(&buf) 87 | scanner := bufio.NewScanner(&buf) 88 | for scanner.Scan() { 89 | fmt.Println(" " + scanner.Text()) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Tensai75/nzbrefresh 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/Tensai75/cmpb v0.0.0-20240707075110-16fb79fca928 7 | github.com/Tensai75/nntp v0.1.2 8 | github.com/Tensai75/nntpPool v0.1.2 9 | github.com/Tensai75/nzbparser v0.1.0 10 | github.com/alexflint/go-arg v1.5.1 11 | github.com/fatih/color v1.17.0 12 | ) 13 | 14 | require ( 15 | github.com/Tensai75/subjectparser v0.1.1 // indirect 16 | github.com/alexflint/go-scalar v1.2.0 // indirect 17 | github.com/mattn/go-colorable v0.1.13 // indirect 18 | github.com/mattn/go-isatty v0.0.20 // indirect 19 | github.com/nu11ptr/cmpb v0.0.0-20181013131528-0306ae9a87d1 // indirect 20 | github.com/stretchr/testify v1.9.0 // indirect 21 | golang.org/x/net v0.27.0 // indirect 22 | golang.org/x/sys v0.22.0 // indirect 23 | golang.org/x/text v0.16.0 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Tensai75/cmpb v0.0.0-20240601075616-76ce7bf974ae h1:dFWOmaGMKsBE4QxCo8lbp3ZEFuzty6r8Byp30en7BUA= 2 | github.com/Tensai75/cmpb v0.0.0-20240601075616-76ce7bf974ae/go.mod h1:4AfvlWOTwC8PyO/62X4Yvtjwt7zioHssN37HuyxodMA= 3 | github.com/Tensai75/cmpb v0.0.0-20240601135142-b4420b2b9bf4 h1:8OY/kus3vpjT40O8vuvfoszle2TKwteHgL/wv/NsQ24= 4 | github.com/Tensai75/cmpb v0.0.0-20240601135142-b4420b2b9bf4/go.mod h1:4AfvlWOTwC8PyO/62X4Yvtjwt7zioHssN37HuyxodMA= 5 | github.com/Tensai75/cmpb v0.0.0-20240707075110-16fb79fca928 h1:XmkOOoPacohN8CPV5W38TlqnHNZQYOBWI/jL9dtTo9k= 6 | github.com/Tensai75/cmpb v0.0.0-20240707075110-16fb79fca928/go.mod h1:4AfvlWOTwC8PyO/62X4Yvtjwt7zioHssN37HuyxodMA= 7 | github.com/Tensai75/nntp v0.1.0 h1:+KkaJ/1owm/6P2TwteSnX/bRgaLugtQJpbw9o1/urW8= 8 | github.com/Tensai75/nntp v0.1.0/go.mod h1:pOyal56FP3lyS2PldUbtPZEltrUje30KZReZiUJFwzE= 9 | github.com/Tensai75/nntp v0.1.1 h1:FpHT49fGO/P78kGFAkXp0ecawMYjTDgxQkwgfSBZclg= 10 | github.com/Tensai75/nntp v0.1.1/go.mod h1:tey0EOBjZngjCOTo8/WfMbDnzvYyyLbRtOLwvv36rG0= 11 | github.com/Tensai75/nntp v0.1.2 h1:3OMjXYptNdkl7dY5NGhcIRiSK6dfPm96/nLms1AyFSg= 12 | github.com/Tensai75/nntp v0.1.2/go.mod h1:tey0EOBjZngjCOTo8/WfMbDnzvYyyLbRtOLwvv36rG0= 13 | github.com/Tensai75/nntpPool v0.1.0 h1:Oez/wioqARK1K/0T4jv1PP1CjuyaMx+/OU12FJXByww= 14 | github.com/Tensai75/nntpPool v0.1.0/go.mod h1:VR6l4l91fB5JqInBRnYfJAO7EPyoDAiogbvidIIEJ5A= 15 | github.com/Tensai75/nntpPool v0.1.1 h1:uTLQflExR6LY7n6c868JSq/cHctEsvNbpIH4EDVcPAc= 16 | github.com/Tensai75/nntpPool v0.1.1/go.mod h1:cbIkz4S7+ex1Q+yjssONATY4zyzAJC0Bq0D+19e53ks= 17 | github.com/Tensai75/nntpPool v0.1.2 h1:nTDMZMmnjSUwm4aqvaYIyhT47bc6uINMTAKaWT0/kBw= 18 | github.com/Tensai75/nntpPool v0.1.2/go.mod h1:cbIkz4S7+ex1Q+yjssONATY4zyzAJC0Bq0D+19e53ks= 19 | github.com/Tensai75/nzbparser v0.0.0-20240511081422-ab40863e6df5 h1:3XCiYjDG0Gx3FiToJyibiMLda4zBRLnqcV6T9AnEFuI= 20 | github.com/Tensai75/nzbparser v0.0.0-20240511081422-ab40863e6df5/go.mod h1:GsLExFYgUcBQvETN0D91HeALBoAJPmXGvICFfsqEBtw= 21 | github.com/Tensai75/nzbparser v0.1.0 h1:6RppAuWFahqu/kKjWO5Br0xuEYcxGz+XBTxYc+qvPo4= 22 | github.com/Tensai75/nzbparser v0.1.0/go.mod h1:IUIIaeGaYp2dLAAF29BWYeKTfI4COvXaeQAzQiTOfMY= 23 | github.com/Tensai75/subjectparser v0.0.0-20240511081302-1e97329a5fe7 h1:ix/Vvx/MkRRmKQ4zm1BTpX6AxGqVlFiabiW7uRNlzZI= 24 | github.com/Tensai75/subjectparser v0.0.0-20240511081302-1e97329a5fe7/go.mod h1:1oK70GK+0ul/Ko+eRiSRzRZ86LCNnDFCg0rAiSy9wMQ= 25 | github.com/Tensai75/subjectparser v0.1.0 h1:6fEWnRov8lDHxJS2EWqY6VonwYfrIRN+k8h8H7fFwHA= 26 | github.com/Tensai75/subjectparser v0.1.0/go.mod h1:PNBFBnkOGbVDfX+56ZmC4GKSpqoRMCF1Y44xYd7NLGI= 27 | github.com/Tensai75/subjectparser v0.1.1 h1:SAlaEKUmaalt4QH+UFBzEP6iHqA994iouFqvFPSM9y0= 28 | github.com/Tensai75/subjectparser v0.1.1/go.mod h1:PNBFBnkOGbVDfX+56ZmC4GKSpqoRMCF1Y44xYd7NLGI= 29 | github.com/alexflint/go-arg v1.5.0 h1:rwMKGiaQuRbXfZNyRUvIfke63QvOBt1/QTshlGQHohM= 30 | github.com/alexflint/go-arg v1.5.0/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8= 31 | github.com/alexflint/go-arg v1.5.1 h1:nBuWUCpuRy0snAG+uIJ6N0UvYxpxA0/ghA/AaHxlT8Y= 32 | github.com/alexflint/go-arg v1.5.1/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8= 33 | github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw= 34 | github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= 35 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 36 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 37 | github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= 38 | github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= 39 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 40 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 41 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 42 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 43 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 44 | github.com/nu11ptr/cmpb v0.0.0-20181013131528-0306ae9a87d1 h1:p6c1PD50zlZZXTKKU8NvEfOLH2hsuHHFE7ubf9hIKYI= 45 | github.com/nu11ptr/cmpb v0.0.0-20181013131528-0306ae9a87d1/go.mod h1:sGDCGGL1GDbrTwlP1EFRhJVUuaGXFBGA5S0wxC2ddmo= 46 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 47 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 48 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 49 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 50 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 51 | golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= 52 | golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= 53 | golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= 54 | golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= 55 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 56 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 57 | golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= 58 | golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 59 | golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= 60 | golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 61 | golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= 62 | golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 63 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 64 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 65 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 66 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 67 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/csv" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "log" 11 | "os" 12 | "path/filepath" 13 | "slices" 14 | "sort" 15 | "strings" 16 | "sync" 17 | "sync/atomic" 18 | "time" 19 | 20 | "github.com/Tensai75/cmpb" 21 | "github.com/Tensai75/nntp" 22 | "github.com/Tensai75/nntpPool" 23 | "github.com/Tensai75/nzbparser" 24 | "github.com/fatih/color" 25 | ) 26 | 27 | type ( 28 | Provider struct { 29 | Name string 30 | Host string 31 | Port uint32 32 | SSL bool 33 | SkipSslCheck bool 34 | Username string 35 | Password string 36 | MaxConns uint32 37 | ConnWaitTime time.Duration 38 | IdleTimeout time.Duration 39 | HealthCheck bool 40 | MaxTooManyConnsErrors uint32 41 | MaxConnErrors uint32 42 | 43 | pool nntpPool.ConnectionPool 44 | capabilities struct { 45 | ihave bool 46 | post bool 47 | } 48 | articles struct { 49 | checked atomic.Uint64 50 | available atomic.Uint64 51 | missing atomic.Uint64 52 | refreshed atomic.Uint64 53 | } 54 | } 55 | 56 | Config struct { 57 | providers []Provider 58 | } 59 | ) 60 | 61 | type ( 62 | segmentChanItem struct { 63 | segment nzbparser.NzbSegment 64 | fileName string 65 | } 66 | providerStatistic map[string]uint64 67 | fileStatistic struct { 68 | available providerStatistic 69 | totalSegments uint64 70 | } 71 | filesStatistic map[string]*fileStatistic 72 | ) 73 | 74 | var ( 75 | appName = "NZBRefresh" 76 | appVersion = "" // Github tag 77 | nzbfile *nzbparser.Nzb // the parsed NZB file structure 78 | providerList []Provider // the parsed provider list structure 79 | 80 | ihaveProviders []*Provider // Providers with IHAVE capability 81 | postProviders []*Provider // Providers with POST capability 82 | 83 | err error 84 | maxConns uint32 85 | maxConnsLock sync.Mutex 86 | segmentChan chan segmentChanItem 87 | segmentChanWG sync.WaitGroup 88 | sendArticleWG sync.WaitGroup 89 | 90 | preparationStartTime time.Time 91 | segmentCheckStartTime time.Time 92 | segmentBar *cmpb.Bar 93 | uploadBar *cmpb.Bar 94 | uploadBarStarted bool 95 | uploadBarMutex sync.Mutex 96 | progressBars = cmpb.NewWithParam(&cmpb.Param{ 97 | Interval: 200 * time.Microsecond, 98 | Out: color.Output, 99 | ScrollUp: cmpb.AnsiScrollUp, 100 | PrePad: 0, 101 | KeyWidth: 18, 102 | MsgWidth: 5, 103 | PreBarWidth: 15, 104 | BarWidth: 42, 105 | PostBarWidth: 25, 106 | Post: "...", 107 | KeyDiv: ':', 108 | LBracket: '[', 109 | RBracket: ']', 110 | Empty: '-', 111 | Full: '=', 112 | Curr: '>', 113 | }) 114 | fileStat = make(filesStatistic) 115 | fileStatLock sync.Mutex 116 | ) 117 | 118 | func init() { 119 | parseArguments() 120 | fmt.Println(args.Version()) 121 | 122 | if args.Debug { 123 | logFileName := strings.TrimSuffix(filepath.Base(args.NZBFile), filepath.Ext(filepath.Base(args.NZBFile))) + ".log" 124 | f, err := os.Create(logFileName) 125 | if err != nil { 126 | exit(fmt.Errorf("unable to open debug log file: %v", err)) 127 | } 128 | log.SetOutput(f) 129 | } else { 130 | log.SetOutput(io.Discard) 131 | } 132 | 133 | log.Print("preparing...") 134 | preparationStartTime = time.Now() 135 | // parse the argument 136 | 137 | // load the NZB file 138 | if nzbfile, err = loadNzbFile(args.NZBFile); err != nil { 139 | exit(fmt.Errorf("unable to load NZB file '%s': %v'", args.NZBFile, err)) 140 | } 141 | 142 | // load the provider list 143 | if providerList, err = loadProviderList(args.Provider); err != nil { 144 | exit(fmt.Errorf("unable to load provider list: %v", err)) 145 | } 146 | 147 | go func() { 148 | for { 149 | select { 150 | case v := <-nntpPool.LogChan: 151 | log.Printf("NNTPPool: %v", v) 152 | // case d := <-nntpPool.DebugChan: 153 | // log.Printf("NNTPPool: %v", d) 154 | case w := <-nntpPool.WarnChan: 155 | log.Printf("NNTPPool Error: %v", w) 156 | } 157 | } 158 | }() 159 | 160 | // setup the nntp connection pool for each provider 161 | var providerWG sync.WaitGroup 162 | for n := range providerList { 163 | n := n 164 | providerWG.Add(1) 165 | go func(provider *Provider) { 166 | defer providerWG.Done() 167 | if pool, err := nntpPool.New(&nntpPool.Config{ 168 | Name: provider.Name, 169 | Host: provider.Host, 170 | Port: provider.Port, 171 | SSL: provider.SSL, 172 | SkipSSLCheck: provider.SkipSslCheck, 173 | User: provider.Username, 174 | Pass: provider.Password, 175 | MaxConns: provider.MaxConns, 176 | ConnWaitTime: time.Duration(provider.ConnWaitTime) * time.Second, 177 | IdleTimeout: time.Duration(provider.IdleTimeout) * time.Second, 178 | HealthCheck: provider.HealthCheck, 179 | MaxTooManyConnsErrors: provider.MaxTooManyConnsErrors, 180 | MaxConnErrors: provider.MaxConnErrors, 181 | }, 0); err != nil { 182 | exit(fmt.Errorf("unable to create the connection pool for provider '%s': %v", providerList[n].Name, err)) 183 | } else { 184 | provider.pool = pool 185 | } 186 | 187 | // calculate the max connections 188 | maxConnsLock.Lock() 189 | if maxConns < provider.MaxConns { 190 | maxConns = provider.MaxConns 191 | } 192 | maxConnsLock.Unlock() 193 | 194 | // check the ihave and post capabilities of the provider 195 | if ihave, post, err := checkCapabilities(&providerList[n]); err != nil { 196 | exit(fmt.Errorf("unable to check capabilities of provider '%s': %v", providerList[n].Name, err)) 197 | } else { 198 | providerList[n].capabilities.ihave = ihave 199 | providerList[n].capabilities.post = post 200 | log.Printf("capabilities of '%s': IHAVE: %v | POST: %v", providerList[n].Name, ihave, post) 201 | } 202 | }(&providerList[n]) 203 | } 204 | providerWG.Wait() 205 | 206 | // check if we have at least one provider with IHAVE or POST capability 207 | for n := range providerList { 208 | if providerList[n].capabilities.ihave { 209 | ihaveProviders = append(ihaveProviders, &providerList[n]) 210 | } 211 | if providerList[n].capabilities.post { 212 | postProviders = append(postProviders, &providerList[n]) 213 | } 214 | } 215 | if len(ihaveProviders) == 0 && len(postProviders) == 0 { 216 | log.Print("no provider has IHAVE or POST capability") 217 | } 218 | 219 | // make the channels 220 | segmentChan = make(chan segmentChanItem, 8*maxConns) 221 | 222 | // run the go routines 223 | for i := uint32(0); i < 4*maxConns; i++ { 224 | go processSegment() 225 | } 226 | 227 | log.Printf("preparation took %v", time.Since(preparationStartTime)) 228 | } 229 | 230 | func main() { 231 | 232 | startString := fmt.Sprintf("starting segment check of %v segments", nzbfile.TotalSegments) 233 | if args.CheckOnly { 234 | startString = startString + " (check only, no re-upload)" 235 | } 236 | fmt.Println(strings.ToUpper(startString[:1]) + startString[1:]) 237 | log.Print(startString) 238 | segmentCheckStartTime = time.Now() 239 | 240 | // segment check progressbar 241 | segmentBar = progressBars.NewBar("Checking segments", nzbfile.TotalSegments) 242 | segmentBar.SetPreBar(cmpb.CalcSteps) 243 | segmentBar.SetPostBar(cmpb.CalcTime) 244 | 245 | // start progressbar 246 | progressBars.Start() 247 | 248 | // loop through all file tags within the NZB file 249 | for _, file := range nzbfile.Files { 250 | fileStatLock.Lock() 251 | fileStat[file.Filename] = new(fileStatistic) 252 | fileStat[file.Filename].available = make(providerStatistic) 253 | fileStat[file.Filename].totalSegments = uint64(file.TotalSegments) 254 | fileStatLock.Unlock() 255 | // loop through all segment tags within each file tag 256 | for _, segment := range file.Segments { 257 | segmentChanWG.Add(1) 258 | segmentChan <- segmentChanItem{segment, file.Filename} 259 | } 260 | } 261 | segmentChanWG.Wait() 262 | segmentBar.SetMessage("done") 263 | sendArticleWG.Wait() 264 | if uploadBarStarted { 265 | uploadBar.SetMessage("done") 266 | } 267 | progressBars.Wait() 268 | log.Printf("segment check took %v | %v ms/segment", time.Since(segmentCheckStartTime), float32(time.Since(segmentCheckStartTime).Milliseconds())/float32(nzbfile.Segments)) 269 | for n := range providerList { 270 | result := fmt.Sprintf("Results for '%s': checked: %v | available: %v | missing: %v | refreshed: %v | %v connections used", 271 | providerList[n].Name, 272 | providerList[n].articles.checked.Load(), 273 | providerList[n].articles.available.Load(), 274 | providerList[n].articles.missing.Load(), 275 | providerList[n].articles.refreshed.Load(), 276 | providerList[n].pool.MaxConns(), 277 | ) 278 | fmt.Println(result) 279 | log.Print(result) 280 | } 281 | for n := range providerList { 282 | go providerList[n].pool.Close() 283 | } 284 | runtime := fmt.Sprintf("Total runtime %v | %v ms/segment", time.Since(preparationStartTime), float32(time.Since(preparationStartTime).Milliseconds())/float32(nzbfile.Segments)) 285 | fmt.Println(runtime) 286 | log.Print(runtime) 287 | writeCsvFile() 288 | } 289 | 290 | func loadNzbFile(path string) (*nzbparser.Nzb, error) { 291 | if b, err := os.Open(path); err != nil { 292 | return nil, err 293 | } else { 294 | defer b.Close() 295 | if nzbfile, err := nzbparser.Parse(b); err != nil { 296 | return nil, err 297 | } else { 298 | return nzbfile, nil 299 | } 300 | } 301 | } 302 | 303 | func loadProviderList(path string) ([]Provider, error) { 304 | if file, err := os.ReadFile(path); err != nil { 305 | return nil, err 306 | } else { 307 | cfg := Config{} 308 | if err := json.Unmarshal(file, &cfg.providers); err != nil { 309 | return nil, err 310 | } 311 | return cfg.providers, nil 312 | } 313 | } 314 | 315 | func checkCapabilities(provider *Provider) (bool, bool, error) { 316 | if conn, err := provider.pool.Get(context.TODO()); err != nil { 317 | return false, false, err 318 | } else { 319 | defer provider.pool.Put(conn) 320 | var ihave, post bool 321 | if capabilities, err := conn.Capabilities(); err == nil { 322 | for _, capability := range capabilities { 323 | if strings.ToLower(capability) == "ihave" { 324 | ihave = true 325 | } 326 | if strings.ToLower(capability) == "post" { 327 | post = true 328 | } 329 | } 330 | } else { 331 | // nntp server is not RFC 3977 compliant 332 | // check post capability 333 | article := new(nntp.Article) 334 | if err := conn.Post(article); err != nil { 335 | if err.Error()[0:3] != "440" { 336 | post = true 337 | } 338 | } else { 339 | post = true 340 | } 341 | // check ihave capability 342 | if err := conn.IHave(article); err != nil { 343 | if err.Error()[0:3] != "500" { 344 | ihave = true 345 | } 346 | } else { 347 | ihave = true 348 | } 349 | } 350 | return ihave, post, nil 351 | } 352 | } 353 | 354 | func processSegment() { 355 | for segmentChanItem := range segmentChan { 356 | segment := segmentChanItem.segment 357 | fileName := segmentChanItem.fileName 358 | func() { 359 | defer func() { 360 | segmentChanWG.Done() 361 | segmentBar.Increment() 362 | }() 363 | // positiv provider list (providers who have the article) 364 | var availableOn []*Provider 365 | var availableOnLock sync.Mutex 366 | // negative provider list (providers who don't have the article) 367 | var missingOn []*Provider 368 | var missingOnLock sync.Mutex 369 | // segment check waitgroup 370 | var segmentCheckWG sync.WaitGroup 371 | // loop through each provider in the provider list 372 | for n := range providerList { 373 | n := n 374 | segmentCheckWG.Add(1) 375 | go func() { 376 | defer segmentCheckWG.Done() 377 | // check if message is available on the provider 378 | if isAvailable, err := checkMessageID(&providerList[n], segment.Id); err != nil { 379 | // error handling 380 | log.Print(fmt.Errorf("unable to check article <%s> on provider '%s': %v", segment.Id, providerList[n].Name, err)) 381 | // TODO: What do we do with such errors?? 382 | } else { 383 | providerList[n].articles.checked.Add(1) 384 | if isAvailable { 385 | providerList[n].articles.available.Add(1) 386 | fileStatLock.Lock() 387 | fileStat[fileName].available[providerList[n].Name]++ 388 | fileStatLock.Unlock() 389 | // if yes add the provider to the positiv list 390 | availableOnLock.Lock() 391 | availableOn = append(availableOn, &providerList[n]) 392 | availableOnLock.Unlock() 393 | } else { 394 | providerList[n].articles.missing.Add(1) 395 | // if yes add the provider to the positiv list 396 | missingOnLock.Lock() 397 | missingOn = append(missingOn, &providerList[n]) 398 | missingOnLock.Unlock() 399 | } 400 | } 401 | }() 402 | } 403 | segmentCheckWG.Wait() 404 | // if negativ list contains entries at least one provider is missing the article 405 | if !args.CheckOnly && len(missingOn) > 0 { 406 | log.Printf("article <%s> is missing on at least one provider", segment.Id) 407 | // check if positiv list contains entries 408 | // without at least on provider having the article we cannot fix the others 409 | if len(availableOn) > 0 { 410 | uploadBarMutex.Lock() 411 | if uploadBarStarted { 412 | uploadBar.IncrementTotal() 413 | } else { 414 | uploadBar = progressBars.NewBar("Uploading articles", 1) 415 | uploadBar.SetPreBar(cmpb.CalcSteps) 416 | uploadBar.SetPostBar(cmpb.CalcTime) 417 | uploadBarStarted = true 418 | } 419 | uploadBarMutex.Unlock() 420 | // load article 421 | if article, err := loadArticle(availableOn, segment.Id); err != nil { 422 | log.Print(err) 423 | uploadBar.Increment() 424 | } else { 425 | // reupload article 426 | sendArticleWG.Add(1) 427 | go func() { 428 | // reupload article 429 | if err := reuploadArticle(missingOn, article, segment.Id); err != nil { 430 | // on error, try re-uploading on one of the providers having the article 431 | if err := reuploadArticle(availableOn, article, segment.Id); err != nil { 432 | log.Print(err) 433 | } 434 | } 435 | uploadBar.Increment() 436 | sendArticleWG.Done() 437 | }() 438 | } 439 | } else { 440 | // error handling if article is missing on all providers 441 | log.Print(fmt.Errorf("article <%s> is missing on all providers", segment.Id)) 442 | } 443 | } 444 | }() 445 | } 446 | } 447 | 448 | func checkMessageID(provider *Provider, messageID string) (bool, error) { 449 | if conn, err := provider.pool.Get(context.TODO()); err != nil { 450 | return false, err 451 | } else { 452 | defer provider.pool.Put(conn) 453 | if _, _, err := conn.Stat("<" + messageID + ">"); err == nil { 454 | // if article is availabel return true 455 | return true, nil 456 | } else { 457 | if err.Error()[0:3] == "430" { 458 | // upon error "430 No Such Article" return false 459 | return false, nil 460 | } else { 461 | // upon any other error return error 462 | return false, err 463 | } 464 | } 465 | } 466 | } 467 | 468 | func loadArticle(providerList []*Provider, messageID string) (*nntp.Article, error) { 469 | for _, provider := range providerList { 470 | // try to load the article from the provider 471 | log.Printf("loading article <%s> from provider '%s'", messageID, provider.Name) 472 | if article, err := getArticleFromProvider(provider, messageID); err != nil { 473 | // if the article cannot be loaded continue with the next provider on the list 474 | log.Print(fmt.Errorf("unable to load article <%s> from provider '%s': %v", messageID, provider.Name, err)) 475 | continue 476 | } else { 477 | return article, err 478 | } 479 | } 480 | return nil, fmt.Errorf("unable to load article <%s> from any provider", messageID) 481 | } 482 | 483 | func getArticleFromProvider(provider *Provider, messageID string) (*nntp.Article, error) { 484 | if conn, err := provider.pool.Get(context.TODO()); err != nil { 485 | return nil, err 486 | } else { 487 | defer provider.pool.Put(conn) 488 | if article, err := conn.Article("<" + messageID + ">"); err != nil { 489 | return nil, err 490 | } else { 491 | return copyArticle(article, []byte{}) 492 | } 493 | } 494 | } 495 | 496 | func reuploadArticle(providerList []*Provider, article *nntp.Article, segmentID string) error { 497 | var body []byte 498 | body, err := io.ReadAll(article.Body) 499 | if err != nil { 500 | return err 501 | } 502 | article.Body = bytes.NewReader(body) 503 | for n, provider := range providerList { 504 | if provider.capabilities.post { 505 | if copiedArticle, err := copyArticle(article, body); err != nil { 506 | return err 507 | } else { 508 | // send the article to the provider 509 | log.Printf("re-uploading article <%s> to provider '%s' (%v. attempt)", segmentID, provider.Name, n+1) 510 | if err := postArticleToProvider(provider, copiedArticle); err != nil { 511 | // error handling if re-uploading the article was unsuccessfull 512 | log.Print(fmt.Errorf("error re-uploading article <%s> to provider '%s': %v", segmentID, provider.Name, err)) 513 | } else { 514 | provider.articles.refreshed.Add(1) 515 | // handling of successfull send 516 | log.Printf("article <%s> successfully sent to provider '%s'", segmentID, provider.Name) 517 | // if post was successfull return 518 | // other providers missing this article will get it from this provider 519 | return nil 520 | } 521 | } 522 | } 523 | } 524 | return fmt.Errorf("unable to re-upload article <%s> to any provider", segmentID) 525 | } 526 | 527 | func postArticleToProvider(provider *Provider, article *nntp.Article) error { 528 | if conn, err := provider.pool.Get(context.TODO()); err != nil { 529 | return err 530 | } else { 531 | defer provider.pool.Put(conn) 532 | // for post, first clean the headers 533 | cleanHeaders(article) 534 | // post the article 535 | if err := conn.Post(article); err != nil { 536 | return err 537 | } else { 538 | return nil 539 | } 540 | } 541 | } 542 | 543 | func cleanHeaders(article *nntp.Article) { 544 | // minimum headers required for post 545 | headers := []string{ 546 | "From", 547 | "Subject", 548 | "Newsgroups", 549 | "Message-Id", 550 | "Date", 551 | "Path", 552 | } 553 | for header := range article.Header { 554 | if slices.Contains(headers, header) { 555 | // clean Path header 556 | if header == "Path" { 557 | article.Header[header] = []string{"not-for-mail"} 558 | } 559 | // update Date header to now 560 | if header == "Date" { 561 | article.Header[header] = []string{time.Now().Format(time.RFC1123Z)} 562 | } 563 | } else { 564 | delete(article.Header, header) 565 | } 566 | } 567 | } 568 | 569 | func copyArticle(article *nntp.Article, body []byte) (*nntp.Article, error) { 570 | var err error 571 | if len(body) == 0 { 572 | body, err = io.ReadAll(article.Body) 573 | if err != nil { 574 | return nil, err 575 | } 576 | } 577 | newArticle := nntp.Article{ 578 | Header: make(map[string][]string), 579 | } 580 | for header := range article.Header { 581 | newArticle.Header[header] = append(newArticle.Header[header], article.Header[header]...) 582 | } 583 | newArticle.Body = bytes.NewReader(body) 584 | return &newArticle, nil 585 | } 586 | 587 | func writeCsvFile() { 588 | if args.Csv { 589 | csvFileName := strings.TrimSuffix(filepath.Base(args.NZBFile), filepath.Ext(filepath.Base(args.NZBFile))) + ".csv" 590 | f, err := os.Create(csvFileName) 591 | if err != nil { 592 | exit(fmt.Errorf("unable to open csv file: %v", err)) 593 | } 594 | log.Println("writing csv file...") 595 | fmt.Print("Writing csv file... ") 596 | csvWriter := csv.NewWriter(f) 597 | firstLine := true 598 | // make sorted provider name slice 599 | providers := make([]string, 0, len(providerList)) 600 | for n := range providerList { 601 | providers = append(providers, providerList[n].Name) 602 | } 603 | sort.Strings(providers) 604 | for fileName, file := range fileStat { 605 | // write first line 606 | if firstLine { 607 | line := make([]string, len(providers)+2) 608 | line[0] = "Filename" 609 | line[1] = "Total segments" 610 | for n, providerName := range providers { 611 | line[n+2] = providerName 612 | } 613 | if err := csvWriter.Write(line); err != nil { 614 | exit(fmt.Errorf("unable to write to the csv file: %v", err)) 615 | } 616 | firstLine = false 617 | } 618 | // write line 619 | line := make([]string, len(providers)+2) 620 | line[0] = fileName 621 | line[1] = fmt.Sprintf("%v", file.totalSegments) 622 | for n, providerName := range providers { 623 | if value, ok := file.available[providerName]; ok { 624 | line[n+2] = fmt.Sprintf("%v", value) 625 | } else { 626 | line[n+2] = "0" 627 | } 628 | } 629 | if err := csvWriter.Write(line); err != nil { 630 | exit(fmt.Errorf("unable to write to the csv file: %v", err)) 631 | } 632 | } 633 | csvWriter.Flush() 634 | if err := csvWriter.Error(); err != nil { 635 | exit(fmt.Errorf("unable to write to the csv file: %v", err)) 636 | } 637 | f.Close() 638 | fmt.Print("done") 639 | } 640 | } 641 | 642 | func exit(err error) { 643 | if err != nil { 644 | fmt.Printf("Fatal error: %v\n", err) 645 | log.Fatal(err) 646 | } else { 647 | os.Exit(0) 648 | } 649 | } 650 | -------------------------------------------------------------------------------- /provider.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Name": "Provider 1", 4 | "Host": "", 5 | "Port": 119, 6 | "SSL": false, 7 | "SkipSslCheck": true, 8 | "Username": "", 9 | "Password": "", 10 | "ConnWaitTime": 10, 11 | "MaxConns": 50, 12 | "IdleTimeout": 30, 13 | "HealthCheck": false, 14 | "MaxTooManyConnsErrors": 3, 15 | "MaxConnErrors": 3 16 | }, 17 | { 18 | "Name": "Provider 2", 19 | "Host": "", 20 | "Port": 119, 21 | "SSL": false, 22 | "SkipSslCheck": true, 23 | "Username": "", 24 | "Password": "", 25 | "ConnWaitTime": 10, 26 | "MaxConns": 50, 27 | "IdleTimeout": 30, 28 | "HealthCheck": false, 29 | "MaxTooManyConnsErrors": 3, 30 | "MaxConnErrors": 3 31 | }, 32 | { 33 | "Name": "Provider 3", 34 | "Host": "", 35 | "Port": 119, 36 | "SSL": false, 37 | "SkipSslCheck": true, 38 | "Username": "", 39 | "Password": "", 40 | "ConnWaitTime": 10, 41 | "MaxConns": 50, 42 | "IdleTimeout": 30, 43 | "HealthCheck": false, 44 | "MaxTooManyConnsErrors": 3, 45 | "MaxConnErrors": 3 46 | }, 47 | { 48 | "Name": "Provider 4", 49 | "Host": "", 50 | "Port": 119, 51 | "SSL": false, 52 | "SkipSslCheck": true, 53 | "Username": "", 54 | "Password": "", 55 | "ConnWaitTime": 10, 56 | "MaxConns": 50, 57 | "IdleTimeout": 30, 58 | "HealthCheck": false, 59 | "MaxTooManyConnsErrors": 3, 60 | "MaxConnErrors": 3 61 | } 62 | ] --------------------------------------------------------------------------------