├── CODEOWNERS ├── go.mod ├── go.sum ├── .gitignore ├── .github ├── workflows │ ├── go.yml │ └── coverage.yml └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── .octocov.yml ├── LICENSE ├── examples ├── text │ └── text_scanner.go └── file │ └── file_scanner.go ├── webhook_test.go ├── alert.go ├── text_test.go ├── redaction.go ├── webhook.go ├── README.md ├── CONTRIBUTING.md ├── nightfall_test.go ├── text.go ├── model.go ├── file_test.go ├── file.go └── nightfall.go /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @victor88121 @evanfuller @dhertz 2 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nightfallai/nightfall-go-sdk 2 | 3 | go 1.18 4 | 5 | require github.com/google/uuid v1.3.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 2 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Test binary, build with `go test -c` 3 | *.test 4 | 5 | # Output of the go coverage tool, specifically when used with LiteIDE 6 | *.out 7 | 8 | # different editor settings 9 | .idea 10 | .vscode 11 | *.swo 12 | *.swp 13 | 14 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: [push] 4 | 5 | jobs: 6 | 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | 12 | - name: Set up Go 13 | uses: actions/setup-go@v2 14 | with: 15 | go-version: 1.17 16 | 17 | - name: Build 18 | run: go build -v ./... 19 | 20 | - name: Test 21 | run: go test -v ./... 22 | -------------------------------------------------------------------------------- /.octocov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | acceptable: current >= 89% 3 | codeToTestRatio: 4 | code: 5 | - '**/*.go' 6 | - '!**/*_test.go' 7 | test: 8 | - '**/*_test.go' 9 | diff: 10 | datastores: 11 | - artifact://${GITHUB_REPOSITORY} 12 | push: 13 | if: is_default_branch 14 | comment: 15 | if: is_pull_request 16 | summary: 17 | if: true 18 | report: 19 | if: is_default_branch 20 | datastores: 21 | - artifact://${GITHUB_REPOSITORY} 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | Code Sample: 17 | ```go 18 | func (c *Client) NewAndImprovedScan() { 19 | 20 | } 21 | ``` 22 | 23 | **Describe alternatives you've considered** 24 | A clear and concise description of any alternative solutions or features you've considered. 25 | 26 | **Additional context** 27 | Add any other context or screenshots about the feature request here. 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Report an issue with the library or API 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Please do not file bugs for active security vulnerabilities here! Instead, please email security@nightfall.ai** 11 | 12 | **Describe the bug** 13 | A clear and concise description of what the bug is. 14 | 15 | **To Reproduce** 16 | Steps to reproduce the behavior: 17 | 1. Go to '...' 18 | 2. Click on '....' 19 | 3. Scroll down to '....' 20 | 4. See error 21 | 22 | **Expected behavior** 23 | A clear and concise description of what you expected to happen. 24 | 25 | **Screenshots** 26 | If applicable, add screenshots to help explain your problem. 27 | 28 | **Environment Information** 29 | - Go version: [e.g. 1.17] 30 | - OS: [e.g. macOS] 31 | - Architecture: [e.g. x86-64] 32 | 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: octocov 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | 8 | jobs: 9 | coverage: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Granting private modules access 13 | run: | 14 | git config --global url."https://${{ secrets.engbot_token }}:x-oauth-basic@github.com/watchtowerai".insteadOf "https://github.com/watchtowerai" 15 | - uses: actions/checkout@v4 16 | - uses: actions/setup-go@v4 17 | with: 18 | go-version-file: go.mod 19 | - name: Run tests with coverage report output 20 | # can skip packages with external dependencies (e.g. entity with live DB) with a command like 21 | # ```go test $(go list ./... | grep -v entity) -coverprofile=coverage.out``` 22 | run: go test $(go list ./... | grep -v /internal/services/entity | grep -v /internal/entities | grep -v /internal/handler | grep -v /internal/services/entity) -coverprofile=coverage.out 23 | env: 24 | GO_ENV: "test" 25 | GOPRIVATE: "github.com/watchtowerai" 26 | - uses: k1LoW/octocov-action@v0 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Nightfall AI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/text/text_scanner.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/nightfallai/nightfall-go-sdk" 9 | ) 10 | 11 | func scanText() (*nightfall.ScanTextResponse, error) { 12 | nc, err := nightfall.NewClient() 13 | if err != nil { 14 | return nil, fmt.Errorf("Error initializing client: %w", err) 15 | } 16 | 17 | resp, err := nc.ScanText(context.Background(), &nightfall.ScanTextRequest{ 18 | Payload: []string{"4242 4242 4242 4242 is my ccn"}, 19 | Config: &nightfall.Config{ 20 | // A rule contains a set of detectors to scan with 21 | DetectionRules: []nightfall.DetectionRule{{ 22 | // Define some detectors to use to scan your data 23 | Detectors: []nightfall.Detector{{ 24 | MinNumFindings: 1, 25 | MinConfidence: nightfall.ConfidencePossible, 26 | DisplayName: "cc#", 27 | DetectorType: nightfall.DetectorTypeNightfallDetector, 28 | NightfallDetector: "CREDIT_CARD_NUMBER", 29 | }}, 30 | LogicalOp: nightfall.LogicalOpAny, 31 | }, 32 | }, 33 | }, 34 | }) 35 | if err != nil { 36 | return nil, fmt.Errorf("Error scanning text: %w", err) 37 | } 38 | return resp, nil 39 | } 40 | 41 | func main() { 42 | resp, err := scanText() 43 | if err != nil { 44 | fmt.Printf("Got error: %v", err) 45 | os.Exit(-1) 46 | } 47 | for _, findings := range resp.Findings { 48 | for _, finding := range findings { 49 | fmt.Printf("Got finding %v", finding) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /webhook_test.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestValidate(t *testing.T) { 9 | tests := []struct { 10 | name string 11 | threshold time.Duration 12 | reqBody string 13 | reqSignature string 14 | reqTimestamp string 15 | expValid bool 16 | }{ 17 | { 18 | name: "happy path", 19 | threshold: time.Hour * 24 * 365 * 100, 20 | reqBody: "hello world", 21 | reqSignature: "3ccf9cc16507ca9f55b73c94fc2e872bb7ea312b2ab9d90785c6c55f153df4f3", 22 | reqTimestamp: "1633368643", // 2021-10-04T17:30:43Z 23 | expValid: true, 24 | }, 25 | { 26 | name: "invalid signature", 27 | threshold: time.Hour * 24 * 365 * 100, 28 | reqBody: "hello world", 29 | reqSignature: "fe07c9a938ac1da7e1c14774bff295f27a05b3cb4e78275eeb873977322b63d1", 30 | reqTimestamp: "1633368643", // 2021-10-04T17:30:43Z 31 | expValid: false, 32 | }, 33 | { 34 | name: "request time past threshold", 35 | threshold: time.Minute, 36 | reqBody: "hello world", 37 | reqSignature: "3ccf9cc16507ca9f55b73c94fc2e872bb7ea312b2ab9d90785c6c55f153df4f3", 38 | reqTimestamp: "1633368643", // 2021-10-04T17:30:43Z 39 | expValid: false, 40 | }, 41 | } 42 | 43 | for _, test := range tests { 44 | validator := NewWebhookValidator([]byte("some secret"), OptionThreshold(test.threshold)) 45 | valid, err := validator.Validate(test.reqBody, test.reqSignature, test.reqTimestamp) 46 | if err != nil { 47 | t.Errorf("unexpected error validating request: %v", err) 48 | } 49 | if valid != test.expValid { 50 | t.Error("did not get expected validation result") 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /alert.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | // AlertConfig allows clients to specify where alerts should be delivered when findings are discovered as 4 | // part of a scan. These alerts are delivered asynchronously to all destinations specified in the object instance. 5 | type AlertConfig struct { 6 | Slack *SlackAlert `json:"slack"` 7 | Email *EmailAlert `json:"email"` 8 | Webhook *WebhookAlert `json:"url"` 9 | } 10 | 11 | // SlackAlert contains the configuration required to allow clients to send asynchronous alerts to a Slack 12 | // workspace when findings are detected. 13 | // 14 | // Note that in order for Slack alerts to be delivered to your workspace, you must use authenticate Nightfall 15 | // to your Slack workspace under the Settings menu on the Nightfall Dashboard. 16 | // 17 | // Currently, Nightfall supports delivering alerts to public channels, formatted like "#general". 18 | // Alerts are only sent if findings are detected. 19 | type SlackAlert struct { 20 | Target string `json:"target"` 21 | } 22 | 23 | // EmailAlert contains the configuration required to allow clients to send an asynchronous email message 24 | // when findings are detected. The findings themselves will be delivered as a file attachment on the email. 25 | // Alerts are only sent if findings are detected. 26 | type EmailAlert struct { 27 | Address string `json:"address"` 28 | } 29 | 30 | // WebhookAlert contains the configuration required to allow clients to send a webhook event to an external 31 | // URL when findings are detected. The URL provided must have a route defined on the HTTP POST method, 32 | // and should return a 200 status code upon receipt of the event. 33 | // 34 | // In contrast to other platforms, when using the file scanning APIs, an alert is also sent to this webhook 35 | // *even when there are no findings*. 36 | type WebhookAlert struct { 37 | Address string `json:"address"` 38 | } 39 | -------------------------------------------------------------------------------- /text_test.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | ) 9 | 10 | func TestScanText(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | handler http.HandlerFunc 14 | wantErr bool 15 | }{ 16 | { 17 | name: "happy path", 18 | handler: func(w http.ResponseWriter, r *http.Request) { 19 | w.WriteHeader(http.StatusOK) 20 | }, 21 | wantErr: false, 22 | }, 23 | { 24 | name: "transient error", 25 | handler: func(w http.ResponseWriter, r *http.Request) { 26 | w.WriteHeader(http.StatusInternalServerError) 27 | }, 28 | wantErr: true, 29 | }, 30 | } 31 | s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) 32 | defer s.Close() 33 | 34 | client, err := NewClient(OptionAPIKey("some key")) 35 | if err != nil { 36 | t.Fatal("Error initializing client") 37 | } 38 | client.baseURL = s.URL + "/" 39 | 40 | for _, test := range tests { 41 | t.Run(test.name, func(t *testing.T) { 42 | s.Config.Handler = test.handler 43 | _, err = client.ScanText(context.Background(), &ScanTextRequest{ 44 | Payload: []string{"4242 4242 4242 4242"}, 45 | Policy: &Config{ 46 | DetectionRules: []DetectionRule{{ 47 | Detectors: []Detector{{ 48 | MinNumFindings: 1, 49 | MinConfidence: ConfidencePossible, 50 | DisplayName: "cc#", 51 | DetectorType: DetectorTypeNightfallDetector, 52 | NightfallDetector: "CREDIT_CARD_NUMBER", 53 | }}, 54 | LogicalOp: LogicalOpAny, 55 | }}, 56 | DefaultRedactionConfig: &RedactionConfig{ 57 | MaskConfig: &MaskConfig{ 58 | MaskingChar: "🤫", 59 | CharsToIgnore: []string{"-"}, 60 | }, 61 | RemoveFinding: true, 62 | }, 63 | }, 64 | }) 65 | if !test.wantErr && err != nil { 66 | t.Errorf("Got unexpected error: %v", err) 67 | } 68 | if test.wantErr && err == nil { 69 | t.Error("Did not get expected error") 70 | } 71 | }) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /redaction.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | // RedactionConfig describes how any detected findings should be redacted when returned to the client. When this 4 | // configuration is provided as part of a request, exactly one of the four types of redaction should be non-nil: 5 | // 1. Masking: replacing the characters of a finding with another character, such as '*' or '👀' 6 | // 2. Info Type Substitution: replacing the finding with the name of the detector it matched, such 7 | // as CREDIT_CARD_NUMBER 8 | // 3. Substitution: replacing the finding with a custom string, such as "oh no!" 9 | // 4. Encryption: encrypting the finding with an RSA public key 10 | type RedactionConfig struct { 11 | MaskConfig *MaskConfig `json:"maskConfig,omitempty"` 12 | InfoTypeSubstitutionConfig *InfoTypeSubstitutionConfig `json:"infoTypeSubstitutionConfig,omitempty"` 13 | SubstitutionConfig *SubstitutionConfig `json:"substitutionConfig,omitempty"` 14 | CryptoConfig *CryptoConfig `json:"cryptoConfig,omitempty"` 15 | RemoveFinding bool `json:"removeFinding"` 16 | } 17 | 18 | // MaskConfig specifies how findings should be masked when returned by the API. 19 | type MaskConfig struct { 20 | MaskingChar string `json:"maskingChar"` 21 | CharsToIgnore []string `json:"charsToIgnore"` 22 | NumCharsToLeaveUnmasked int `json:"numCharsToLeaveUnmasked"` 23 | MaskLeftToRight bool `json:"maskLeftToRight"` 24 | } 25 | 26 | // InfoTypeSubstitutionConfig specifies that findings should be masked with the name of the matched info type. 27 | type InfoTypeSubstitutionConfig struct { 28 | } 29 | 30 | // SubstitutionConfig specifies that findings should be masked with a configured custom phrase. 31 | type SubstitutionConfig struct { 32 | SubstitutionPhrase string `json:"substitutionPhrase"` 33 | } 34 | 35 | // CryptoConfig specifies that findings should be encrypted with the provided RSA public key. 36 | type CryptoConfig struct { 37 | PublicKey string `json:"publicKey"` 38 | } 39 | -------------------------------------------------------------------------------- /examples/file/file_scanner.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/nightfallai/nightfall-go-sdk" 9 | ) 10 | 11 | func scanFile(webhookURL, filePath string) (*nightfall.ScanFileResponse, error) { 12 | nc, err := nightfall.NewClient() 13 | if err != nil { 14 | return nil, fmt.Errorf("Error initializing client: %w", err) 15 | } 16 | 17 | f, err := os.Open(filePath) 18 | if err != nil { 19 | return nil, fmt.Errorf("Error opening file: %w", err) 20 | } 21 | defer f.Close() 22 | 23 | fi, err := f.Stat() 24 | if err != nil { 25 | return nil, fmt.Errorf("Error getting file info: %w", err) 26 | } 27 | 28 | resp, err := nc.ScanFile(context.Background(), &nightfall.ScanFileRequest{ 29 | Policy: &nightfall.ScanPolicy{ 30 | // File scans are conducted asynchronously, so provide a webhook route to an HTTPS server to send results to. 31 | WebhookURL: webhookURL, 32 | // A rule contains a set of detectors to scan with 33 | DetectionRules: []nightfall.DetectionRule{{ 34 | // Define some detectors to use to scan your data 35 | Detectors: []nightfall.Detector{{ 36 | MinNumFindings: 1, 37 | MinConfidence: nightfall.ConfidencePossible, 38 | DisplayName: "cc#", 39 | DetectorType: nightfall.DetectorTypeNightfallDetector, 40 | NightfallDetector: "CREDIT_CARD_NUMBER", 41 | }}, 42 | LogicalOp: nightfall.LogicalOpAny, 43 | }, 44 | }, 45 | }, 46 | RequestMetadata: "{\"hello\": \"world\", \"goodnight\": \"moon\"}", 47 | Content: f, 48 | ContentSizeBytes: fi.Size(), 49 | Timeout: 0, 50 | }) 51 | if err != nil { 52 | return nil, fmt.Errorf("Error scanning file: %w", err) 53 | } 54 | return resp, nil 55 | } 56 | 57 | func main() { 58 | if len(os.Args) != 3 { 59 | fmt.Printf("Usage: file_scanner ") 60 | os.Exit(-1) 61 | } 62 | webhookURL := os.Args[1] 63 | filename := os.Args[2] 64 | 65 | resp, err := scanFile(webhookURL, filename) 66 | 67 | if err != nil { 68 | fmt.Printf("Error: %v", err) 69 | os.Exit(-1) 70 | } 71 | 72 | fmt.Printf("Got response with ID %s, message %s\n", resp.ID, resp.Message) 73 | } 74 | -------------------------------------------------------------------------------- /webhook.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "fmt" 8 | "strconv" 9 | "time" 10 | ) 11 | 12 | const DefaultThreshold = 5 * time.Minute 13 | 14 | // WebhookValidator validates incoming requests from Nightfall using the signing secret which can be fetched from the 15 | // Nightfall dashboard 16 | type WebhookValidator struct { 17 | signingSecret []byte 18 | threshold time.Duration 19 | } 20 | 21 | // WebhookValidatorOption defines an option for a WebhookValidator 22 | type WebhookValidatorOption func(*WebhookValidator) 23 | 24 | // NewWebhookValidator returns a new webhook validator 25 | func NewWebhookValidator(signingSecret []byte, options ...WebhookValidatorOption) *WebhookValidator { 26 | w := &WebhookValidator{ 27 | signingSecret: signingSecret, 28 | threshold: DefaultThreshold, 29 | } 30 | 31 | for _, opt := range options { 32 | opt(w) 33 | } 34 | 35 | return w 36 | } 37 | 38 | // OptionThreshold sets the threshold of the webhook validator. If the difference between the time the webhook is 39 | // received and the timestamp value sent in the X-Nightfall-Timestamp header is greater than this threshold, the 40 | // request will be rejected. 41 | func OptionThreshold(threshold time.Duration) func(*WebhookValidator) { 42 | return func(w *WebhookValidator) { 43 | w.threshold = threshold 44 | } 45 | } 46 | 47 | // Validates that the provided request payload is an authentic request that originated from Nightfall. If this 48 | // method returns false, request handlers shall not process the provided body any further. 49 | func (w *WebhookValidator) Validate(requestBody, requestSignature, requestTime string) (bool, error) { 50 | if requestBody == "" || requestSignature == "" || requestTime == "" { 51 | return false, nil 52 | } 53 | 54 | i, err := strconv.ParseInt(requestTime, 10, 64) 55 | if err != nil { 56 | return false, err 57 | } 58 | 59 | unixTime := time.Unix(i, 0) 60 | if time.Now().Sub(unixTime) > w.threshold { 61 | return false, nil 62 | } 63 | 64 | h := hmac.New(sha256.New, w.signingSecret) 65 | hashPayload := fmt.Sprintf("%s:%s", requestTime, requestBody) 66 | h.Write([]byte(hashPayload)) 67 | hexHash := hex.EncodeToString(h.Sum(nil)) 68 | 69 | return hexHash == requestSignature, nil 70 | } 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nightfall Go SDK # 2 | 3 | nightfall-go-sdk is a Go client library for accessing the Nightfall API. 4 | It allows you to add functionality to your applications to 5 | scan plain text and files in order to detect different categories of information. You can leverage any of 6 | the detectors in Nightfall's pre-built library, or you may programmatically define your own custom detectors. 7 | 8 | Additionally, this library provides convenient features including a streamlined function to manage the multi-stage file upload process. 9 | 10 | To obtain an API Key, login to the [Nightfall dashboard](https://app.nightfall.ai/) and click the section 11 | titled "Manage API Keys". 12 | 13 | See our [developer documentation](https://docs.nightfall.ai/docs/entities-and-terms-to-know) for more details about 14 | integrating with the Nightfall API. 15 | 16 | ## Installation ## 17 | 18 | Nightfall Go SDK is compatible with modern Go releases in module mode, with Go installed: 19 | 20 | ```bash 21 | go get github.com/nightfallai/nightfall-go-sdk 22 | ``` 23 | 24 | will resolve and add the package to the current development module, along with its dependencies. 25 | 26 | ## Usage 27 | 28 | ### Scanning Plain Text 29 | 30 | Nightfall provides pre-built detector types, covering data types ranging from PII to PHI to credentials. The following 31 | snippet shows an example of how to scan using pre-built detectors. 32 | 33 | #### Sample Code 34 | See [examples/text/text\_scanner.go](examples/text/text_scanner.go) for an example 35 | 36 | ### Scanning Files 37 | 38 | Scanning common file types like PDFs or office documents typically requires cumbersome text 39 | extraction methods like OCR. 40 | 41 | Rather than implementing this functionality yourself, the Nightfall API allows you to upload the 42 | original files, and then we'll handle the heavy lifting. 43 | 44 | The file upload process is implemented as a series of requests to upload the file in chunks. The library 45 | provides a single method that wraps the steps required to upload your file. Please refer to the 46 | [API Reference](https://docs.nightfall.ai/reference) for more details. 47 | 48 | The file is uploaded synchronously, but as files can be arbitrarily large, the scan itself is conducted asynchronously. 49 | The results from the scan are delivered by webhook; for more information about setting up a webhook server, refer to 50 | [the docs](https://docs.nightfall.ai/docs/creating-a-webhook-server). 51 | 52 | #### Sample Code 53 | 54 | See [examples/file/file\_scanner.go](examples/file/file_scanner.go) for an example 55 | 56 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for considering contributing to the Nightfall Go SDK. 4 | 5 | There are many ways to contribute, such as: 6 | * Writing code samples 7 | * Suggesting documentation improvements 8 | * Submitting bug reports and feature requests 9 | * Writing code to improve the library itself. 10 | 11 | Please, don't use the issue tracker for personal support questions. Feel free to reach out to `support@nightfall.ai` 12 | to address those issues. 13 | 14 | ## Responsibilities 15 | * Ensure cross-platform compatibility for every change that's accepted. Windows, Mac, Debian & Ubuntu Linux, etc. 16 | * Ensure backwards compatibility with older Go versions 17 | * Create issues for each major change and enhancement that you wish to make. 18 | * Discuss proposed changes transparently and collect community feedback. 19 | * Avoid introducing new external dependencies whenever possible. When absolutely required, validate the software 20 | licenses used by these dependencies (e.g. avoid unintentional copyleft requirements). 21 | 22 | ## How to report a bug 23 | 24 | ### Security Disclosures 25 | If you find a security vulnerability, do NOT open an issue. Email `security@nightfall.ai` instead. 26 | 27 | In order to determine whether you are dealing with a security issue, ask yourself the following questions: 28 | * Can I access something that's not mine, or something I shouldn't have access to? 29 | * Can I disable something for other people? 30 | * Is there a potential vulnerability stemming from a library dependency? 31 | 32 | If you answered yes to any of the above questions, then you're probably dealing with a security issue. 33 | Note that even if you answer "no" to all questions, you may still be dealing with a security issue, so if you're 34 | unsure, just email us at `security@nightfall.ai`. 35 | 36 | ### Creating an Issue 37 | When filing an issue, make sure to answer these questions: 38 | 1. What version of Go are you using? 39 | 2. What operating system and processor architecture are you using? 40 | 3. How did you discover the issue? 41 | 4. Is the issue reproducible? What are the steps to reproduce? 42 | 5. What did you expect to see? 43 | 6. What did you see instead? 44 | 45 | 46 | ## Suggesting a New Feature 47 | 48 | If you find yourself wishing for a feature that doesn't exist in this SDK, you are probably not alone. 49 | There are bound to be others out there with similar needs. Open an issue on our issues list on GitHub which 50 | describes the feature you would like to see, why you need it, and how it should work. 51 | 52 | ## Code Review 53 | 54 | The core team looks at open pull requests on a regular basis. In order for your pull request to be merged, it 55 | must meet the following requirements: 56 | * It must pass the linter; make sure to run `gofmt -w *.go`. 57 | * It must add unit tests to cover any new functionality. 58 | * It must get approval from one of the code owners. 59 | 60 | If a pull request remains idle for more than two weeks, we may close it. 61 | -------------------------------------------------------------------------------- /nightfall_test.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | ) 9 | 10 | func TestDo(t *testing.T) { 11 | var callCount int 12 | tests := []struct { 13 | name string 14 | handler http.HandlerFunc 15 | expCalls int 16 | wantErr bool 17 | }{ 18 | { 19 | name: "happy path", 20 | handler: func(w http.ResponseWriter, r *http.Request) { 21 | callCount++ 22 | w.WriteHeader(http.StatusOK) 23 | }, 24 | expCalls: 1, 25 | wantErr: false, 26 | }, 27 | { 28 | name: "happy path - retry 2 times", 29 | handler: func(w http.ResponseWriter, r *http.Request) { 30 | callCount++ 31 | if callCount == 3 { 32 | w.WriteHeader(http.StatusOK) 33 | return 34 | } 35 | w.WriteHeader(http.StatusTooManyRequests) 36 | }, 37 | expCalls: 3, 38 | wantErr: false, 39 | }, 40 | { 41 | name: "429 error after 5 retries", 42 | handler: func(w http.ResponseWriter, r *http.Request) { 43 | callCount++ 44 | w.WriteHeader(http.StatusTooManyRequests) 45 | }, 46 | expCalls: 6, 47 | wantErr: true, 48 | }, 49 | { 50 | name: "transient error", 51 | handler: func(w http.ResponseWriter, r *http.Request) { 52 | callCount++ 53 | w.WriteHeader(http.StatusInternalServerError) 54 | }, 55 | expCalls: 1, 56 | wantErr: true, 57 | }, 58 | } 59 | s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) 60 | defer s.Close() 61 | 62 | client, err := NewClient(OptionAPIKey("some key")) 63 | if err != nil { 64 | t.Fatal("Error initializing client") 65 | } 66 | 67 | reqParams := requestParams{ 68 | method: http.MethodPost, 69 | url: s.URL, 70 | } 71 | 72 | for _, test := range tests { 73 | t.Run(test.name, func(t *testing.T) { 74 | callCount = 0 75 | s.Config.Handler = test.handler 76 | err = client.do(context.Background(), reqParams, nil) 77 | if !test.wantErr && err != nil { 78 | t.Errorf("Got unexpected error: %v", err) 79 | } 80 | if test.wantErr && err == nil { 81 | t.Error("Did not get expected error") 82 | } 83 | if callCount != test.expCalls { 84 | t.Error("Did not call expected number of times") 85 | } 86 | }) 87 | } 88 | } 89 | 90 | func TestNewClient(t *testing.T) { 91 | tests := []struct { 92 | name string 93 | apiKey string 94 | fileUploadConcurrency int 95 | wantErr bool 96 | }{ 97 | { 98 | name: "happy path", 99 | apiKey: "some key", 100 | fileUploadConcurrency: 5, 101 | wantErr: false, 102 | }, 103 | { 104 | name: "missing api key", 105 | fileUploadConcurrency: 5, 106 | wantErr: true, 107 | }, 108 | { 109 | name: "file concurrency too high", 110 | apiKey: "some key", 111 | fileUploadConcurrency: 101, 112 | wantErr: true, 113 | }, 114 | } 115 | 116 | for _, test := range tests { 117 | t.Run(test.name, func(t *testing.T) { 118 | _, err := NewClient(OptionAPIKey(test.apiKey), OptionFileUploadConcurrency(test.fileUploadConcurrency)) 119 | if !test.wantErr && err != nil { 120 | t.Errorf("Got unexpected error: %v", err) 121 | } 122 | if test.wantErr && err == nil { 123 | t.Error("Did not get expected error") 124 | } 125 | }) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /text.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | ) 7 | 8 | // ScanTextRequest is the request struct to scan inline plaintext with the Nightfall API. 9 | type ScanTextRequest struct { 10 | Payload []string `json:"payload"` 11 | Config *Config `json:"config"` // Deprecated; use Policy instead 12 | Policy *Config `json:"policy"` 13 | PolicyUUIDs []string `json:"policyUUIDs"` 14 | } 15 | 16 | // ScanTextResponse is the response object returned by a text scan request. Each index i in the field `findings` 17 | // corresponds one-to-one with the input request payload, so all findings stored in a given sub-list 18 | // refer to matches that occurred in the ith index of the request payload. 19 | type ScanTextResponse struct { 20 | Findings [][]*Finding `json:"findings"` 21 | RedactedPayload []string `json:"redactedPayload"` 22 | } 23 | 24 | // Config is the configuration object to use when scanning inline plaintext with the Nightfall API. This 25 | // object represents an inline policy. 26 | type Config struct { 27 | DetectionRules []DetectionRule `json:"detectionRules"` 28 | DetectionRuleUUIDs []string `json:"detectionRuleUUIDs"` 29 | ContextBytes int `json:"contextBytes"` 30 | DefaultRedactionConfig *RedactionConfig `json:"defaultRedactionConfig"` 31 | AlertConfig *AlertConfig `json:"alertConfig"` 32 | } 33 | 34 | // Finding represents an occurrence of a configured detector (i.e. finding) in the provided data. 35 | type Finding struct { 36 | Finding string `json:"finding"` 37 | RedactedFinding string `json:"redactedFinding"` 38 | BeforeContext string `json:"beforeContext"` 39 | AfterContext string `json:"afterContext"` 40 | Detector DetectorMetadata `json:"detector"` 41 | Confidence string `json:"confidence"` 42 | Location *Location `json:"location"` 43 | RedactedLocation *Location `json:"redactedLocation"` 44 | MatchedDetectionRuleUUIDs []string `json:"matchedDetectionRuleUUIDs"` 45 | MatchedDetectionRules []string `json:"matchedDetectionRules"` 46 | FindingMetadata *FindingMetadata `json:"findingMetadata"` 47 | } 48 | 49 | type FindingMetadata struct { 50 | APIKeyMetadata *APIKeyMetadata `json:"apiKeyMetadata"` 51 | } 52 | 53 | type APIKeyMetadata struct { 54 | Status string `json:"status"` 55 | Kind string `json:"kind"` 56 | Description string `json:"description"` 57 | } 58 | 59 | // Location represents where a finding was discovered in content. 60 | // The Range fields may be nil depending on context; for example, `rowRange` and `columnRange` will only be non-nil if a finding is tabular. 61 | type Location struct { 62 | ByteRange *Range `json:"byteRange"` 63 | CodepointRange *Range `json:"codepointRange"` 64 | RowRange *Range `json:"rowRange"` 65 | ColumnRange *Range `json:"columnRange"` 66 | CommitHash string `json:"commitHash"` 67 | CommitAuthor string `json:"commitAuthor"` 68 | } 69 | 70 | // Range contains references to the start and end of the eponymous range. 71 | type Range struct { 72 | Start int64 `json:"start"` 73 | End int64 `json:"end"` 74 | } 75 | 76 | // ScanText scans the provided plaintext against the provided detectors, and returns all findings. The response 77 | // object will contain a list of lists representing the findings. Each index i in the findings array will 78 | // correspond one-to-one with the input request payload list, so all findings stored in a given sub-list refer to 79 | // matches that occurred in the ith index of the request payload. 80 | func (c *Client) ScanText(ctx context.Context, request *ScanTextRequest) (*ScanTextResponse, error) { 81 | body, err := encodeBodyAsJSON(request) 82 | if err != nil { 83 | return nil, err 84 | } 85 | reqParams := requestParams{ 86 | method: http.MethodPost, 87 | url: c.baseURL + "v3/scan", 88 | body: body, 89 | headers: c.defaultHeaders(), 90 | } 91 | 92 | scanResponse := &ScanTextResponse{} 93 | err = c.do(ctx, reqParams, scanResponse) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | return scanResponse, nil 99 | } 100 | -------------------------------------------------------------------------------- /model.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | const ( 4 | DetectorTypeNightfallDetector DetectorType = "NIGHTFALL_DETECTOR" 5 | DetectorTypeRegex DetectorType = "REGEX" 6 | DetectorTypeWordList DetectorType = "WORD_LIST" 7 | 8 | MatchTypeFull MatchType = "FULL" 9 | MatchTypePartial MatchType = "PARTIAL" 10 | 11 | ExclusionRuleTypeRegex ExclusionType = "REGEX" 12 | ExclusionRuleTypeWordlist ExclusionType = "WORD_LIST" 13 | 14 | // A modifier that is used to decide when a finding should be surfaced in the context of a detection rule. 15 | // When ALL is specified, all detectors in a detection rule must trigger a match in order for the 16 | // finding to be reported. This is the equivalent of a logical "AND" operator. 17 | // When ANY is specified, only one of the detectors in a detection rule must trigger a match in order 18 | // for the finding to be reported. This is the equivalent of a logical "OR" operator. 19 | LogicalOpAny LogicalOp = "ANY" 20 | LogicalOpAll LogicalOp = "ALL" 21 | 22 | // Confidence describes the certainty that a piece of content matches a detector. 23 | ConfidenceVeryUnlikely Confidence = "VERY_UNLIKELY" 24 | ConfidenceUnlikely Confidence = "UNLIKELY" 25 | ConfidencePossible Confidence = "POSSIBLE" 26 | ConfidenceLikely Confidence = "LIKELY" 27 | ConfidenceVeryLikely Confidence = "VERY_LIKELY" 28 | ) 29 | 30 | type ( 31 | DetectorType string 32 | MatchType string 33 | ExclusionType string 34 | LogicalOp string 35 | Confidence string 36 | ) 37 | 38 | // DetectionRule is an object that contains a set of detectors to be used when scanning content. Findings matches are 39 | // triggered according to the provided logicalOp; valid values are ANY(logical 40 | // OR, i.e. a finding is emitted only if any of the provided detectors match), or ALL 41 | // (logical AND, i.e. a finding is emitted only if all provided detectors match). 42 | type DetectionRule struct { 43 | Name string `json:"name"` 44 | Detectors []Detector `json:"detectors"` 45 | LogicalOp LogicalOp `json:"logicalOp"` 46 | } 47 | 48 | // A Detector represents a data type or category of information. Detectors are used to scan content 49 | // for findings. 50 | type Detector struct { 51 | DetectorUUID string `json:"detectorUUID,omitempty"` 52 | MinNumFindings int `json:"minNumFindings"` 53 | MinConfidence Confidence `json:"minConfidence"` 54 | DisplayName string `json:"displayName"` 55 | DetectorType DetectorType `json:"detectorType"` 56 | NightfallDetector string `json:"nightfallDetector"` 57 | Regex *Regex `json:"regex,omitempty"` 58 | WordList *WordList `json:"wordList,omitempty"` 59 | ContextRules []ContextRule `json:"contextRules"` 60 | ExclusionRules []ExclusionRule `json:"exclusionRules"` 61 | RedactionConfig *RedactionConfig `json:"redactionConfig,omitempty"` 62 | } 63 | 64 | // An ExclusionRule describes a regular expression or list of keywords that may be used to disqualify a 65 | // candidate finding from triggering a detector match. 66 | type ExclusionRule struct { 67 | MatchType MatchType `json:"matchType"` 68 | ExclusionType ExclusionType `json:"exclusionType"` 69 | Regex *Regex `json:"regex"` 70 | WordList *WordList `json:"wordList"` 71 | } 72 | 73 | // A ContextRule describes how a regular expression may be used to adjust the confidence of a candidate finding. 74 | // This context rule will be applied within the provided byte proximity, and if the regular expression matches, then 75 | // the confidence associated with the finding will be adjusted to the value prescribed. 76 | type ContextRule struct { 77 | Regex Regex `json:"regex"` 78 | Proximity Proximity `json:"proximity"` 79 | ConfidenceAdjustment ConfidenceAdjustment `json:"confidenceAdjustment"` 80 | } 81 | 82 | // A Regex represents a regular expression to customize the behavior of a detector while Nightfall performs a scan. 83 | type Regex struct { 84 | Pattern string `json:"pattern"` 85 | IsCaseSensitive bool `json:"isCaseSensitive"` 86 | } 87 | 88 | // A WordList is a list of words that can be used to customize the behavior of a detector while Nightfall performs a scan. 89 | type WordList struct { 90 | Values []string `json:"values"` 91 | IsCaseSensitive bool `json:"isCaseSensitive"` 92 | } 93 | 94 | // Proximity represents a range of bytes to consider around a candidate finding. 95 | type Proximity struct { 96 | WindowBefore int `json:"windowBefore"` 97 | WindowAfter int `json:"windowAfter"` 98 | } 99 | 100 | // ConfidenceAdjustment describes how to adjust confidence on a given finding. Valid values for the adjustment are 101 | // VERY_UNLIKELY, UNLIKELY, POSSIBLE, LIKELY, and VERY_LIKELY. 102 | type ConfidenceAdjustment struct { 103 | FixedConfidence Confidence `json:"fixedConfidence"` 104 | } 105 | 106 | // DetectorMetadata contains the minimal information representing a detector. A detector may be uniquely 107 | // identified by its UUID; the name field helps provide human-readability. 108 | type DetectorMetadata struct { 109 | DisplayName string `json:"name"` 110 | DetectorUUID string `json:"uuid"` 111 | } 112 | -------------------------------------------------------------------------------- /file_test.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net/http" 7 | "net/http/httptest" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | "github.com/google/uuid" 13 | ) 14 | 15 | func TestScanFile(t *testing.T) { 16 | uuidStr := "430d42aa-1e1f-405d-8799-7f5f26486a0d" 17 | reqUUID := uuid.MustParse(uuidStr) 18 | var chunkCount int 19 | defaultInitUploadHandler := func(w http.ResponseWriter, r *http.Request) { 20 | resp := fileUploadResponse{ 21 | ID: reqUUID, 22 | FileSizeBytes: 15, 23 | ChunkSize: 5, 24 | } 25 | b, _ := json.Marshal(resp) 26 | _, _ = w.Write(b) 27 | } 28 | defaultUploadHandler := func(w http.ResponseWriter, r *http.Request) { 29 | chunkCount++ 30 | w.WriteHeader(http.StatusOK) 31 | } 32 | defaultFinishHandler := func(w http.ResponseWriter, r *http.Request) { 33 | w.WriteHeader(http.StatusOK) 34 | } 35 | defaultScanHandler := func(w http.ResponseWriter, r *http.Request) { 36 | resp := ScanFileResponse{ 37 | ID: uuidStr, 38 | Message: "scan initiated", 39 | } 40 | b, _ := json.Marshal(resp) 41 | _, _ = w.Write(b) 42 | } 43 | 44 | tests := []struct { 45 | name string 46 | handlers map[string]http.HandlerFunc 47 | clientTimeOut time.Duration 48 | expChunks int 49 | wantErr bool 50 | }{ 51 | { 52 | name: "happy path - 1 chunk", 53 | handlers: map[string]http.HandlerFunc{ 54 | "/v3/upload": func(w http.ResponseWriter, r *http.Request) { 55 | resp := fileUploadResponse{ 56 | ID: reqUUID, 57 | FileSizeBytes: 15, 58 | ChunkSize: 15, 59 | } 60 | b, _ := json.Marshal(resp) 61 | _, _ = w.Write(b) 62 | }, 63 | "/v3/upload/" + uuidStr: defaultUploadHandler, 64 | "/v3/upload/" + uuidStr + "/finish": defaultFinishHandler, 65 | "/v3/upload/" + uuidStr + "/scan": defaultScanHandler, 66 | }, 67 | expChunks: 1, 68 | wantErr: false, 69 | }, 70 | { 71 | name: "happy path - 3 chunks", 72 | handlers: map[string]http.HandlerFunc{ 73 | "/v3/upload": defaultInitUploadHandler, 74 | "/v3/upload/" + uuidStr: defaultUploadHandler, 75 | "/v3/upload/" + uuidStr + "/finish": defaultFinishHandler, 76 | "/v3/upload/" + uuidStr + "/scan": defaultScanHandler, 77 | }, 78 | expChunks: 3, 79 | wantErr: false, 80 | }, 81 | { 82 | name: "upload timed out", 83 | handlers: map[string]http.HandlerFunc{ 84 | "/v3/upload": defaultInitUploadHandler, 85 | "/v3/upload/" + uuidStr: func(w http.ResponseWriter, r *http.Request) { 86 | if chunkCount == 1 { 87 | time.Sleep(2 * time.Second) 88 | w.WriteHeader(http.StatusOK) 89 | return 90 | } 91 | chunkCount++ 92 | w.WriteHeader(http.StatusOK) 93 | }, 94 | "/v3/upload/" + uuidStr + "/finish": defaultFinishHandler, 95 | "/v3/upload/" + uuidStr + "/scan": defaultScanHandler, 96 | }, 97 | clientTimeOut: 1 * time.Second, 98 | expChunks: 1, 99 | wantErr: true, 100 | }, 101 | { 102 | name: "upload init failed", 103 | handlers: map[string]http.HandlerFunc{ 104 | "/v3/upload": func(w http.ResponseWriter, r *http.Request) { 105 | w.WriteHeader(http.StatusInternalServerError) 106 | }, 107 | }, 108 | wantErr: true, 109 | }, 110 | { 111 | name: "upload failed", 112 | handlers: map[string]http.HandlerFunc{ 113 | "/v3/upload": defaultInitUploadHandler, 114 | "/v3/upload/" + uuid.UUID{}.String(): func(w http.ResponseWriter, r *http.Request) { 115 | w.WriteHeader(http.StatusInternalServerError) 116 | }, 117 | }, 118 | wantErr: true, 119 | }, 120 | { 121 | name: "upload finish failed", 122 | handlers: map[string]http.HandlerFunc{ 123 | "/v3/upload": defaultInitUploadHandler, 124 | "/v3/upload/" + uuidStr: defaultUploadHandler, 125 | "/v3/upload/" + uuidStr + "/finish": func(w http.ResponseWriter, r *http.Request) { 126 | w.WriteHeader(http.StatusInternalServerError) 127 | }, 128 | }, 129 | expChunks: 3, 130 | wantErr: true, 131 | }, 132 | { 133 | name: "upload init failed", 134 | handlers: map[string]http.HandlerFunc{ 135 | "/v3/upload": defaultInitUploadHandler, 136 | "/v3/upload/" + uuidStr: defaultUploadHandler, 137 | "/v3/upload/" + uuidStr + "/finish": defaultFinishHandler, 138 | "/v3/upload/" + uuidStr + "/scan": func(w http.ResponseWriter, r *http.Request) { 139 | w.WriteHeader(http.StatusInternalServerError) 140 | }, 141 | }, 142 | expChunks: 3, 143 | wantErr: true, 144 | }, 145 | } 146 | 147 | for _, test := range tests { 148 | chunkCount = 0 149 | 150 | t.Run(test.name, func(t *testing.T) { 151 | mux := http.NewServeMux() 152 | for pattern, handler := range test.handlers { 153 | mux.HandleFunc(pattern, handler) 154 | } 155 | s := httptest.NewServer(mux) 156 | defer s.Close() 157 | 158 | client, err := NewClient(OptionAPIKey("some key"), OptionFileUploadConcurrency(2)) 159 | if err != nil { 160 | t.Fatal("Error initializing client") 161 | } 162 | client.baseURL = s.URL + "/" 163 | 164 | _, err = client.ScanFile(context.Background(), &ScanFileRequest{ 165 | Content: strings.NewReader("4242 4242 4242 4242"), 166 | ContentSizeBytes: 15, 167 | Timeout: test.clientTimeOut, 168 | }) 169 | if !test.wantErr && err != nil { 170 | t.Errorf("Got unexpected error: %v", err) 171 | } 172 | if test.wantErr && err == nil { 173 | t.Error("Did not get expected error") 174 | } 175 | if chunkCount != test.expChunks { 176 | t.Error("Did not upload expected number of chunks") 177 | } 178 | }) 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | "sync" 8 | "time" 9 | 10 | "github.com/google/uuid" 11 | ) 12 | 13 | // ScanPolicy contains configuration that describes how to scan a file. Since the file is scanned asynchronously, 14 | // the results from the scan are delivered to the provided webhook URL. The scan configuration may contain both 15 | // inline detection rule definitions and UUID's referring to existing detection rules (up to 10 of each). 16 | type ScanPolicy struct { 17 | WebhookURL string `json:"webhookURL"` // Deprecated: use AlertConfig instead 18 | DetectionRules []DetectionRule `json:"detectionRules"` 19 | DetectionRuleUUIDs []string `json:"detectionRuleUUIDs"` 20 | AlertConfig *AlertConfig `json:"alertConfig"` 21 | } 22 | 23 | // ScanFileRequest represents a request to scan a file that was uploaded via the Nightfall API. Exactly one of 24 | // PolicyUUID or Policy should be provided. 25 | type ScanFileRequest struct { 26 | PolicyUUID *string `json:"policyUUID"` 27 | Policy *ScanPolicy `json:"policy"` 28 | RequestMetadata string `json:"requestMetadata"` 29 | Content io.Reader `json:"-"` 30 | ContentSizeBytes int64 `json:"-"` 31 | Timeout time.Duration `json:"-"` 32 | } 33 | 34 | // ScanFileResponse is the object returned by the Nightfall API when an (asynchronous) file scan request 35 | // was successfully triggered. 36 | type ScanFileResponse struct { 37 | ID string `json:"id"` 38 | Message string `json:"message"` 39 | } 40 | 41 | type fileUploadResponse struct { 42 | ID uuid.UUID `json:"id"` 43 | FileSizeBytes int64 `json:"fileSizeBytes"` 44 | ChunkSize int64 `json:"chunkSize"` 45 | MIMEType string `json:"mimeType"` 46 | } 47 | 48 | type fileUploadRequest struct { 49 | FileSizeBytes int64 `json:"fileSizeBytes"` 50 | } 51 | 52 | // ScanFile is a convenience method that abstracts the details of the multi-step file upload and scan process. 53 | // Calling this method for a given file is equivalent to (1) manually initializing a file upload session, 54 | // (2) uploading all chunks of the file, (3) completing the upload, and (4) triggering a scan of the file. 55 | // 56 | // The maximum allowed ContentSizeBytes is dependent on the terms of your current 57 | // Nightfall usage plan agreement; check the Nightfall dashboard for more details. 58 | // 59 | // This method consumes the provided reader, but it does not close it; closing remains 60 | // the caller's responsibility. 61 | func (c *Client) ScanFile(ctx context.Context, request *ScanFileRequest) (*ScanFileResponse, error) { 62 | var cancel context.CancelFunc 63 | if request.Timeout > 0 { 64 | ctx, cancel = context.WithTimeout(ctx, request.Timeout) 65 | } else { 66 | ctx, cancel = context.WithCancel(ctx) 67 | } 68 | defer cancel() 69 | 70 | fileUpload, err := c.initFileUpload(ctx, &fileUploadRequest{FileSizeBytes: request.ContentSizeBytes}) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | err = c.doChunkedUpload(ctx, fileUpload, request.Content) 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | err = c.completeFileUpload(ctx, fileUpload.ID) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | return c.scanUploadedFile(ctx, request, fileUpload.ID) 86 | } 87 | 88 | func (c *Client) initFileUpload(ctx context.Context, request *fileUploadRequest) (*fileUploadResponse, error) { 89 | body, err := encodeBodyAsJSON(request) 90 | if err != nil { 91 | return nil, err 92 | } 93 | reqParams := requestParams{ 94 | method: http.MethodPost, 95 | url: c.baseURL + "v3/upload", 96 | body: body, 97 | headers: c.defaultHeaders(), 98 | } 99 | 100 | uploadResponse := &fileUploadResponse{} 101 | err = c.do(ctx, reqParams, uploadResponse) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | return uploadResponse, nil 107 | } 108 | 109 | func (c *Client) doChunkedUpload(ctx context.Context, fileUpload *fileUploadResponse, content io.Reader) error { 110 | errChan := make(chan error, 1) 111 | wg := &sync.WaitGroup{} 112 | concurrencyChan := make(chan struct{}, c.fileUploadConcurrency) 113 | 114 | uploadCtx, cancel := context.WithCancel(ctx) 115 | defer cancel() 116 | 117 | upload: 118 | for offset := int64(0); offset < fileUpload.FileSizeBytes; offset += fileUpload.ChunkSize { 119 | // Check if we are at max upload concurrency limit and block if we are 120 | concurrencyChan <- struct{}{} 121 | 122 | // Check if there were any errors from uploading previous chunks, and break if there were 123 | select { 124 | case <-uploadCtx.Done(): 125 | break upload 126 | default: 127 | } 128 | 129 | buf := make([]byte, fileUpload.ChunkSize) 130 | bytesRead, err := content.Read(buf) 131 | if err == io.EOF { 132 | break 133 | } else if err != nil { 134 | return err 135 | } 136 | if int64(bytesRead) < fileUpload.ChunkSize { 137 | buf = buf[:bytesRead] 138 | } 139 | 140 | wg.Add(1) 141 | go func(o int64, data []byte) { 142 | defer func() { 143 | wg.Done() 144 | <-concurrencyChan 145 | }() 146 | 147 | reqParams := requestParams{ 148 | method: http.MethodPatch, 149 | url: c.baseURL + "v3/upload/" + fileUpload.ID.String(), 150 | body: data, 151 | headers: c.chunkedUploadHeaders(o), 152 | } 153 | err = c.do(uploadCtx, reqParams, nil) 154 | if err != nil { 155 | // If error channel is full already just discard this error, first error is most likely the most useful one anyways 156 | select { 157 | case errChan <- err: 158 | default: 159 | } 160 | cancel() 161 | return 162 | } 163 | }(offset, buf) 164 | } 165 | 166 | wg.Wait() 167 | close(errChan) 168 | 169 | if err := <-errChan; err != nil { 170 | return err 171 | } 172 | 173 | return nil 174 | } 175 | 176 | func (c *Client) completeFileUpload(ctx context.Context, fileUUID uuid.UUID) error { 177 | reqParams := requestParams{ 178 | method: http.MethodPost, 179 | url: c.baseURL + "v3/upload/" + fileUUID.String() + "/finish", 180 | body: nil, 181 | headers: c.defaultHeaders(), 182 | } 183 | return c.do(ctx, reqParams, nil) 184 | } 185 | 186 | func (c *Client) scanUploadedFile(ctx context.Context, request *ScanFileRequest, fileUUID uuid.UUID) (*ScanFileResponse, error) { 187 | body, err := encodeBodyAsJSON(request) 188 | if err != nil { 189 | return nil, err 190 | } 191 | reqParams := requestParams{ 192 | method: http.MethodPost, 193 | url: c.baseURL + "v3/upload/" + fileUUID.String() + "/scan", 194 | body: body, 195 | headers: c.defaultHeaders(), 196 | } 197 | 198 | scanResponse := &ScanFileResponse{} 199 | err = c.do(ctx, reqParams, scanResponse) 200 | if err != nil { 201 | return nil, err 202 | } 203 | 204 | return scanResponse, nil 205 | } 206 | -------------------------------------------------------------------------------- /nightfall.go: -------------------------------------------------------------------------------- 1 | package nightfall 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "net/http" 12 | "os" 13 | "runtime/debug" 14 | "strconv" 15 | "time" 16 | ) 17 | 18 | const ( 19 | APIURL = "https://api.nightfall.ai/" 20 | 21 | DefaultFileUploadConcurrency = 1 22 | DefaultRetryCount = 5 23 | ) 24 | 25 | // Client manages communication with the Nightfall API 26 | type Client struct { 27 | baseURL string 28 | apiKey string 29 | httpClient *http.Client 30 | fileUploadConcurrency int 31 | retryCount int 32 | } 33 | 34 | // ClientOption defines an option for a Client 35 | type ClientOption func(*Client) error 36 | 37 | var ( 38 | errMissingAPIKey = errors.New("missing api key") 39 | errInvalidFileUploadConcurrency = errors.New("fileUploadConcurrency must be in range [1,100]") 40 | errRetryable429 = errors.New("429 retryable error") 41 | 42 | userAgent = loadUserAgent() 43 | ) 44 | 45 | // NewClient configures, validates, then creates an instance of a Nightfall Client. 46 | func NewClient(options ...ClientOption) (*Client, error) { 47 | c := &Client{ 48 | baseURL: APIURL, 49 | apiKey: os.Getenv("NIGHTFALL_API_KEY"), 50 | httpClient: &http.Client{}, 51 | fileUploadConcurrency: DefaultFileUploadConcurrency, 52 | retryCount: DefaultRetryCount, 53 | } 54 | 55 | for _, opt := range options { 56 | err := opt(c) 57 | if err != nil { 58 | return nil, err 59 | } 60 | } 61 | 62 | if c.apiKey == "" { 63 | return nil, errMissingAPIKey 64 | } 65 | 66 | return c, nil 67 | } 68 | 69 | // OptionAPIKey sets the api key used in the Nightfall client 70 | func OptionAPIKey(apiKey string) func(*Client) error { 71 | return func(c *Client) error { 72 | c.apiKey = apiKey 73 | return nil 74 | } 75 | } 76 | 77 | // OptionHTTPClient sets the http client used in the Nightfall client 78 | func OptionHTTPClient(client *http.Client) func(*Client) error { 79 | return func(c *Client) error { 80 | c.httpClient = client 81 | return nil 82 | } 83 | } 84 | 85 | // OptionFileUploadConcurrency sets the number of goroutines that will upload chunks of data when scanning files with the Nightfall client 86 | func OptionFileUploadConcurrency(fileUploadConcurrency int) func(*Client) error { 87 | return func(c *Client) error { 88 | if fileUploadConcurrency > 100 || fileUploadConcurrency <= 0 { 89 | return errInvalidFileUploadConcurrency 90 | } 91 | c.fileUploadConcurrency = fileUploadConcurrency 92 | return nil 93 | } 94 | } 95 | 96 | func loadUserAgent() string { 97 | prefix := "nightfall-go-sdk" 98 | 99 | buildInfo, ok := debug.ReadBuildInfo() 100 | if !ok { 101 | return prefix 102 | } 103 | for _, dep := range buildInfo.Deps { 104 | if dep.Path == "github.com/nightfallai/nightfall-go-sdk" { 105 | return fmt.Sprintf("%s/%s", prefix, dep.Version) 106 | } 107 | } 108 | 109 | return prefix 110 | } 111 | 112 | type requestParams struct { 113 | method string 114 | url string 115 | body []byte 116 | headers map[string]string 117 | } 118 | 119 | func (c *Client) defaultHeaders() map[string]string { 120 | headers := map[string]string{ 121 | "Content-Type": "application/json", 122 | "Authorization": "Bearer " + c.apiKey, 123 | "User-Agent": userAgent, 124 | } 125 | return headers 126 | } 127 | 128 | func (c *Client) chunkedUploadHeaders(o int64) map[string]string { 129 | headers := map[string]string{ 130 | "X-Upload-Offset": strconv.FormatInt(o, 10), 131 | "Content-Type": "application/octet-stream", 132 | "Authorization": "Bearer " + c.apiKey, 133 | "User-Agent": userAgent, 134 | } 135 | return headers 136 | } 137 | 138 | func encodeBodyAsJSON(body interface{}) ([]byte, error) { 139 | var buf io.ReadWriter 140 | if body != nil { 141 | buf = &bytes.Buffer{} 142 | enc := json.NewEncoder(buf) 143 | // Marshal() does not encode some special characters like "&" properly so we need to do this 144 | enc.SetEscapeHTML(false) 145 | err := enc.Encode(body) 146 | if err != nil { 147 | return nil, err 148 | } 149 | } 150 | b, err := io.ReadAll(buf) 151 | if err != nil { 152 | return nil, err 153 | } 154 | return b, nil 155 | } 156 | 157 | func (c *Client) do(ctx context.Context, reqParams requestParams, retResp interface{}) error { 158 | for attempt := 1; attempt <= c.retryCount+1; attempt++ { 159 | req, err := http.NewRequestWithContext(ctx, reqParams.method, reqParams.url, bytes.NewReader(reqParams.body)) 160 | if err != nil { 161 | return err 162 | } 163 | for k, v := range reqParams.headers { 164 | req.Header.Set(k, v) 165 | } 166 | err = func() error { 167 | resp, err := c.httpClient.Do(req) 168 | if err != nil { 169 | select { 170 | case <-ctx.Done(): 171 | return ctx.Err() 172 | default: 173 | return err 174 | } 175 | } 176 | defer resp.Body.Close() 177 | 178 | err = checkResponse(resp) 179 | if err != nil { 180 | if resp.StatusCode == http.StatusTooManyRequests { 181 | if attempt >= c.retryCount+1 { 182 | // We've hit the retry count limit, so just return the error 183 | return err 184 | } 185 | return errRetryable429 186 | } 187 | return err 188 | } 189 | 190 | // Request was successful so read response if any then return 191 | if retResp != nil { 192 | err = json.NewDecoder(resp.Body).Decode(retResp) 193 | if errors.Is(err, io.EOF) { 194 | err = nil 195 | } 196 | } 197 | 198 | return err 199 | }() 200 | if err == nil { 201 | break 202 | } else if errors.Is(err, errRetryable429) { 203 | // Sleep for 1s then retry on 429's 204 | time.Sleep(time.Second) 205 | continue 206 | } else { 207 | return err 208 | } 209 | } 210 | 211 | return nil 212 | } 213 | 214 | // Error is the struct returned by Nightfall API requests that are unsuccessful. This struct is generally returned 215 | // when the HTTP status code is outside the range 200-299. 216 | type Error struct { 217 | Code int `json:"code"` 218 | Message string `json:"message"` 219 | Description string `json:"description"` 220 | AdditionalData map[string]string `json:"additionalData"` 221 | } 222 | 223 | func (e *Error) Error() string { 224 | return e.Message 225 | } 226 | 227 | func checkResponse(r *http.Response) error { 228 | if 200 <= r.StatusCode && r.StatusCode <= 299 { 229 | return nil 230 | } 231 | 232 | e := &Error{} 233 | b, err := ioutil.ReadAll(r.Body) 234 | if err != nil || len(b) == 0 { 235 | e.Code = r.StatusCode 236 | return e 237 | } 238 | 239 | err = json.Unmarshal(b, e) 240 | if err != nil { 241 | e.Code = r.StatusCode 242 | return e 243 | } 244 | 245 | return e 246 | } 247 | --------------------------------------------------------------------------------