├── .github ├── PULL_REQUEST_TEMPLATE.md ├── CODEOWNERS └── workflows │ ├── ci-pr.yaml │ ├── release.yaml │ └── ci.yaml ├── .gitignore ├── .goreleaser.yml ├── README.md ├── pkg ├── utils │ ├── topics_stats_stream.go │ ├── topic_auto_creation_config.go │ ├── update_options.go │ ├── function_state.go │ ├── broker_ns_isolation_data.go │ ├── metrics.go │ ├── internal_configuration_data.go │ ├── worker_info.go │ ├── publish_rate.go │ ├── resources.go │ ├── batch_source_config.go │ ├── ns_ownership_status.go │ ├── retention_policies.go │ ├── utils.go │ ├── package_metadata.go │ ├── subscription_auth_mode.go │ ├── auth_polices.go │ ├── crypto_config.go │ ├── long_running_process_status.go │ ├── producer_config.go │ ├── topic_domain.go │ ├── connector_definition.go │ ├── topic_type.go │ ├── bundles_data.go │ ├── resource_quota.go │ ├── package_type.go │ ├── consumer_config.go │ ├── dispatch_rate.go │ ├── utils_test.go │ ├── auth_action.go │ ├── persistence_policies.go │ ├── window_confing.go │ ├── source_status.go │ ├── schema_strategy.go │ ├── message_id_test.go │ ├── function_status.go │ ├── auth_polices_test.go │ ├── namespace_name_test.go │ ├── message_id.go │ ├── inactive_topic_policies.go │ ├── schema_util.go │ ├── sink_status.go │ ├── backlog_quota.go │ ├── source_config.go │ ├── namespace_name.go │ ├── topic_name_test.go │ ├── package_name_test.go │ ├── ns_isolation_data.go │ ├── home_dir.go │ ├── allocator_stats.go │ ├── message.go │ ├── package_name.go │ ├── sink_config.go │ ├── policies.go │ ├── topic_name.go │ ├── functions_stats.go │ ├── function_confg.go │ └── load_manage_report.go ├── rest │ ├── errors.go │ └── client_test.go └── admin │ ├── config │ ├── api_version_test.go │ ├── api_version.go │ └── config.go │ ├── auth │ ├── transport.go │ ├── token.go │ ├── tls.go │ ├── provider.go │ ├── oauth2_test.go │ └── oauth2.go │ ├── tenant.go │ ├── resource_quotas.go │ ├── functions_worker.go │ ├── broker_stats.go │ ├── admin.go │ ├── admin_test.go │ ├── schema.go │ ├── ns_isolation_policy.go │ ├── cluster.go │ ├── brokers.go │ └── packages.go ├── .licenserc.yaml ├── .golangci.yaml ├── alias.go ├── go.mod ├── Makefile └── CONTRIBUTING.md /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 10 | 11 | 12 | 13 | 14 | Fixes #TODO 15 | 16 | ## Motivation 17 | 18 | 19 | 20 | ## Modifications 21 | 22 | 23 | 24 | ## Verification 25 | 26 | 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2023 StreamNative, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, 10 | # software distributed under the License is distributed on an 11 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | # KIND, either express or implied. See the License for the 13 | # specific language governing permissions and limitations 14 | # under the License. 15 | # 16 | 17 | .DS_Store 18 | .idea/ 19 | .vscode/ 20 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 StreamNative, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, 10 | # software distributed under the License is distributed on an 11 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | # KIND, either express or implied. See the License for the 13 | # specific language governing permissions and limitations 14 | # under the License. 15 | # 16 | 17 | builds: 18 | - skip: true 19 | release: 20 | prerelease: auto 21 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Copyright 2023 StreamNative, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, 10 | # software distributed under the License is distributed on an 11 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | # KIND, either express or implied. See the License for the 13 | # specific language governing permissions and limitations 14 | # under the License. 15 | # 16 | 17 | # Owning team's slack is #f_sn_data_plane 18 | * @streamnative/cloud 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 15 | 16 | This project has been donated to the ASF and the repository has been moved to https://github.com/apache/pulsar-client-go/tree/master/pulsaradmin. 17 | -------------------------------------------------------------------------------- /pkg/utils/topics_stats_stream.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | // var TopicsMap map[string]map[string]map[string]TopicStats 19 | 20 | type TopicStatsStream struct { 21 | TopicsMap map[string]map[string]map[string]TopicStats `json:"topicStatsBuf"` 22 | } 23 | -------------------------------------------------------------------------------- /pkg/utils/topic_auto_creation_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type TopicAutoCreationConfig struct { 19 | Allow bool `json:"allowAutoTopicCreation"` 20 | Type TopicType `json:"topicType"` 21 | Partitions int `json:"defaultNumPartitions"` 22 | } 23 | -------------------------------------------------------------------------------- /pkg/utils/update_options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | // Options while updating the sink 19 | type UpdateOptions struct { 20 | UpdateAuthData bool 21 | } 22 | 23 | func NewUpdateOptions() *UpdateOptions { 24 | return &UpdateOptions{ 25 | UpdateAuthData: false, 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /pkg/utils/function_state.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type FunctionState struct { 19 | Key string `json:"key"` 20 | StringValue string `json:"stringValue"` 21 | ByteValue []byte `json:"byteValue"` 22 | NumValue int64 `json:"numberValue"` 23 | Version int64 `json:"version"` 24 | } 25 | -------------------------------------------------------------------------------- /pkg/utils/broker_ns_isolation_data.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type BrokerNamespaceIsolationData struct { 19 | BrokerName string `json:"brokerName"` 20 | PolicyName string `json:"policyName"` 21 | IsPrimary bool `json:"isPrimary"` 22 | NamespaceRegex []string `json:"namespaceRegex"` 23 | } 24 | -------------------------------------------------------------------------------- /.licenserc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 StreamNative, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | header: 16 | license: 17 | spdx-id: Apache-2.0 18 | copyright-owner: StreamNative, Inc. 19 | copyright-year: '2023' 20 | software-name: pulsar-admin-go 21 | 22 | paths-ignore: 23 | - '**/go.mod' 24 | - '**/go.sum' 25 | - 'LICENSE' 26 | - 'NOTICE' 27 | - PULL_REQUEST_TEMPLATE.md 28 | 29 | comment: on-failure 30 | -------------------------------------------------------------------------------- /pkg/utils/metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type Metrics struct { 19 | Metrics map[string]interface{} `json:"metrics"` 20 | Dimensions map[string]string `json:"dimensions"` 21 | } 22 | 23 | func NewMetrics(dimensionMap map[string]string) *Metrics { 24 | return &Metrics{ 25 | Metrics: make(map[string]interface{}), 26 | Dimensions: dimensionMap, 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /pkg/utils/internal_configuration_data.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type InternalConfigurationData struct { 19 | ZookeeperServers string `json:"zookeeperServers"` 20 | ConfigurationStoreServers string `json:"configurationStoreServers"` 21 | LedgersRootPath string `json:"ledgersRootPath"` 22 | StateStorageServiceURL string `json:"stateStorageServiceUrl"` 23 | } 24 | -------------------------------------------------------------------------------- /pkg/utils/worker_info.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type WorkerInfo struct { 19 | WorkerID string `json:"workerId"` 20 | WorkerHostname string `json:"workerHostname"` 21 | Port int `json:"port"` 22 | } 23 | 24 | type WorkerFunctionInstanceStats struct { 25 | Name string `json:"name"` 26 | Metrics FunctionInstanceStatsData `json:"metrics"` 27 | } 28 | -------------------------------------------------------------------------------- /pkg/rest/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package rest 17 | 18 | import "fmt" 19 | 20 | // Error is a admin error type 21 | type Error struct { 22 | Reason string `json:"reason"` 23 | Code int 24 | } 25 | 26 | func (e Error) Error() string { 27 | return fmt.Sprintf("code: %d reason: %s", e.Code, e.Reason) 28 | } 29 | 30 | func IsAdminError(err error) bool { 31 | _, ok := err.(Error) 32 | return ok 33 | } 34 | -------------------------------------------------------------------------------- /pkg/utils/publish_rate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type PublishRate struct { 19 | PublishThrottlingRateInMsg int `json:"publishThrottlingRateInMsg"` 20 | PublishThrottlingRateInByte int64 `json:"publishThrottlingRateInByte"` 21 | } 22 | 23 | func NewPublishRate() *PublishRate { 24 | return &PublishRate{ 25 | PublishThrottlingRateInMsg: -1, 26 | PublishThrottlingRateInByte: -1, 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /.github/workflows/ci-pr.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 StreamNative, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, 10 | # software distributed under the License is distributed on an 11 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | # KIND, either express or implied. See the License for the 13 | # specific language governing permissions and limitations 14 | # under the License. 15 | # 16 | 17 | name: CI / PR 18 | 19 | on: 20 | pull_request: 21 | branches: 22 | - '*' 23 | types: 24 | - opened 25 | - reopened 26 | - edited 27 | - synchronize 28 | 29 | jobs: 30 | title-check: 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: amannn/action-semantic-pull-request@v5 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | -------------------------------------------------------------------------------- /pkg/admin/config/api_version_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package config 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestApiVersion_String(t *testing.T) { 25 | assert.Equal(t, "", V1.String()) 26 | assert.Equal(t, "v2", V2.String()) 27 | assert.Equal(t, "v3", V3.String()) 28 | var undefinedAPIVersion APIVersion 29 | assert.Equal(t, DefaultAPIVersion, undefinedAPIVersion.String()) 30 | } 31 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2023 StreamNative, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, 11 | # software distributed under the License is distributed on an 12 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 13 | # KIND, either express or implied. See the License for the 14 | # specific language governing permissions and limitations 15 | # under the License. 16 | # 17 | 18 | run: 19 | deadline: 6m 20 | 21 | linters: 22 | # disable all for explicit enable 23 | disable-all: true 24 | enable: 25 | - errcheck 26 | - gosimple 27 | - govet 28 | - ineffassign 29 | - staticcheck 30 | - typecheck 31 | - unused 32 | - lll 33 | - goimports 34 | - bodyclose 35 | - misspell 36 | - prealloc 37 | - revive 38 | - stylecheck 39 | - unconvert 40 | - unparam 41 | -------------------------------------------------------------------------------- /pkg/utils/resources.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type Resources struct { 19 | CPU float64 `json:"cpu"` 20 | Disk int64 `json:"disk"` 21 | RAM int64 `json:"ram"` 22 | } 23 | 24 | func NewDefaultResources() *Resources { 25 | resources := &Resources{ 26 | // Default cpu is 1 core 27 | CPU: 1, 28 | // Default memory is 1GB 29 | RAM: 1073741824, 30 | // Default disk is 10GB 31 | Disk: 10737418240, 32 | } 33 | 34 | return resources 35 | } 36 | -------------------------------------------------------------------------------- /pkg/admin/config/api_version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package config 17 | 18 | type APIVersion int 19 | 20 | const ( 21 | undefined APIVersion = iota 22 | V1 23 | V2 24 | V3 25 | ) 26 | 27 | const DefaultAPIVersion = "v2" 28 | 29 | func (v APIVersion) String() string { 30 | switch v { 31 | case undefined: 32 | return DefaultAPIVersion 33 | case V1: 34 | return "" 35 | case V2: 36 | return "v2" 37 | case V3: 38 | return "v3" 39 | } 40 | 41 | return DefaultAPIVersion 42 | } 43 | -------------------------------------------------------------------------------- /pkg/utils/batch_source_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | const ( 19 | BatchsourceConfigKey string = "__BATCHSOURCECONFIGS__" 20 | BatchsourceClassnameKey string = "__BATCHSOURCECLASSNAME__" 21 | ) 22 | 23 | type BatchSourceConfig struct { 24 | DiscoveryTriggererClassName string `json:"discoveryTriggererClassName" yaml:"discoveryTriggererClassName"` 25 | 26 | DiscoveryTriggererConfig map[string]interface{} `json:"discoveryTriggererConfig" yaml:"discoveryTriggererConfig"` 27 | } 28 | -------------------------------------------------------------------------------- /pkg/utils/ns_ownership_status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type NamespaceOwnershipStatus struct { 19 | BrokerAssignment BrokerAssignment `json:"broker_assignment"` 20 | IsControlled bool `json:"is_controlled"` 21 | IsActive bool `json:"is_active"` 22 | } 23 | 24 | type BrokerAssignment string 25 | 26 | const ( 27 | Primary BrokerAssignment = "primary" 28 | Secondary BrokerAssignment = "secondary" 29 | Shared BrokerAssignment = "shared" 30 | ) 31 | -------------------------------------------------------------------------------- /pkg/utils/retention_policies.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type RetentionPolicies struct { 19 | RetentionTimeInMinutes int `json:"retentionTimeInMinutes"` 20 | RetentionSizeInMB int64 `json:"retentionSizeInMB"` 21 | } 22 | 23 | func NewRetentionPolicies(retentionTimeInMinutes int, retentionSizeInMB int) RetentionPolicies { 24 | return RetentionPolicies{ 25 | RetentionTimeInMinutes: retentionTimeInMinutes, 26 | RetentionSizeInMB: int64(retentionSizeInMB), 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /alias.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package pulsaradmin 17 | 18 | import ( 19 | "github.com/streamnative/pulsar-admin-go/pkg/admin" 20 | "github.com/streamnative/pulsar-admin-go/pkg/admin/config" 21 | ) 22 | 23 | // Client contains all admin interfaces for operating pulsar resources 24 | type Client = admin.Client 25 | 26 | // Config are the arguments for creating a new admin Client 27 | type Config = config.Config 28 | 29 | var ( 30 | // NewClient returns a new admin Client for operating pulsar resources 31 | NewClient = admin.New 32 | ) 33 | -------------------------------------------------------------------------------- /pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "fmt" 20 | "reflect" 21 | ) 22 | 23 | func MakeHTTPPath(apiVersion string, componentPath string) string { 24 | return fmt.Sprintf("/admin/%s%s", apiVersion, componentPath) 25 | } 26 | 27 | func IsNilFixed(i interface{}) bool { 28 | if i == nil { 29 | return true 30 | } 31 | switch reflect.TypeOf(i).Kind() { 32 | case reflect.Ptr, reflect.Map, reflect.Array, reflect.Chan, reflect.Slice: 33 | return reflect.ValueOf(i).IsNil() 34 | } 35 | return false 36 | } 37 | -------------------------------------------------------------------------------- /pkg/utils/package_metadata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type PackageMetadata struct { 19 | Description string `json:"description,omitempty" yaml:"description"` 20 | Contact string `json:"contact,omitempty" yaml:"contact"` 21 | CreateTime int64 `json:"createTime,omitempty" yaml:"createTime"` 22 | ModificationTime int64 `json:"modificationTime,omitempty" yaml:"modificationTime"` 23 | Properties map[string]string `json:"properties,omitempty" yaml:"properties"` 24 | } 25 | -------------------------------------------------------------------------------- /pkg/utils/subscription_auth_mode.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import "github.com/pkg/errors" 19 | 20 | type SubscriptionAuthMode string 21 | 22 | const ( 23 | None SubscriptionAuthMode = "None" 24 | Prefix SubscriptionAuthMode = "Prefix" 25 | ) 26 | 27 | func ParseSubscriptionAuthMode(s string) (SubscriptionAuthMode, error) { 28 | switch s { 29 | case "None": 30 | return None, nil 31 | case "Prefix": 32 | return Prefix, nil 33 | default: 34 | return "", errors.New("Invalid subscription auth mode") 35 | } 36 | } 37 | 38 | func (s SubscriptionAuthMode) String() string { 39 | return string(s) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/utils/auth_polices.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type AuthPolicies struct { 19 | NamespaceAuth map[string][]AuthAction `json:"namespace_auth"` 20 | DestinationAuth map[string]map[string][]AuthAction `json:"destination_auth"` 21 | SubscriptionAuthRoles map[string][]string `json:"subscription_auth_roles"` 22 | } 23 | 24 | func NewAuthPolicies() *AuthPolicies { 25 | return &AuthPolicies{ 26 | NamespaceAuth: make(map[string][]AuthAction), 27 | DestinationAuth: make(map[string]map[string][]AuthAction), 28 | SubscriptionAuthRoles: make(map[string][]string), 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /pkg/utils/crypto_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type CryptoConfig struct { 19 | CryptoKeyReaderClassName string `json:"cryptoKeyReaderClassName" yaml:"cryptoKeyReaderClassName"` 20 | CryptoKeyReaderConfig map[string]interface{} `json:"cryptoKeyReaderConfig" yaml:"cryptoKeyReaderConfig"` 21 | 22 | EncryptionKeys []string `json:"encryptionKeys" yaml:"encryptionKeys"` 23 | ProducerCryptoFailureAction string `json:"producerCryptoFailureAction" yaml:"producerCryptoFailureAction"` 24 | ConsumerCryptoFailureAction string `json:"consumerCryptoFailureAction" yaml:"consumerCryptoFailureAction"` 25 | } 26 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/streamnative/pulsar-admin-go 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/99designs/keyring v1.2.1 7 | github.com/apache/pulsar-client-go v0.9.0 8 | github.com/golang/protobuf v1.5.2 9 | github.com/pkg/errors v0.9.1 10 | github.com/stretchr/testify v1.8.0 11 | golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 12 | ) 13 | 14 | require ( 15 | github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect 16 | github.com/danieljoos/wincred v1.1.2 // indirect 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/dvsekhvalnov/jose2go v1.5.0 // indirect 19 | github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect 20 | github.com/golang-jwt/jwt v3.2.1+incompatible // indirect 21 | github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect 22 | github.com/mtibben/percent v0.2.1 // indirect 23 | github.com/pmezard/go-difflib v1.0.0 // indirect 24 | golang.org/x/net v0.7.0 // indirect 25 | golang.org/x/sys v0.5.0 // indirect 26 | golang.org/x/term v0.5.0 // indirect 27 | google.golang.org/appengine v1.6.7 // indirect 28 | google.golang.org/protobuf v1.26.0 // indirect 29 | gopkg.in/yaml.v3 v3.0.1 // indirect 30 | ) 31 | 32 | replace golang.org/x/sys => golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 33 | -------------------------------------------------------------------------------- /pkg/rest/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package rest 17 | 18 | import ( 19 | "io" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestEncodeJSONBody(t *testing.T) { 26 | testcases := []struct { 27 | obj interface{} 28 | expected int 29 | }{ 30 | {obj: "1", expected: 3}, 31 | {obj: "12", expected: 4}, 32 | {obj: 1, expected: 1}, 33 | {obj: 12, expected: 2}, 34 | } 35 | 36 | for _, testcase := range testcases { 37 | r, err := encodeJSONBody(testcase.obj) 38 | require.NoError(t, err) 39 | 40 | b, err := io.ReadAll(r) 41 | require.NoError(t, err) 42 | 43 | require.Equal(t, testcase.expected, len(b)) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /pkg/utils/long_running_process_status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type Status string 19 | 20 | const ( 21 | NOTRUN Status = "NOT_RUN" 22 | RUNNING Status = "RUNNING" 23 | SUCCESS Status = "SUCCESS" 24 | ERROR Status = "ERROR" 25 | ) 26 | 27 | type LongRunningProcessStatus struct { 28 | Status Status `json:"status"` 29 | LastError string `json:"lastError"` 30 | } 31 | 32 | type OffloadProcessStatus struct { 33 | Status Status `json:"status"` 34 | LastError string `json:"lastError"` 35 | FirstUnOffloadedMessage MessageID `json:"firstUnoffloadedMessage"` 36 | } 37 | 38 | func (s Status) String() string { 39 | return string(s) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/utils/producer_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type ProducerConfig struct { 19 | MaxPendingMessages int `json:"maxPendingMessages" yaml:"maxPendingMessages"` 20 | //nolint 21 | MaxPendingMessagesAcrossPartitions int `json:"maxPendingMessagesAcrossPartitions" yaml:"maxPendingMessagesAcrossPartitions"` 22 | 23 | UseThreadLocalProducers bool `json:"useThreadLocalProducers" yaml:"useThreadLocalProducers"` 24 | CryptoConfig *CryptoConfig `json:"cryptoConfig" yaml:"cryptoConfig"` 25 | BatchBuilder string `json:"batchBuilder" yaml:"batchBuilder"` 26 | CompressionType string `json:"compressionType" yaml:"compressionType"` 27 | } 28 | -------------------------------------------------------------------------------- /pkg/utils/topic_domain.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import "github.com/pkg/errors" 19 | 20 | type TopicDomain string 21 | 22 | const ( 23 | persistent TopicDomain = "persistent" 24 | nonPersistent TopicDomain = "non-persistent" 25 | ) 26 | 27 | func ParseTopicDomain(domain string) (TopicDomain, error) { 28 | switch domain { 29 | case "persistent": 30 | return persistent, nil 31 | case "non-persistent": 32 | return nonPersistent, nil 33 | default: 34 | return "", errors.Errorf("The domain only can be specified as 'persistent' or "+ 35 | "'non-persistent'. Input domain is '%s'.", domain) 36 | } 37 | } 38 | 39 | func (t TopicDomain) String() string { 40 | return string(t) 41 | } 42 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 StreamNative, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, 10 | # software distributed under the License is distributed on an 11 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | # KIND, either express or implied. See the License for the 13 | # specific language governing permissions and limitations 14 | # under the License. 15 | # 16 | 17 | name: Release 18 | 19 | on: 20 | push: 21 | tags: 22 | - 'v[0-9]+.[0-9]+.[0-9]+-?*' 23 | 24 | permissions: 25 | contents: write 26 | 27 | jobs: 28 | release: 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Checkout 32 | uses: actions/checkout@v3 33 | with: 34 | fetch-depth: 0 35 | 36 | - name: Setup Go 37 | uses: actions/setup-go@v3 38 | 39 | - name: Run GoReleaser 40 | uses: goreleaser/goreleaser-action@v4 41 | with: 42 | distribution: goreleaser 43 | version: latest 44 | args: release --clean 45 | env: 46 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /pkg/utils/connector_definition.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | // Basic information about a Pulsar connector 19 | type ConnectorDefinition struct { 20 | // The name of the connector type 21 | Name string `json:"name"` 22 | 23 | // Description to be used for user help 24 | Description string `json:"description"` 25 | 26 | // The class name for the connector source implementation 27 | //

If not defined, it will be assumed this connector cannot act as a data source 28 | SourceClass string `json:"sourceClass"` 29 | 30 | // The class name for the connector sink implementation 31 | //

If not defined, it will be assumed this connector cannot act as a data sink 32 | SinkClass string `json:"sinkClass"` 33 | } 34 | -------------------------------------------------------------------------------- /pkg/utils/topic_type.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import "github.com/pkg/errors" 19 | 20 | type TopicType string 21 | 22 | const ( 23 | Partitioned TopicType = "partitioned" 24 | NonPartitioned TopicType = "non-partitioned" 25 | ) 26 | 27 | func ParseTopicType(topicType string) (TopicType, error) { 28 | switch topicType { 29 | case "partitioned": 30 | return Partitioned, nil 31 | case "non-partitioned": 32 | return NonPartitioned, nil 33 | default: 34 | return "", errors.Errorf("The topic type can only be specified as 'partitioned' or "+ 35 | "'non-partitioned'. Input topic type is '%s'.", topicType) 36 | } 37 | } 38 | 39 | func (t TopicType) String() string { 40 | return string(t) 41 | } 42 | -------------------------------------------------------------------------------- /pkg/utils/bundles_data.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type BundlesData struct { 19 | Boundaries []string `json:"boundaries"` 20 | NumBundles int `json:"numBundles"` 21 | } 22 | 23 | func NewBundlesData(boundaries []string) BundlesData { 24 | return BundlesData{ 25 | Boundaries: boundaries, 26 | NumBundles: len(boundaries) - 1, 27 | } 28 | } 29 | 30 | func NewBundlesDataWithNumBundles(numBundles int) *BundlesData { 31 | return &BundlesData{ 32 | Boundaries: nil, 33 | NumBundles: numBundles, 34 | } 35 | } 36 | 37 | func NewDefaultBoundle() *BundlesData { 38 | bundleData := NewBundlesDataWithNumBundles(1) 39 | bundleData.Boundaries = append(bundleData.Boundaries, FirstBoundary, LastBoundary) 40 | return bundleData 41 | } 42 | -------------------------------------------------------------------------------- /pkg/utils/resource_quota.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type ResourceQuota struct { 19 | // messages published per second 20 | MsgRateIn float64 `json:"msgRateIn"` 21 | // messages consumed per second 22 | MsgRateOut float64 `json:"msgRateOut"` 23 | // incoming bytes per second 24 | BandwidthIn float64 `json:"bandwidthIn"` 25 | // outgoing bytes per second 26 | BandwidthOut float64 `json:"bandwidthOut"` 27 | // used memory in Mbytes 28 | Memory float64 `json:"memory"` 29 | // allow the quota be dynamically re-calculated according to real traffic 30 | Dynamic bool `json:"dynamic"` 31 | } 32 | 33 | func NewResourceQuota() *ResourceQuota { 34 | return &ResourceQuota{ 35 | MsgRateIn: 0.0, 36 | MsgRateOut: 0.0, 37 | BandwidthIn: 0.0, 38 | BandwidthOut: 0.0, 39 | Memory: 0.0, 40 | Dynamic: true, 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pkg/utils/package_type.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import "github.com/pkg/errors" 19 | 20 | type PackageType string 21 | 22 | const ( 23 | PackageTypeFunction PackageType = "function" 24 | PackageTypeSink PackageType = "sink" 25 | PackageTypeSource PackageType = "source" 26 | ) 27 | 28 | func parsePackageType(packageTypeName string) (PackageType, error) { 29 | switch packageTypeName { 30 | case PackageTypeFunction.String(): 31 | return PackageTypeFunction, nil 32 | case PackageTypeSink.String(): 33 | return PackageTypeSink, nil 34 | case PackageTypeSource.String(): 35 | return PackageTypeSource, nil 36 | default: 37 | return "", errors.Errorf("Invalid package type '%s', it should be "+ 38 | "function, sink, or source", packageTypeName) 39 | } 40 | } 41 | 42 | func (p PackageType) String() string { 43 | return string(p) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/utils/consumer_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type ConsumerConfig struct { 19 | SchemaType string `json:"schemaType,omitempty" yaml:"schemaType"` 20 | SerdeClassName string `json:"serdeClassName,omitempty" yaml:"serdeClassName"` 21 | RegexPattern bool `json:"regexPattern,omitempty" yaml:"regexPattern"` 22 | ReceiverQueueSize int `json:"receiverQueueSize,omitempty" yaml:"receiverQueueSize"` 23 | SchemaProperties map[string]string `json:"schemaProperties,omitempty" yaml:"schemaProperties"` 24 | ConsumerProperties map[string]string `json:"consumerProperties,omitempty" yaml:"consumerProperties"` 25 | CryptoConfig *CryptoConfig `json:"cryptoConfig,omitempty" yaml:"cryptoConfig"` 26 | PoolMessages bool `json:"poolMessages,omitempty" yaml:"poolMessages"` 27 | } 28 | -------------------------------------------------------------------------------- /pkg/utils/dispatch_rate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type DispatchRate struct { 19 | DispatchThrottlingRateInMsg int `json:"dispatchThrottlingRateInMsg"` 20 | DispatchThrottlingRateInByte int64 `json:"dispatchThrottlingRateInByte"` 21 | RatePeriodInSecond int `json:"ratePeriodInSecond"` 22 | } 23 | 24 | func NewDispatchRate() *DispatchRate { 25 | return &DispatchRate{ 26 | DispatchThrottlingRateInMsg: -1, 27 | DispatchThrottlingRateInByte: -1, 28 | RatePeriodInSecond: 1, 29 | } 30 | } 31 | 32 | type SubscribeRate struct { 33 | SubscribeThrottlingRatePerConsumer int `json:"subscribeThrottlingRatePerConsumer"` 34 | RatePeriodInSecond int `json:"ratePeriodInSecond"` 35 | } 36 | 37 | func NewSubscribeRate() *SubscribeRate { 38 | return &SubscribeRate{ 39 | SubscribeThrottlingRatePerConsumer: -1, 40 | RatePeriodInSecond: 30, 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2023 StreamNative, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | .PHONY: all 16 | all: license-check lint test 17 | 18 | .PHONY: lint 19 | lint: golangci-lint 20 | $(GOLANGCI_LINT) run 21 | 22 | .PHONY: test 23 | test: 24 | @go build ./... && go test -race ./... 25 | 26 | .PHONY: license-check 27 | license-check: license-eye 28 | $(LICENSE_EYE) header check 29 | 30 | .PHONY: license-fix 31 | license-fix: 32 | $(LICENSE_EYE) header fix 33 | 34 | # Install development dependencies 35 | 36 | ifeq (,$(shell go env GOBIN)) 37 | GOBIN=$(shell go env GOPATH)/bin 38 | else 39 | GOBIN=$(shell go env GOBIN) 40 | endif 41 | 42 | LICENSE_EYE ?= $(GOBIN)/license-eye 43 | GOLANGCI_LINT ?= $(GOBIN)/golangci-lint 44 | 45 | .PHONY: license-eye 46 | license-eye: $(LICENSE_EYE) 47 | $(LICENSE_EYE): $(GOBIN) 48 | test -s $(GOBIN)/license-eye || go install github.com/apache/skywalking-eyes/cmd/license-eye@e1a0235 49 | 50 | .PHONY: golangci-lint 51 | golangci-lint: $(GOLANGCI_LINT) 52 | $(GOLANGCI_LINT): $(GOBIN) 53 | test -s $(GOBIN)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2 54 | -------------------------------------------------------------------------------- /pkg/utils/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | type People interface { 25 | MakeSound() string 26 | } 27 | 28 | type Student struct{} 29 | 30 | func (s *Student) MakeSound() string { 31 | return "Student" 32 | } 33 | 34 | type Teacher struct{} 35 | 36 | func (t Teacher) MakeSound() string { 37 | return "Teacher" 38 | } 39 | 40 | // nolint 41 | func TestIsNilFixed(t *testing.T) { 42 | var stu *Student = nil 43 | var people People 44 | people = stu 45 | 46 | var teacher Teacher 47 | people = teacher 48 | 49 | assert.False(t, IsNilFixed(people)) 50 | 51 | var m map[string]string 52 | assert.True(t, IsNilFixed(m)) 53 | 54 | var s []string 55 | assert.True(t, IsNilFixed(s)) 56 | 57 | var ch chan string 58 | assert.True(t, IsNilFixed(ch)) 59 | 60 | var nilInterface People 61 | assert.True(t, IsNilFixed(nilInterface)) 62 | 63 | // pointer to an interface, the IsNilFixed method cannot check this. 64 | assert.False(t, IsNilFixed(&nilInterface)) 65 | } 66 | -------------------------------------------------------------------------------- /pkg/utils/auth_action.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import "github.com/pkg/errors" 19 | 20 | type AuthAction string 21 | 22 | const ( 23 | produce AuthAction = "produce" 24 | consume AuthAction = "consume" 25 | functionsAuth AuthAction = "functions" 26 | packages AuthAction = "packages" 27 | sinks AuthAction = "sinks" 28 | sources AuthAction = "sources" 29 | ) 30 | 31 | func ParseAuthAction(action string) (AuthAction, error) { 32 | switch action { 33 | case "produce": 34 | return produce, nil 35 | case "consume": 36 | return consume, nil 37 | case "functions": 38 | return functionsAuth, nil 39 | case "packages": 40 | return packages, nil 41 | case "sinks": 42 | return sinks, nil 43 | case "sources": 44 | return sources, nil 45 | default: 46 | return "", errors.Errorf("The auth action only can be specified as 'produce', "+ 47 | "'consume', 'sources', 'sinks', 'packages', or 'functions'. Invalid auth action '%s'", action) 48 | } 49 | } 50 | 51 | func (a AuthAction) String() string { 52 | return string(a) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/utils/persistence_policies.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type PersistencePolicies struct { 19 | BookkeeperEnsemble int `json:"bookkeeperEnsemble"` 20 | BookkeeperWriteQuorum int `json:"bookkeeperWriteQuorum"` 21 | BookkeeperAckQuorum int `json:"bookkeeperAckQuorum"` 22 | ManagedLedgerMaxMarkDeleteRate float64 `json:"managedLedgerMaxMarkDeleteRate"` 23 | } 24 | 25 | func NewPersistencePolicies(bookkeeperEnsemble, bookkeeperWriteQuorum, bookkeeperAckQuorum int, 26 | managedLedgerMaxMarkDeleteRate float64) PersistencePolicies { 27 | return PersistencePolicies{ 28 | BookkeeperEnsemble: bookkeeperEnsemble, 29 | BookkeeperWriteQuorum: bookkeeperWriteQuorum, 30 | BookkeeperAckQuorum: bookkeeperAckQuorum, 31 | ManagedLedgerMaxMarkDeleteRate: managedLedgerMaxMarkDeleteRate, 32 | } 33 | } 34 | 35 | type BookieAffinityGroupData struct { 36 | BookkeeperAffinityGroupPrimary string `json:"bookkeeperAffinityGroupPrimary"` 37 | BookkeeperAffinityGroupSecondary string `json:"bookkeeperAffinityGroupSecondary"` 38 | } 39 | -------------------------------------------------------------------------------- /pkg/admin/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package config 17 | 18 | type Config struct { 19 | // the web service url that pulsarctl connects to. Default is http://localhost:8080 20 | WebServiceURL string 21 | 22 | // the bookkeeper service url that pulsarctl connects to. 23 | BKWebServiceURL string 24 | // Set the path to the trusted TLS certificate file 25 | TLSTrustCertsFilePath string 26 | // Configure whether the Pulsar client accept untrusted TLS certificate from broker (default: false) 27 | TLSAllowInsecureConnection bool 28 | 29 | TLSEnableHostnameVerification bool 30 | 31 | AuthPlugin string 32 | 33 | AuthParams string 34 | 35 | // TLS Cert and Key Files for authentication 36 | TLSCertFile string 37 | TLSKeyFile string 38 | 39 | // Token and TokenFile is used to config the pulsarctl using token to authentication 40 | Token string 41 | TokenFile string 42 | PulsarAPIVersion APIVersion 43 | 44 | // OAuth2 configuration 45 | IssuerEndpoint string 46 | ClientID string 47 | Audience string 48 | KeyFile string 49 | Scope string 50 | } 51 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 StreamNative, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, 10 | # software distributed under the License is distributed on an 11 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | # KIND, either express or implied. See the License for the 13 | # specific language governing permissions and limitations 14 | # under the License. 15 | # 16 | 17 | name: CI 18 | 19 | on: 20 | pull_request: 21 | branches: 22 | - '*' 23 | 24 | jobs: 25 | license-check: 26 | runs-on: ubuntu-latest 27 | steps: 28 | - name: Checkout 29 | uses: actions/checkout@v3 30 | 31 | - name: Check license header 32 | uses: apache/skywalking-eyes@e1a02359b239bd28de3f6d35fdc870250fa513d5 33 | 34 | lint: 35 | runs-on: ubuntu-latest 36 | steps: 37 | - name: Checkout 38 | uses: actions/checkout@v3 39 | 40 | - name: Setup Go 41 | uses: actions/setup-go@v3 42 | with: 43 | go-version: 1.18 44 | 45 | - name: Run golangci-lint 46 | uses: golangci/golangci-lint-action@v3 47 | with: 48 | version: v1.51.2 49 | 50 | test: 51 | runs-on: ubuntu-latest 52 | steps: 53 | - name: Checkout 54 | uses: actions/checkout@v3 55 | 56 | - name: Setup Go 57 | uses: actions/setup-go@v3 58 | with: 59 | go-version: 1.18 60 | 61 | - name: Run build and test 62 | run: | 63 | go build ./... 64 | go test -race ./... 65 | 66 | -------------------------------------------------------------------------------- /pkg/admin/auth/transport.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package auth 17 | 18 | import ( 19 | "crypto/tls" 20 | "crypto/x509" 21 | "net/http" 22 | "os" 23 | 24 | "github.com/streamnative/pulsar-admin-go/pkg/admin/config" 25 | ) 26 | 27 | type Transport struct { 28 | T http.RoundTripper 29 | } 30 | 31 | // GetDefaultTransport gets a default transport. 32 | // Deprecated: Use NewDefaultTransport instead. 33 | func GetDefaultTransport(config *config.Config) http.RoundTripper { 34 | transport, err := NewDefaultTransport(config) 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | return transport 40 | } 41 | 42 | func NewDefaultTransport(config *config.Config) (http.RoundTripper, error) { 43 | transport := http.DefaultTransport.(*http.Transport).Clone() 44 | tlsConfig := &tls.Config{ 45 | InsecureSkipVerify: config.TLSAllowInsecureConnection, 46 | } 47 | if len(config.TLSTrustCertsFilePath) > 0 { 48 | rootCA, err := os.ReadFile(config.TLSTrustCertsFilePath) 49 | if err != nil { 50 | return nil, err 51 | } 52 | tlsConfig.RootCAs = x509.NewCertPool() 53 | tlsConfig.RootCAs.AppendCertsFromPEM(rootCA) 54 | } 55 | transport.MaxIdleConnsPerHost = 10 56 | transport.TLSClientConfig = tlsConfig 57 | return transport, nil 58 | } 59 | -------------------------------------------------------------------------------- /pkg/utils/window_confing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | const WindowConfigKey = "__WINDOWCONFIGS__" 19 | 20 | type WindowConfig struct { 21 | WindowLengthCount *int `json:"windowLengthCount" yaml:"windowLengthCount"` 22 | WindowLengthDurationMs *int64 `json:"windowLengthDurationMs" yaml:"windowLengthDurationMs"` 23 | SlidingIntervalCount *int `json:"slidingIntervalCount" yaml:"slidingIntervalCount"` 24 | SlidingIntervalDurationMs *int64 `json:"slidingIntervalDurationMs" yaml:"slidingIntervalDurationMs"` 25 | LateDataTopic *string `json:"lateDataTopic" yaml:"lateDataTopic"` 26 | MaxLagMs *int64 `json:"maxLagMs" yaml:"maxLagMs"` 27 | WatermarkEmitIntervalMs *int64 `json:"watermarkEmitIntervalMs" yaml:"watermarkEmitIntervalMs"` 28 | TimestampExtractorClassName *string `json:"timestampExtractorClassName" yaml:"timestampExtractorClassName"` 29 | ActualWindowFunctionClassName *string `json:"actualWindowFunctionClassName" yaml:"actualWindowFunctionClassName"` 30 | ProcessingGuarantees *string `json:"processingGuarantees" yaml:"processingGuarantees"` 31 | } 32 | 33 | func NewDefaultWindowConfing() *WindowConfig { 34 | windowConfig := &WindowConfig{} 35 | 36 | return windowConfig 37 | } 38 | -------------------------------------------------------------------------------- /pkg/utils/source_status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type SourceStatus struct { 19 | NumInstances int `json:"numInstances"` 20 | NumRunning int `json:"numRunning"` 21 | Instances []*SourceInstanceStatus `json:"instances"` 22 | } 23 | 24 | type SourceInstanceStatus struct { 25 | InstanceID int `json:"instanceId"` 26 | Status SourceInstanceStatusData `json:"status"` 27 | } 28 | 29 | type SourceInstanceStatusData struct { 30 | Running bool `json:"running"` 31 | Err string `json:"error"` 32 | NumRestarts int64 `json:"numRestarts"` 33 | NumReceivedFromSource int64 `json:"numReceivedFromSource"` 34 | NumSystemExceptions int64 `json:"numSystemExceptions"` 35 | LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"` 36 | NumSourceExceptions int64 `json:"numSourceExceptions"` 37 | LatestSourceExceptions []ExceptionInformation `json:"latestSourceExceptions"` 38 | NumWritten int64 `json:"numWritten"` 39 | LastReceivedTime int64 `json:"lastReceivedTime"` 40 | WorkerID string `json:"workerId"` 41 | } 42 | -------------------------------------------------------------------------------- /pkg/utils/schema_strategy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import "github.com/pkg/errors" 19 | 20 | type SchemaCompatibilityStrategy string 21 | 22 | const ( 23 | AutoUpdateDisabled SchemaCompatibilityStrategy = "AutoUpdateDisabled" 24 | Backward SchemaCompatibilityStrategy = "Backward" 25 | Forward SchemaCompatibilityStrategy = "Forward" 26 | Full SchemaCompatibilityStrategy = "Full" 27 | AlwaysCompatible SchemaCompatibilityStrategy = "AlwaysCompatible" 28 | BackwardTransitive SchemaCompatibilityStrategy = "BackwardTransitive" 29 | ForwardTransitive SchemaCompatibilityStrategy = "ForwardTransitive" 30 | FullTransitive SchemaCompatibilityStrategy = "FullTransitive" 31 | ) 32 | 33 | func ParseSchemaAutoUpdateCompatibilityStrategy(str string) (SchemaCompatibilityStrategy, error) { 34 | switch str { 35 | case "AutoUpdateDisabled": 36 | return AutoUpdateDisabled, nil 37 | case "Backward": 38 | return Backward, nil 39 | case "Forward": 40 | return Forward, nil 41 | case "Full": 42 | return Full, nil 43 | case "AlwaysCompatible": 44 | return AlwaysCompatible, nil 45 | case "BackwardTransitive": 46 | return BackwardTransitive, nil 47 | case "ForwardTransitive": 48 | return ForwardTransitive, nil 49 | case "FullTransitive": 50 | return FullTransitive, nil 51 | default: 52 | return "", errors.Errorf("Invalid auth strategy %s", str) 53 | } 54 | } 55 | 56 | func (s SchemaCompatibilityStrategy) String() string { 57 | return string(s) 58 | } 59 | -------------------------------------------------------------------------------- /pkg/utils/message_id_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestParseMessageId(t *testing.T) { 25 | id, err := ParseMessageID("1:1") 26 | assert.Nil(t, err) 27 | assert.Equal(t, MessageID{LedgerID: 1, EntryID: 1, PartitionIndex: -1, BatchIndex: -1}, *id) 28 | 29 | id, err = ParseMessageID("1:2:3") 30 | assert.Nil(t, err) 31 | assert.Equal(t, MessageID{LedgerID: 1, EntryID: 2, PartitionIndex: 3, BatchIndex: -1}, *id) 32 | 33 | id, err = ParseMessageID("1:2:3:4") 34 | assert.Nil(t, err) 35 | assert.Equal(t, MessageID{LedgerID: 1, EntryID: 2, PartitionIndex: 3, BatchIndex: 4}, *id) 36 | } 37 | 38 | func TestParseMessageIdErrors(t *testing.T) { 39 | id, err := ParseMessageID("1;1") 40 | assert.Nil(t, id) 41 | assert.NotNil(t, err) 42 | assert.Equal(t, "invalid message id string. 1;1", err.Error()) 43 | 44 | id, err = ParseMessageID("a:1") 45 | assert.Nil(t, id) 46 | assert.NotNil(t, err) 47 | assert.Equal(t, "invalid ledger id. a:1", err.Error()) 48 | 49 | id, err = ParseMessageID("1:a") 50 | assert.Nil(t, id) 51 | assert.NotNil(t, err) 52 | assert.Equal(t, "invalid entry id. 1:a", err.Error()) 53 | 54 | id, err = ParseMessageID("1:2:a") 55 | assert.Nil(t, id) 56 | assert.NotNil(t, err) 57 | assert.Equal(t, "invalid partition index. 1:2:a", err.Error()) 58 | 59 | id, err = ParseMessageID("1:2:3:a") 60 | assert.Nil(t, id) 61 | assert.NotNil(t, err) 62 | assert.Equal(t, "invalid batch index. 1:2:3:a", err.Error()) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/utils/function_status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type FunctionStatus struct { 19 | NumInstances int `json:"numInstances"` 20 | NumRunning int `json:"numRunning"` 21 | Instances []FunctionInstanceStatus `json:"instances"` 22 | } 23 | 24 | type FunctionInstanceStatus struct { 25 | InstanceID int `json:"instanceId"` 26 | Status FunctionInstanceStatusData `json:"status"` 27 | } 28 | 29 | type FunctionInstanceStatusData struct { 30 | Running bool `json:"running"` 31 | Err string `json:"error"` 32 | NumRestarts int64 `json:"numRestarts"` 33 | NumReceived int64 `json:"numReceived"` 34 | NumSuccessfullyProcessed int64 `json:"numSuccessfullyProcessed"` 35 | NumUserExceptions int64 `json:"numUserExceptions"` 36 | LatestUserExceptions []ExceptionInformation `json:"latestUserExceptions"` 37 | NumSystemExceptions int64 `json:"numSystemExceptions"` 38 | LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"` 39 | AverageLatency float64 `json:"averageLatency"` 40 | LastInvocationTime int64 `json:"lastInvocationTime"` 41 | WorkerID string `json:"workerId"` 42 | } 43 | 44 | type ExceptionInformation struct { 45 | ExceptionString string `json:"exceptionString"` 46 | TimestampMs int64 `json:"timestampMs"` 47 | } 48 | -------------------------------------------------------------------------------- /pkg/utils/auth_polices_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "encoding/json" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | ) 24 | 25 | func TestAuthPolicies(t *testing.T) { 26 | testData := "{\n" + 27 | " \"namespace_auth\": {\n" + 28 | " \"persistent://public/default/ns_auth\": [\n" + 29 | " \"produce\",\n" + 30 | " \"consume\",\n" + 31 | " \"function\"\n" + 32 | " ]\n" + 33 | " },\n" + 34 | " \"destination_auth\": {\n" + 35 | " \"persistent://public/default/dest_auth\": {\n" + 36 | " \"admin-role\": [\n" + 37 | " \"produce\",\n" + 38 | " \"consume\",\n" + 39 | " \"function\"\n" + 40 | " ]\n" + 41 | " },\n" + 42 | " \"persistent://public/default/dest_auth_1\": {\n" + 43 | " \"grant-partitioned-role\": [\n" + 44 | " \"produce\",\n" + 45 | " \"consume\"\n" + 46 | " ]\n" + 47 | " },\n" + 48 | " \"persistent://public/default/test-revoke-partitioned-topic\": {},\n" + 49 | " \"persistent://public/default/test-revoke-non-partitioned-topic\": {}\n },\n" + 50 | " \"subscription_auth_roles\": {}\n" + 51 | "}" 52 | 53 | policies := &AuthPolicies{} 54 | err := json.Unmarshal([]byte(testData), policies) 55 | if err != nil { 56 | t.Fatal(err) 57 | } 58 | 59 | assert.Equal(t, 3, len(policies.NamespaceAuth["persistent://public/default/ns_auth"])) 60 | assert.Equal(t, 4, len(policies.DestinationAuth)) 61 | assert.Equal(t, 3, len(policies.DestinationAuth["persistent://public/default/dest_auth"]["admin-role"])) 62 | assert.Equal(t, 0, len(policies.SubscriptionAuthRoles)) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/utils/namespace_name_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestGetNamespaceName(t *testing.T) { 25 | success, err := GetNamespaceName("public/default") 26 | assert.Nil(t, err) 27 | assert.Equal(t, "public/default", success.String()) 28 | 29 | empty, err := GetNamespaceName("") 30 | assert.NotNil(t, err) 31 | assert.Equal(t, "the namespace complete name is empty", err.Error()) 32 | assert.Nil(t, empty) 33 | 34 | empty, err = GetNamespaceName("/") 35 | assert.NotNil(t, err) 36 | assert.Equal(t, "Invalid tenant or namespace. [/]", err.Error()) 37 | assert.Nil(t, empty) 38 | 39 | invalid, err := GetNamespaceName("public/default/fail") 40 | assert.NotNil(t, err) 41 | assert.Equal(t, "The complete name of namespace is invalid. complete name : [public/default/fail]", err.Error()) 42 | assert.Nil(t, invalid) 43 | 44 | invalid, err = GetNamespaceName("public") 45 | assert.NotNil(t, err) 46 | assert.Equal(t, "The complete name of namespace is invalid. complete name : [public]", err.Error()) 47 | assert.Nil(t, invalid) 48 | 49 | special, err := GetNamespaceName("-=.:/-=.:") 50 | assert.Nil(t, err) 51 | assert.Equal(t, "-=.:/-=.:", special.String()) 52 | 53 | tenantInvalid, err := GetNamespaceName("\"/namespace") 54 | assert.NotNil(t, err) 55 | assert.Equal(t, "Tenant name include unsupported special chars. tenant : [\"]", err.Error()) 56 | assert.Nil(t, tenantInvalid) 57 | 58 | namespaceInvalid, err := GetNamespaceName("tenant/}") 59 | assert.NotNil(t, err) 60 | assert.Equal(t, "Namespace name include unsupported special chars. namespace : [}]", err.Error()) 61 | assert.Nil(t, namespaceInvalid) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/utils/message_id.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "strconv" 20 | "strings" 21 | 22 | "github.com/pkg/errors" 23 | ) 24 | 25 | type MessageID struct { 26 | LedgerID int64 `json:"ledgerId"` 27 | EntryID int64 `json:"entryId"` 28 | PartitionIndex int `json:"partitionIndex"` 29 | BatchIndex int `json:"-"` 30 | } 31 | 32 | var Latest = MessageID{0x7fffffffffffffff, 0x7fffffffffffffff, -1, -1} 33 | var Earliest = MessageID{-1, -1, -1, -1} 34 | 35 | func ParseMessageID(str string) (*MessageID, error) { 36 | s := strings.Split(str, ":") 37 | 38 | m := Earliest 39 | 40 | if len(s) < 2 || len(s) > 4 { 41 | return nil, errors.Errorf("invalid message id string. %s", str) 42 | } 43 | 44 | ledgerID, err := strconv.ParseInt(s[0], 10, 64) 45 | if err != nil { 46 | return nil, errors.Errorf("invalid ledger id. %s", str) 47 | } 48 | m.LedgerID = ledgerID 49 | 50 | entryID, err := strconv.ParseInt(s[1], 10, 64) 51 | if err != nil { 52 | return nil, errors.Errorf("invalid entry id. %s", str) 53 | } 54 | m.EntryID = entryID 55 | 56 | if len(s) > 2 { 57 | pi, err := strconv.Atoi(s[2]) 58 | if err != nil { 59 | return nil, errors.Errorf("invalid partition index. %s", str) 60 | } 61 | m.PartitionIndex = pi 62 | } 63 | 64 | if len(s) == 4 { 65 | bi, err := strconv.Atoi(s[3]) 66 | if err != nil { 67 | return nil, errors.Errorf("invalid batch index. %s", str) 68 | } 69 | m.BatchIndex = bi 70 | } 71 | 72 | return &m, nil 73 | } 74 | 75 | func (m MessageID) String() string { 76 | return strconv.FormatInt(m.LedgerID, 10) + ":" + 77 | strconv.FormatInt(m.EntryID, 10) + ":" + 78 | strconv.Itoa(m.PartitionIndex) + ":" + 79 | strconv.Itoa(m.BatchIndex) 80 | } 81 | -------------------------------------------------------------------------------- /pkg/utils/inactive_topic_policies.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import "github.com/pkg/errors" 19 | 20 | type InactiveTopicDeleteMode string 21 | 22 | const ( 23 | // The topic can be deleted when no subscriptions and no active producers. 24 | DeleteWhenNoSubscriptions InactiveTopicDeleteMode = "delete_when_no_subscriptions" 25 | // The topic can be deleted when all subscriptions catchup and no active producers/consumers. 26 | DeleteWhenSubscriptionsCaughtUp InactiveTopicDeleteMode = "delete_when_subscriptions_caught_up" 27 | ) 28 | 29 | func (i InactiveTopicDeleteMode) String() string { 30 | return string(i) 31 | } 32 | 33 | func ParseInactiveTopicDeleteMode(str string) (InactiveTopicDeleteMode, error) { 34 | switch str { 35 | case DeleteWhenNoSubscriptions.String(): 36 | return DeleteWhenNoSubscriptions, nil 37 | case DeleteWhenSubscriptionsCaughtUp.String(): 38 | return DeleteWhenSubscriptionsCaughtUp, nil 39 | default: 40 | return "", errors.Errorf("cannot parse %s to InactiveTopicDeleteMode type", str) 41 | } 42 | } 43 | 44 | type InactiveTopicPolicies struct { 45 | InactiveTopicDeleteMode *InactiveTopicDeleteMode `json:"inactiveTopicDeleteMode"` 46 | MaxInactiveDurationSeconds int `json:"maxInactiveDurationSeconds"` 47 | DeleteWhileInactive bool `json:"deleteWhileInactive"` 48 | } 49 | 50 | func NewInactiveTopicPolicies(inactiveTopicDeleteMode *InactiveTopicDeleteMode, maxInactiveDurationSeconds int, 51 | deleteWhileInactive bool) InactiveTopicPolicies { 52 | return InactiveTopicPolicies{ 53 | InactiveTopicDeleteMode: inactiveTopicDeleteMode, 54 | MaxInactiveDurationSeconds: maxInactiveDurationSeconds, 55 | DeleteWhileInactive: deleteWhileInactive, 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /pkg/utils/schema_util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type SchemaInfo struct { 19 | Name string `json:"name"` 20 | Schema []byte `json:"schema"` 21 | Type string `json:"type"` 22 | Properties map[string]string `json:"properties"` 23 | } 24 | 25 | type SchemaInfoWithVersion struct { 26 | Version int64 `json:"version"` 27 | SchemaInfo *SchemaInfo `json:"schemaInfo"` 28 | } 29 | 30 | // Payload with information about a schema 31 | type PostSchemaPayload struct { 32 | SchemaType string `json:"type"` 33 | Schema string `json:"schema"` 34 | Properties map[string]string `json:"properties"` 35 | } 36 | 37 | type GetSchemaResponse struct { 38 | Version int64 `json:"version"` 39 | Type string `json:"type"` 40 | Timestamp int64 `json:"timestamp"` 41 | Data string `json:"data"` 42 | Properties map[string]string `json:"properties"` 43 | } 44 | 45 | func ConvertGetSchemaResponseToSchemaInfo(tn *TopicName, response GetSchemaResponse) *SchemaInfo { 46 | info := new(SchemaInfo) 47 | schema := make([]byte, 0, 10) 48 | if response.Type == "KEY_VALUE" { 49 | // TODO: impl logic 50 | } else { 51 | schema = []byte(response.Data) 52 | } 53 | 54 | info.Schema = schema 55 | info.Type = response.Type 56 | info.Properties = response.Properties 57 | info.Name = tn.GetLocalName() 58 | 59 | return info 60 | } 61 | 62 | func ConvertGetSchemaResponseToSchemaInfoWithVersion(tn *TopicName, response GetSchemaResponse) *SchemaInfoWithVersion { 63 | info := new(SchemaInfoWithVersion) 64 | info.SchemaInfo = ConvertGetSchemaResponseToSchemaInfo(tn, response) 65 | info.Version = response.Version 66 | return info 67 | } 68 | -------------------------------------------------------------------------------- /pkg/utils/sink_status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type SinkStatus struct { 19 | // The total number of sink instances that ought to be running 20 | NumInstances int `json:"numInstances"` 21 | 22 | // The number of source instances that are actually running 23 | NumRunning int `json:"numRunning"` 24 | 25 | Instances []*SinkInstanceStatus `json:"instances"` 26 | } 27 | 28 | type SinkInstanceStatus struct { 29 | InstanceID int `json:"instanceId"` 30 | Status SourceInstanceStatusData `json:"status"` 31 | } 32 | 33 | type SinkInstanceStatusData struct { 34 | // Is this instance running? 35 | Running bool `json:"running"` 36 | 37 | // Do we have any error while running this instance 38 | Err string `json:"error"` 39 | 40 | // Number of times this instance has restarted 41 | NumRestarts int64 `json:"numRestarts"` 42 | 43 | // Number of messages read from Pulsar 44 | NumReadFromPulsar int64 `json:"numReadFromPulsar"` 45 | 46 | // Number of times there was a system exception handling messages 47 | NumSystemExceptions int64 `json:"numSystemExceptions"` 48 | 49 | // A list of the most recent system exceptions 50 | LatestSystemExceptions []ExceptionInformation `json:"latestSystemExceptions"` 51 | 52 | // Number of times there was a sink exception 53 | NumSinkExceptions int64 `json:"numSinkExceptions"` 54 | 55 | // A list of the most recent sink exceptions 56 | LatestSinkExceptions []ExceptionInformation `json:"latestSinkExceptions"` 57 | 58 | // Number of messages written to sink 59 | NumWrittenToSink int64 `json:"numWrittenToSink"` 60 | 61 | // When was the last time we received a Message from Pulsar 62 | LastReceivedTime int64 `json:"lastReceivedTime"` 63 | 64 | WorkerID string `json:"workerId"` 65 | } 66 | -------------------------------------------------------------------------------- /pkg/admin/tenant.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 20 | ) 21 | 22 | // Tenants is admin interface for tenants management 23 | type Tenants interface { 24 | // Create a new tenant 25 | Create(utils.TenantData) error 26 | 27 | // Delete an existing tenant 28 | Delete(string) error 29 | 30 | // Update the admins for a tenant 31 | Update(utils.TenantData) error 32 | 33 | // List returns the list of tenants 34 | List() ([]string, error) 35 | 36 | // Get returns the config of the tenant. 37 | Get(string) (utils.TenantData, error) 38 | } 39 | 40 | type tenants struct { 41 | pulsar *pulsarClient 42 | basePath string 43 | } 44 | 45 | // Tenants is used to access the tenants endpoints 46 | func (c *pulsarClient) Tenants() Tenants { 47 | return &tenants{ 48 | pulsar: c, 49 | basePath: "/tenants", 50 | } 51 | } 52 | 53 | func (c *tenants) Create(data utils.TenantData) error { 54 | endpoint := c.pulsar.endpoint(c.basePath, data.Name) 55 | return c.pulsar.Client.Put(endpoint, &data) 56 | } 57 | 58 | func (c *tenants) Delete(name string) error { 59 | endpoint := c.pulsar.endpoint(c.basePath, name) 60 | return c.pulsar.Client.Delete(endpoint) 61 | } 62 | 63 | func (c *tenants) Update(data utils.TenantData) error { 64 | endpoint := c.pulsar.endpoint(c.basePath, data.Name) 65 | return c.pulsar.Client.Post(endpoint, &data) 66 | } 67 | 68 | func (c *tenants) List() ([]string, error) { 69 | var tenantList []string 70 | endpoint := c.pulsar.endpoint(c.basePath, "") 71 | err := c.pulsar.Client.Get(endpoint, &tenantList) 72 | return tenantList, err 73 | } 74 | 75 | func (c *tenants) Get(name string) (utils.TenantData, error) { 76 | var data utils.TenantData 77 | endpoint := c.pulsar.endpoint(c.basePath, name) 78 | err := c.pulsar.Client.Get(endpoint, &data) 79 | return data, err 80 | } 81 | -------------------------------------------------------------------------------- /pkg/utils/backlog_quota.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import "github.com/pkg/errors" 19 | 20 | type BacklogQuota struct { 21 | LimitTime int64 `json:"limitTime"` 22 | LimitSize int64 `json:"limitSize"` 23 | Policy RetentionPolicy `json:"policy"` 24 | } 25 | 26 | func NewBacklogQuota(limitSize int64, limitTime int64, policy RetentionPolicy) BacklogQuota { 27 | return BacklogQuota{ 28 | LimitSize: limitSize, 29 | LimitTime: limitTime, 30 | Policy: policy, 31 | } 32 | } 33 | 34 | type RetentionPolicy string 35 | 36 | const ( 37 | ProducerRequestHold RetentionPolicy = "producer_request_hold" 38 | ProducerException RetentionPolicy = "producer_exception" 39 | ConsumerBacklogEviction RetentionPolicy = "consumer_backlog_eviction" 40 | ) 41 | 42 | func ParseRetentionPolicy(str string) (RetentionPolicy, error) { 43 | switch str { 44 | case ProducerRequestHold.String(): 45 | return ProducerRequestHold, nil 46 | case ProducerException.String(): 47 | return ProducerException, nil 48 | case ConsumerBacklogEviction.String(): 49 | return ConsumerBacklogEviction, nil 50 | default: 51 | return "", errors.Errorf("Invalid retention policy %s", str) 52 | } 53 | } 54 | 55 | func (s RetentionPolicy) String() string { 56 | return string(s) 57 | } 58 | 59 | type BacklogQuotaType string 60 | 61 | const ( 62 | DestinationStorage BacklogQuotaType = "destination_storage" 63 | MessageAge BacklogQuotaType = "message_age" 64 | ) 65 | 66 | func ParseBacklogQuotaType(str string) (BacklogQuotaType, error) { 67 | switch str { 68 | case "": 69 | fallthrough 70 | case DestinationStorage.String(): 71 | return DestinationStorage, nil 72 | case MessageAge.String(): 73 | return MessageAge, nil 74 | default: 75 | return "", errors.Errorf("Invalid backlog quota type: %s", str) 76 | } 77 | } 78 | 79 | func (b BacklogQuotaType) String() string { 80 | return string(b) 81 | } 82 | -------------------------------------------------------------------------------- /pkg/utils/source_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type SourceConfig struct { 19 | Tenant string `json:"tenant,omitempty" yaml:"tenant"` 20 | Namespace string `json:"namespace,omitempty" yaml:"namespace"` 21 | Name string `json:"name,omitempty" yaml:"name"` 22 | ClassName string `json:"className,omitempty" yaml:"className"` 23 | 24 | ProducerConfig *ProducerConfig `json:"producerConfig,omitempty" yaml:"producerConfig"` 25 | 26 | TopicName string `json:"topicName,omitempty" yaml:"topicName"` 27 | SerdeClassName string `json:"serdeClassName,omitempty" yaml:"serdeClassName"` 28 | SchemaType string `json:"schemaType,omitempty" yaml:"schemaType"` 29 | 30 | Configs map[string]interface{} `json:"configs,omitempty" yaml:"configs"` 31 | 32 | // This is a map of secretName(aka how the secret is going to be 33 | // accessed in the function via context) to an object that 34 | // encapsulates how the secret is fetched by the underlying 35 | // secrets provider. The type of an value here can be found by the 36 | // SecretProviderConfigurator.getSecretObjectType() method. 37 | Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"` 38 | 39 | Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"` 40 | ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"` 41 | Resources *Resources `json:"resources,omitempty" yaml:"resources"` 42 | Archive string `json:"archive,omitempty" yaml:"archive"` 43 | // Any flags that you want to pass to the runtime. 44 | RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"` 45 | 46 | CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"` 47 | 48 | BatchSourceConfig *BatchSourceConfig `json:"batchSourceConfig,omitempty" yaml:"batchSourceConfig"` 49 | BatchBuilder string `json:"batchBuilder,omitempty" yaml:"batchBuilder"` 50 | } 51 | -------------------------------------------------------------------------------- /pkg/utils/namespace_name.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "fmt" 20 | "regexp" 21 | "strings" 22 | 23 | "github.com/pkg/errors" 24 | ) 25 | 26 | type NameSpaceName struct { 27 | tenant string 28 | nameSpace string 29 | } 30 | 31 | func GetNameSpaceName(tenant, namespace string) (*NameSpaceName, error) { 32 | return GetNamespaceName(fmt.Sprintf("%s/%s", tenant, namespace)) 33 | } 34 | 35 | func GetNamespaceName(completeName string) (*NameSpaceName, error) { 36 | var n NameSpaceName 37 | 38 | if completeName == "" { 39 | return nil, errors.New("the namespace complete name is empty") 40 | } 41 | 42 | parts := strings.Split(completeName, "/") 43 | if len(parts) == 2 { 44 | n.tenant = parts[0] 45 | n.nameSpace = parts[1] 46 | err := validateNamespaceName(n.tenant, n.nameSpace) 47 | if err != nil { 48 | return nil, err 49 | } 50 | } else { 51 | return nil, errors.Errorf("The complete name of namespace is invalid. complete name : [%s]", completeName) 52 | } 53 | 54 | return &n, nil 55 | } 56 | 57 | func (n *NameSpaceName) String() string { 58 | return fmt.Sprintf("%s/%s", n.tenant, n.nameSpace) 59 | } 60 | 61 | func validateNamespaceName(tenant, namespace string) error { 62 | if tenant == "" || namespace == "" { 63 | return errors.Errorf("Invalid tenant or namespace. [%s/%s]", tenant, namespace) 64 | } 65 | 66 | ok := CheckName(tenant) 67 | if !ok { 68 | return errors.Errorf("Tenant name include unsupported special chars. tenant : [%s]", tenant) 69 | } 70 | 71 | ok = CheckName(namespace) 72 | if !ok { 73 | return errors.Errorf("Namespace name include unsupported special chars. namespace : [%s]", namespace) 74 | } 75 | 76 | return nil 77 | } 78 | 79 | // allowed characters for property, namespace, cluster and topic 80 | // names are alphanumeric (a-zA-Z0-9) and these special chars -=:. 81 | // and % is allowed as part of valid URL encoding 82 | var patten = regexp.MustCompile(`^[-=:.\w]*$`) 83 | 84 | func CheckName(name string) bool { 85 | return patten.MatchString(name) 86 | } 87 | -------------------------------------------------------------------------------- /pkg/utils/topic_name_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestGetTopicName(t *testing.T) { 25 | success, err := GetTopicName("success") 26 | assert.Nil(t, err) 27 | assert.Equal(t, "persistent://public/default/success", success.String()) 28 | 29 | success, err = GetTopicName("tenant/namespace/success") 30 | assert.Nil(t, err) 31 | assert.Equal(t, "persistent://tenant/namespace/success", success.String()) 32 | 33 | success, err = GetTopicName("persistent://tenant/namespace/success") 34 | assert.Nil(t, err) 35 | assert.Equal(t, "persistent://tenant/namespace/success", success.String()) 36 | 37 | success, err = GetTopicName("non-persistent://tenant/namespace/success") 38 | assert.Nil(t, err) 39 | assert.Equal(t, "non-persistent://tenant/namespace/success", success.String()) 40 | 41 | _, err = GetTopicName("://tenant.namespace.topic") 42 | assert.NotNil(t, err) 43 | assert.Equal(t, "The domain only can be specified as 'persistent' or 'non-persistent'."+ 44 | " Input domain is ''.", err.Error()) 45 | 46 | fail, err := GetTopicName("default/fail") 47 | assert.NotNil(t, err) 48 | assert.Equal(t, "Invalid short topic name 'default/fail', it should be in the "+ 49 | "format of // or ", err.Error()) 50 | assert.Nil(t, fail) 51 | 52 | fail, err = GetTopicName("domain://tenant/namespace/fail") 53 | assert.NotNil(t, err) 54 | assert.Equal(t, "The domain only can be specified as 'persistent' or 'non-persistent'. "+ 55 | "Input domain is 'domain'.", err.Error()) 56 | assert.Nil(t, fail) 57 | 58 | fail, err = GetTopicName("persistent:///tenant/namespace/fail") 59 | assert.NotNil(t, err) 60 | assert.Equal(t, "Invalid tenant or namespace. [/tenant]", err.Error()) 61 | assert.Nil(t, fail) 62 | 63 | fail, err = GetTopicName("persistent://tenant/namespace") 64 | assert.NotNil(t, err) 65 | assert.Equal(t, "invalid topic name 'tenant/namespace', it should be in the format "+ 66 | "of //", err.Error()) 67 | assert.Nil(t, fail) 68 | 69 | fail, err = GetTopicName("persistent://tenant/namespace/") 70 | assert.NotNil(t, err) 71 | assert.Equal(t, "topic name can not be empty", err.Error()) 72 | assert.Nil(t, fail) 73 | } 74 | -------------------------------------------------------------------------------- /pkg/admin/resource_quotas.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 20 | ) 21 | 22 | type ResourceQuotas interface { 23 | // Get default resource quota for new resource bundles. 24 | GetDefaultResourceQuota() (*utils.ResourceQuota, error) 25 | 26 | // Set default resource quota for new namespace bundles. 27 | SetDefaultResourceQuota(quota utils.ResourceQuota) error 28 | 29 | // Get resource quota of a namespace bundle. 30 | GetNamespaceBundleResourceQuota(namespace, bundle string) (*utils.ResourceQuota, error) 31 | 32 | // Set resource quota for a namespace bundle. 33 | SetNamespaceBundleResourceQuota(namespace, bundle string, quota utils.ResourceQuota) error 34 | 35 | // Reset resource quota for a namespace bundle to default value. 36 | ResetNamespaceBundleResourceQuota(namespace, bundle string) error 37 | } 38 | 39 | type resource struct { 40 | pulsar *pulsarClient 41 | basePath string 42 | } 43 | 44 | func (c *pulsarClient) ResourceQuotas() ResourceQuotas { 45 | return &resource{ 46 | pulsar: c, 47 | basePath: "/resource-quotas", 48 | } 49 | } 50 | 51 | func (r *resource) GetDefaultResourceQuota() (*utils.ResourceQuota, error) { 52 | endpoint := r.pulsar.endpoint(r.basePath) 53 | var quota utils.ResourceQuota 54 | err := r.pulsar.Client.Get(endpoint, "a) 55 | if err != nil { 56 | return nil, err 57 | } 58 | return "a, nil 59 | } 60 | 61 | func (r *resource) SetDefaultResourceQuota(quota utils.ResourceQuota) error { 62 | endpoint := r.pulsar.endpoint(r.basePath) 63 | return r.pulsar.Client.Post(endpoint, "a) 64 | } 65 | 66 | func (r *resource) GetNamespaceBundleResourceQuota(namespace, bundle string) (*utils.ResourceQuota, error) { 67 | endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) 68 | var quota utils.ResourceQuota 69 | err := r.pulsar.Client.Get(endpoint, "a) 70 | if err != nil { 71 | return nil, err 72 | } 73 | return "a, nil 74 | } 75 | 76 | func (r *resource) SetNamespaceBundleResourceQuota(namespace, bundle string, quota utils.ResourceQuota) error { 77 | endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) 78 | return r.pulsar.Client.Post(endpoint, "a) 79 | } 80 | 81 | func (r *resource) ResetNamespaceBundleResourceQuota(namespace, bundle string) error { 82 | endpoint := r.pulsar.endpoint(r.basePath, namespace, bundle) 83 | return r.pulsar.Client.Delete(endpoint) 84 | } 85 | -------------------------------------------------------------------------------- /pkg/utils/package_name_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestGetPackageName(t *testing.T) { 25 | success, err := GetPackageName("function://f-tenant/f-ns/f-name@f-version") 26 | assert.Nil(t, err) 27 | assert.Equal(t, "function://f-tenant/f-ns/f-name@f-version", success.String()) 28 | 29 | success, err = GetPackageName("function://f-tenant/f-ns/f-name") 30 | assert.Nil(t, err) 31 | assert.Equal(t, "function://f-tenant/f-ns/f-name@latest", success.String()) 32 | 33 | success, err = GetPackageName("sink://s-tenant/s-ns/s-name@s-version") 34 | assert.Nil(t, err) 35 | assert.Equal(t, "sink://s-tenant/s-ns/s-name@s-version", success.String()) 36 | 37 | success, err = GetPackageName("sink://s-tenant/s-ns/s-name") 38 | assert.Nil(t, err) 39 | assert.Equal(t, "sink://s-tenant/s-ns/s-name@latest", success.String()) 40 | 41 | success, err = GetPackageName("source://s-tenant/s-ns/s-name@s-version") 42 | assert.Nil(t, err) 43 | assert.Equal(t, "source://s-tenant/s-ns/s-name@s-version", success.String()) 44 | 45 | success, err = GetPackageName("source://s-tenant/s-ns/s-name") 46 | assert.Nil(t, err) 47 | assert.Equal(t, "source://s-tenant/s-ns/s-name@latest", success.String()) 48 | 49 | fail, err := GetPackageName("function:///public/default/test-error@v1") 50 | assert.NotNil(t, err) 51 | assert.Equal(t, "Invalid package name 'function:///public/default/test-error@v1', it should be in the "+ 52 | "format of type://tenant/namespace/name@version", err.Error()) 53 | assert.Nil(t, fail) 54 | 55 | fail, err = GetPackageNameWithComponents("functions", "public", "default", "test-error", "v1") 56 | assert.NotNil(t, err) 57 | assert.Equal(t, "Invalid package type 'functions', it should be function, sink, or source", err.Error()) 58 | assert.Nil(t, fail) 59 | 60 | fail, err = GetPackageNameWithComponents("function", "public/default", "default", "test-error", "v1") 61 | assert.NotNil(t, err) 62 | assert.Equal(t, "Invalid package name 'function://public/default/default/test-error@v1', it should be in the "+ 63 | "format of type://tenant/namespace/name@version", err.Error()) 64 | assert.Nil(t, fail) 65 | 66 | fail, err = GetPackageName("function://public/default/test-error-version/v2") 67 | assert.NotNil(t, err) 68 | assert.Equal(t, "Invalid package name 'function://public/default/test-error-version/v2', it should be in the "+ 69 | "format of type://tenant/namespace/name@version", err.Error()) 70 | assert.Nil(t, fail) 71 | } 72 | -------------------------------------------------------------------------------- /pkg/admin/auth/token.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package auth 17 | 18 | import ( 19 | "encoding/json" 20 | "fmt" 21 | "net/http" 22 | "os" 23 | "strings" 24 | 25 | "github.com/pkg/errors" 26 | ) 27 | 28 | const ( 29 | tokenPrefix = "token:" 30 | filePrefix = "file:" 31 | TokenPluginName = "org.apache.pulsar.client.impl.auth.AuthenticationToken" 32 | TokePluginShortName = "token" 33 | ) 34 | 35 | type Token struct { 36 | Token string `json:"token"` 37 | } 38 | 39 | type TokenAuthProvider struct { 40 | T http.RoundTripper 41 | token string 42 | } 43 | 44 | // NewAuthenticationToken return a interface of Provider with a string token. 45 | func NewAuthenticationToken(token string, transport http.RoundTripper) (*TokenAuthProvider, error) { 46 | if len(token) == 0 { 47 | return nil, errors.New("No token provided") 48 | } 49 | return &TokenAuthProvider{token: token, T: transport}, nil 50 | } 51 | 52 | // NewAuthenticationTokenFromFile return a interface of a Provider with a string token file path. 53 | func NewAuthenticationTokenFromFile(tokenFilePath string, transport http.RoundTripper) (*TokenAuthProvider, error) { 54 | data, err := os.ReadFile(tokenFilePath) 55 | if err != nil { 56 | return nil, err 57 | } 58 | token := strings.Trim(string(data), " \n") 59 | return NewAuthenticationToken(token, transport) 60 | } 61 | 62 | func NewAuthenticationTokenFromAuthParams(encodedAuthParam string, 63 | transport http.RoundTripper) (*TokenAuthProvider, error) { 64 | var tokenAuthProvider *TokenAuthProvider 65 | var err error 66 | 67 | var tokenJSON Token 68 | err = json.Unmarshal([]byte(encodedAuthParam), &tokenJSON) 69 | if err != nil { 70 | switch { 71 | case strings.HasPrefix(encodedAuthParam, tokenPrefix): 72 | tokenAuthProvider, err = NewAuthenticationToken(strings.TrimPrefix(encodedAuthParam, tokenPrefix), transport) 73 | case strings.HasPrefix(encodedAuthParam, filePrefix): 74 | tokenAuthProvider, err = NewAuthenticationTokenFromFile(strings.TrimPrefix(encodedAuthParam, filePrefix), transport) 75 | default: 76 | tokenAuthProvider, err = NewAuthenticationToken(encodedAuthParam, transport) 77 | } 78 | } else { 79 | tokenAuthProvider, err = NewAuthenticationToken(tokenJSON.Token, transport) 80 | } 81 | return tokenAuthProvider, err 82 | } 83 | 84 | func (p *TokenAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { 85 | req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", p.token)) 86 | return p.T.RoundTrip(req) 87 | } 88 | 89 | func (p *TokenAuthProvider) Transport() http.RoundTripper { 90 | return p.T 91 | } 92 | 93 | func (p *TokenAuthProvider) WithTransport(tripper http.RoundTripper) { 94 | p.T = tripper 95 | } 96 | -------------------------------------------------------------------------------- /pkg/utils/ns_isolation_data.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "github.com/pkg/errors" 20 | ) 21 | 22 | type NamespaceIsolationData struct { 23 | Namespaces []string `json:"namespaces"` 24 | Primary []string `json:"primary"` 25 | Secondary []string `json:"secondary"` 26 | AutoFailoverPolicy AutoFailoverPolicyData `json:"auto_failover_policy"` 27 | } 28 | 29 | type AutoFailoverPolicyData struct { 30 | PolicyType AutoFailoverPolicyType `json:"policy_type"` 31 | Parameters map[string]string `json:"parameters"` 32 | } 33 | 34 | type AutoFailoverPolicyType string 35 | 36 | const ( 37 | MinAvailable AutoFailoverPolicyType = "min_available" 38 | ) 39 | 40 | func fromString(autoFailoverPolicyTypeName string) AutoFailoverPolicyType { 41 | switch autoFailoverPolicyTypeName { 42 | case "min_available": 43 | return MinAvailable 44 | default: 45 | return "" 46 | } 47 | } 48 | 49 | func CreateNamespaceIsolationData(namespaces, primary, secondry []string, autoFailoverPolicyTypeName string, 50 | autoFailoverPolicyParams map[string]string) (*NamespaceIsolationData, error) { 51 | nsIsolationData := new(NamespaceIsolationData) 52 | if len(namespaces) == 0 { 53 | return nil, errors.New("unable to parse namespaces parameter list") 54 | } 55 | 56 | if len(primary) == 0 { 57 | return nil, errors.New("unable to parse primary parameter list") 58 | } 59 | 60 | if len(secondry) == 0 { 61 | return nil, errors.New("unable to parse secondry parameter list") 62 | } 63 | 64 | nsIsolationData.Namespaces = namespaces 65 | nsIsolationData.Primary = primary 66 | nsIsolationData.Secondary = secondry 67 | nsIsolationData.AutoFailoverPolicy.PolicyType = fromString(autoFailoverPolicyTypeName) 68 | nsIsolationData.AutoFailoverPolicy.Parameters = autoFailoverPolicyParams 69 | 70 | // validation if necessary 71 | if nsIsolationData.AutoFailoverPolicy.PolicyType == MinAvailable { 72 | err := true 73 | expectParamKeys := []string{"min_limit", "usage_threshold"} 74 | 75 | if len(autoFailoverPolicyParams) == len(expectParamKeys) { 76 | for _, paramKey := range expectParamKeys { 77 | if _, ok := autoFailoverPolicyParams[paramKey]; !ok { 78 | break 79 | } 80 | } 81 | err = false 82 | } 83 | 84 | if err { 85 | return nil, errors.Errorf("Unknown auto failover policy params specified: %v", autoFailoverPolicyParams) 86 | } 87 | } else { 88 | // either we don't handle the new type or user has specified a bad type 89 | return nil, errors.Errorf("Unknown auto failover policy type specified : %v", autoFailoverPolicyTypeName) 90 | } 91 | 92 | return nsIsolationData, nil 93 | } 94 | -------------------------------------------------------------------------------- /pkg/admin/auth/tls.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package auth 17 | 18 | import ( 19 | "crypto/tls" 20 | "encoding/json" 21 | "net/http" 22 | "strings" 23 | ) 24 | 25 | const ( 26 | TLSPluginName = "org.apache.pulsar.client.impl.auth.AuthenticationTls" 27 | TLSPluginShortName = "tls" 28 | ) 29 | 30 | type TLS struct { 31 | TLSCertFile string `json:"tlsCertFile"` 32 | TLSKeyFile string `json:"tlsKeyFile"` 33 | } 34 | 35 | type TLSAuthProvider struct { 36 | certificatePath string 37 | privateKeyPath string 38 | T http.RoundTripper 39 | } 40 | 41 | // NewAuthenticationTLS initialize the authentication provider 42 | func NewAuthenticationTLS(certificatePath string, privateKeyPath string, 43 | transport http.RoundTripper) (*TLSAuthProvider, error) { 44 | provider := &TLSAuthProvider{ 45 | certificatePath: certificatePath, 46 | privateKeyPath: privateKeyPath, 47 | T: transport, 48 | } 49 | if err := provider.configTLS(); err != nil { 50 | return nil, err 51 | } 52 | return provider, nil 53 | } 54 | 55 | func NewAuthenticationTLSFromAuthParams(encodedAuthParams string, 56 | transport http.RoundTripper) (*TLSAuthProvider, error) { 57 | var certificatePath string 58 | var privateKeyPath string 59 | 60 | var tlsJSON TLS 61 | err := json.Unmarshal([]byte(encodedAuthParams), &tlsJSON) 62 | if err != nil { 63 | parts := strings.Split(encodedAuthParams, ",") 64 | for _, part := range parts { 65 | kv := strings.Split(part, ":") 66 | switch kv[0] { 67 | case "tlsCertFile": 68 | certificatePath = kv[1] 69 | case "tlsKeyFile": 70 | privateKeyPath = kv[1] 71 | } 72 | } 73 | } else { 74 | certificatePath = tlsJSON.TLSCertFile 75 | privateKeyPath = tlsJSON.TLSKeyFile 76 | } 77 | 78 | return NewAuthenticationTLS(certificatePath, privateKeyPath, transport) 79 | } 80 | 81 | func (p *TLSAuthProvider) GetTLSCertificate() (*tls.Certificate, error) { 82 | cert, err := tls.LoadX509KeyPair(p.certificatePath, p.privateKeyPath) 83 | return &cert, err 84 | } 85 | 86 | func (p *TLSAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { 87 | return p.T.RoundTrip(req) 88 | } 89 | 90 | func (p *TLSAuthProvider) Transport() http.RoundTripper { 91 | return p.T 92 | } 93 | 94 | func (p *TLSAuthProvider) configTLS() error { 95 | cert, err := p.GetTLSCertificate() 96 | if err != nil { 97 | return err 98 | } 99 | transport := p.T.(*http.Transport) 100 | transport.TLSClientConfig.Certificates = []tls.Certificate{*cert} 101 | return nil 102 | } 103 | 104 | func (p *TLSAuthProvider) WithTransport(tripper http.RoundTripper) { 105 | p.T = tripper 106 | } 107 | -------------------------------------------------------------------------------- /pkg/admin/auth/provider.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package auth 17 | 18 | import ( 19 | "net/http" 20 | 21 | "github.com/apache/pulsar-client-go/oauth2" 22 | 23 | "github.com/streamnative/pulsar-admin-go/pkg/admin/config" 24 | ) 25 | 26 | // Provider provide a general method to add auth message 27 | type Provider interface { 28 | RoundTrip(req *http.Request) (*http.Response, error) 29 | Transport() http.RoundTripper 30 | WithTransport(tripper http.RoundTripper) 31 | } 32 | 33 | type DefaultProvider struct { 34 | transport http.RoundTripper 35 | } 36 | 37 | func NewDefaultProvider(t http.RoundTripper) Provider { 38 | return &DefaultProvider{ 39 | transport: t, 40 | } 41 | } 42 | 43 | func (dp *DefaultProvider) RoundTrip(req *http.Request) (*http.Response, error) { 44 | return dp.transport.RoundTrip(req) 45 | } 46 | 47 | func (dp *DefaultProvider) Transport() http.RoundTripper { 48 | return dp.transport 49 | } 50 | 51 | func (dp *DefaultProvider) WithTransport(t http.RoundTripper) { 52 | dp.transport = t 53 | } 54 | 55 | func GetAuthProvider(config *config.Config) (Provider, error) { 56 | var provider Provider 57 | defaultTransport, err := NewDefaultTransport(config) 58 | if err != nil { 59 | return nil, err 60 | } 61 | switch config.AuthPlugin { 62 | case TLSPluginShortName: 63 | fallthrough 64 | case TLSPluginName: 65 | provider, err = NewAuthenticationTLSFromAuthParams(config.AuthParams, defaultTransport) 66 | case TokenPluginName: 67 | fallthrough 68 | case TokePluginShortName: 69 | provider, err = NewAuthenticationTokenFromAuthParams(config.AuthParams, defaultTransport) 70 | case OAuth2PluginName: 71 | fallthrough 72 | case OAuth2PluginShortName: 73 | provider, err = NewAuthenticationOAuth2WithDefaultFlow(oauth2.Issuer{ 74 | IssuerEndpoint: config.IssuerEndpoint, 75 | ClientID: config.ClientID, 76 | Audience: config.Audience, 77 | }, config.KeyFile) 78 | default: 79 | switch { 80 | case len(config.TLSCertFile) > 0 && len(config.TLSKeyFile) > 0: 81 | provider, err = NewAuthenticationTLS(config.TLSCertFile, config.TLSKeyFile, defaultTransport) 82 | case len(config.Token) > 0: 83 | provider, err = NewAuthenticationToken(config.Token, defaultTransport) 84 | case len(config.TokenFile) > 0: 85 | provider, err = NewAuthenticationTokenFromFile(config.TokenFile, defaultTransport) 86 | case len(config.IssuerEndpoint) > 0 || len(config.ClientID) > 0 || len(config.Audience) > 0 || len(config.Scope) > 0: 87 | provider, err = NewAuthenticationOAuth2WithParams( 88 | config.IssuerEndpoint, config.ClientID, config.Audience, config.Scope, defaultTransport) 89 | default: 90 | provider = NewDefaultProvider(defaultTransport) 91 | } 92 | } 93 | return provider, err 94 | } 95 | -------------------------------------------------------------------------------- /pkg/utils/home_dir.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "os" 20 | "path/filepath" 21 | "runtime" 22 | ) 23 | 24 | // HomeDir returns the home directory for the current user. 25 | // On Windows: 26 | // 1. the first of %HOME%, %HOMEDRIVE%%HOMEPATH%, %USERPROFILE% containing a `.pulsar\config` file is returned. 27 | // 2. if none of those locations contain a `.pulsar\config` file, the first of 28 | // %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% 29 | // that exists and is writeable is returned. 30 | // 3. if none of those locations are writeable, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% 31 | // that exists is returned. 32 | // 4. if none of those locations exists, the first of %HOME%, %USERPROFILE%, %HOMEDRIVE%%HOMEPATH% 33 | // that is set is returned. 34 | func HomeDir() string { 35 | if runtime.GOOS == "windows" { 36 | home := os.Getenv("HOME") 37 | homeDriveHomePath := "" 38 | if homeDrive, homePath := os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH"); len(homeDrive) > 0 && len(homePath) > 0 { 39 | homeDriveHomePath = homeDrive + homePath 40 | } 41 | userProfile := os.Getenv("USERPROFILE") 42 | 43 | // Return first of %HOME%, %HOMEDRIVE%/%HOMEPATH%, %USERPROFILE% that contains a `.pulsar\config` file. 44 | // %HOMEDRIVE%/%HOMEPATH% is preferred over %USERPROFILE% for backwards-compatibility. 45 | for _, p := range []string{home, homeDriveHomePath, userProfile} { 46 | if len(p) == 0 { 47 | continue 48 | } 49 | if _, err := os.Stat(filepath.Join(p, ".pulsar", "config")); err != nil { 50 | continue 51 | } 52 | return p 53 | } 54 | 55 | firstSetPath := "" 56 | firstExistingPath := "" 57 | 58 | // Prefer %USERPROFILE% over %HOMEDRIVE%/%HOMEPATH% for compatibility with other auth-writing tools 59 | for _, p := range []string{home, userProfile, homeDriveHomePath} { 60 | if len(p) == 0 { 61 | continue 62 | } 63 | if len(firstSetPath) == 0 { 64 | // remember the first path that is set 65 | firstSetPath = p 66 | } 67 | info, err := os.Stat(p) 68 | if err != nil { 69 | continue 70 | } 71 | if len(firstExistingPath) == 0 { 72 | // remember the first path that exists 73 | firstExistingPath = p 74 | } 75 | if info.IsDir() && info.Mode().Perm()&(1<<(uint(7))) != 0 { 76 | // return first path that is writeable 77 | return p 78 | } 79 | } 80 | 81 | // If none are writeable, return first location that exists 82 | if len(firstExistingPath) > 0 { 83 | return firstExistingPath 84 | } 85 | 86 | // If none exist, return first location that is set 87 | if len(firstSetPath) > 0 { 88 | return firstSetPath 89 | } 90 | 91 | // We've got nothing 92 | return "" 93 | } 94 | return os.Getenv("HOME") 95 | } 96 | -------------------------------------------------------------------------------- /pkg/admin/functions_worker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 20 | ) 21 | 22 | type FunctionsWorker interface { 23 | // Get all functions stats on a worker 24 | GetFunctionsStats() ([]*utils.WorkerFunctionInstanceStats, error) 25 | 26 | // Get worker metrics 27 | GetMetrics() ([]*utils.Metrics, error) 28 | 29 | // Get List of all workers belonging to this cluster 30 | GetCluster() ([]*utils.WorkerInfo, error) 31 | 32 | // Get the worker who is the leader of the clusterv 33 | GetClusterLeader() (*utils.WorkerInfo, error) 34 | 35 | // Get the function assignment among the cluster 36 | GetAssignments() (map[string][]string, error) 37 | } 38 | 39 | type worker struct { 40 | pulsar *pulsarClient 41 | workerPath string 42 | workerStatsPath string 43 | } 44 | 45 | func (c *pulsarClient) FunctionsWorker() FunctionsWorker { 46 | return &worker{ 47 | pulsar: c, 48 | workerPath: "/worker", 49 | workerStatsPath: "/worker-stats", 50 | } 51 | } 52 | 53 | func (w *worker) GetFunctionsStats() ([]*utils.WorkerFunctionInstanceStats, error) { 54 | endpoint := w.pulsar.endpoint(w.workerStatsPath, "functionsmetrics") 55 | var workerStats []*utils.WorkerFunctionInstanceStats 56 | err := w.pulsar.Client.Get(endpoint, &workerStats) 57 | if err != nil { 58 | return nil, err 59 | } 60 | return workerStats, nil 61 | } 62 | 63 | func (w *worker) GetMetrics() ([]*utils.Metrics, error) { 64 | endpoint := w.pulsar.endpoint(w.workerStatsPath, "metrics") 65 | var metrics []*utils.Metrics 66 | err := w.pulsar.Client.Get(endpoint, &metrics) 67 | if err != nil { 68 | return nil, err 69 | } 70 | return metrics, nil 71 | } 72 | 73 | func (w *worker) GetCluster() ([]*utils.WorkerInfo, error) { 74 | endpoint := w.pulsar.endpoint(w.workerPath, "cluster") 75 | var workersInfo []*utils.WorkerInfo 76 | err := w.pulsar.Client.Get(endpoint, &workersInfo) 77 | if err != nil { 78 | return nil, err 79 | } 80 | return workersInfo, nil 81 | } 82 | 83 | func (w *worker) GetClusterLeader() (*utils.WorkerInfo, error) { 84 | endpoint := w.pulsar.endpoint(w.workerPath, "cluster", "leader") 85 | var workerInfo utils.WorkerInfo 86 | err := w.pulsar.Client.Get(endpoint, &workerInfo) 87 | if err != nil { 88 | return nil, err 89 | } 90 | return &workerInfo, nil 91 | } 92 | 93 | func (w *worker) GetAssignments() (map[string][]string, error) { 94 | endpoint := w.pulsar.endpoint(w.workerPath, "assignments") 95 | var assignments map[string][]string 96 | err := w.pulsar.Client.Get(endpoint, &assignments) 97 | if err != nil { 98 | return nil, err 99 | } 100 | return assignments, nil 101 | } 102 | -------------------------------------------------------------------------------- /pkg/admin/broker_stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 20 | ) 21 | 22 | // BrokerStats is admin interface for broker stats management 23 | type BrokerStats interface { 24 | // GetMetrics returns Monitoring metrics 25 | GetMetrics() ([]utils.Metrics, error) 26 | 27 | // GetMBeans requests JSON string server mbean dump 28 | GetMBeans() ([]utils.Metrics, error) 29 | 30 | // GetTopics returns JSON string topics stats 31 | GetTopics() (string, error) 32 | 33 | // GetLoadReport returns load report of broker 34 | GetLoadReport() (*utils.LocalBrokerData, error) 35 | 36 | // GetAllocatorStats returns stats from broker 37 | GetAllocatorStats(allocatorName string) (*utils.AllocatorStats, error) 38 | } 39 | 40 | type brokerStats struct { 41 | pulsar *pulsarClient 42 | basePath string 43 | } 44 | 45 | // BrokerStats is used to access the broker stats endpoints 46 | func (c *pulsarClient) BrokerStats() BrokerStats { 47 | return &brokerStats{ 48 | pulsar: c, 49 | basePath: "/broker-stats", 50 | } 51 | } 52 | 53 | func (bs *brokerStats) GetMetrics() ([]utils.Metrics, error) { 54 | endpoint := bs.pulsar.endpoint(bs.basePath, "/metrics") 55 | var response []utils.Metrics 56 | err := bs.pulsar.Client.Get(endpoint, &response) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | return response, nil 62 | } 63 | 64 | func (bs *brokerStats) GetMBeans() ([]utils.Metrics, error) { 65 | endpoint := bs.pulsar.endpoint(bs.basePath, "/mbeans") 66 | var response []utils.Metrics 67 | err := bs.pulsar.Client.Get(endpoint, &response) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | return response, nil 73 | } 74 | 75 | func (bs *brokerStats) GetTopics() (string, error) { 76 | endpoint := bs.pulsar.endpoint(bs.basePath, "/topics") 77 | buf, err := bs.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) 78 | if err != nil { 79 | return "", err 80 | } 81 | 82 | return string(buf), nil 83 | } 84 | 85 | func (bs *brokerStats) GetLoadReport() (*utils.LocalBrokerData, error) { 86 | endpoint := bs.pulsar.endpoint(bs.basePath, "/load-report") 87 | response := utils.NewLocalBrokerData() 88 | err := bs.pulsar.Client.Get(endpoint, &response) 89 | if err != nil { 90 | return nil, nil 91 | } 92 | return &response, nil 93 | } 94 | 95 | func (bs *brokerStats) GetAllocatorStats(allocatorName string) (*utils.AllocatorStats, error) { 96 | endpoint := bs.pulsar.endpoint(bs.basePath, "/allocator-stats", allocatorName) 97 | var allocatorStats utils.AllocatorStats 98 | err := bs.pulsar.Client.Get(endpoint, &allocatorStats) 99 | if err != nil { 100 | return nil, err 101 | } 102 | return &allocatorStats, nil 103 | } 104 | -------------------------------------------------------------------------------- /pkg/utils/allocator_stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type AllocatorStats struct { 19 | NumDirectArenas int `json:"numDirectArenas"` 20 | NumHeapArenas int `json:"numHeapArenas"` 21 | NumThreadLocalCaches int `json:"numThreadLocalCaches"` 22 | NormalCacheSize int `json:"normalCacheSize"` 23 | SmallCacheSize int `json:"smallCacheSize"` 24 | TinyCacheSize int `json:"tinyCacheSize"` 25 | DirectArenas []PoolArenaStats `json:"directArenas"` 26 | HeapArenas []PoolArenaStats `json:"heapArenas"` 27 | } 28 | 29 | type PoolArenaStats struct { 30 | NumTinySubpages int `json:"numTinySubpages"` 31 | NumSmallSubpages int `json:"numSmallSubpages"` 32 | NumChunkLists int `json:"numChunkLists"` 33 | TinySubpages []PoolSubpageStats `json:"tinySubpages"` 34 | SmallSubpages []PoolSubpageStats `json:"smallSubpages"` 35 | ChunkLists []PoolChunkListStats `json:"chunkLists"` 36 | NumAllocations int64 `json:"numAllocations"` 37 | NumTinyAllocations int64 `json:"numTinyAllocations"` 38 | NumSmallAllocations int64 `json:"numSmallAllocations"` 39 | NumNormalAllocations int64 `json:"numNormalAllocations"` 40 | NumHugeAllocations int64 `json:"numHugeAllocations"` 41 | NumDeallocations int64 `json:"numDeallocations"` 42 | NumTinyDeallocations int64 `json:"numTinyDeallocations"` 43 | NumSmallDeallocations int64 `json:"numSmallDeallocations"` 44 | NumNormalDeallocations int64 `json:"numNormalDeallocations"` 45 | NumHugeDeallocations int64 `json:"numHugeDeallocations"` 46 | NumActiveAllocations int64 `json:"numActiveAllocations"` 47 | NumActiveTinyAllocations int64 `json:"numActiveTinyAllocations"` 48 | NumActiveSmallAllocations int64 `json:"numActiveSmallAllocations"` 49 | NumActiveNormalAllocations int64 `json:"numActiveNormalAllocations"` 50 | NumActiveHugeAllocations int64 `json:"numActiveHugeAllocations"` 51 | } 52 | 53 | type PoolSubpageStats struct { 54 | MaxNumElements int `json:"maxNumElements"` 55 | NumAvailable int `json:"numAvailable"` 56 | ElementSize int `json:"elementSize"` 57 | PageSize int `json:"pageSize"` 58 | } 59 | 60 | type PoolChunkListStats struct { 61 | MinUsage int `json:"minUsage"` 62 | MaxUsage int `json:"maxUsage"` 63 | Chunks []PoolChunkStats `json:"chunks"` 64 | } 65 | 66 | type PoolChunkStats struct { 67 | Usage int `json:"usage"` 68 | ChunkSize int `json:"chunkSize"` 69 | FreeBytes int `json:"freeBytes"` 70 | } 71 | -------------------------------------------------------------------------------- /pkg/admin/admin.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "net/http" 20 | "net/url" 21 | "path" 22 | "time" 23 | 24 | "github.com/streamnative/pulsar-admin-go/pkg/admin/auth" 25 | "github.com/streamnative/pulsar-admin-go/pkg/admin/config" 26 | "github.com/streamnative/pulsar-admin-go/pkg/rest" 27 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 28 | ) 29 | 30 | const ( 31 | DefaultWebServiceURL = "http://localhost:8080" 32 | DefaultHTTPTimeOutDuration = 5 * time.Minute 33 | ReleaseVersion = "None" 34 | ) 35 | 36 | type TLSOptions struct { 37 | TrustCertsFilePath string 38 | AllowInsecureConnection bool 39 | } 40 | 41 | // Client provides a client to the Pulsar Restful API 42 | type Client interface { 43 | Clusters() Clusters 44 | Functions() Functions 45 | Tenants() Tenants 46 | Topics() Topics 47 | Subscriptions() Subscriptions 48 | Sources() Sources 49 | Sinks() Sinks 50 | Namespaces() Namespaces 51 | Schemas() Schema 52 | NsIsolationPolicy() NsIsolationPolicy 53 | Brokers() Brokers 54 | BrokerStats() BrokerStats 55 | ResourceQuotas() ResourceQuotas 56 | FunctionsWorker() FunctionsWorker 57 | Packages() Packages 58 | } 59 | 60 | type pulsarClient struct { 61 | Client *rest.Client 62 | APIVersion config.APIVersion 63 | } 64 | 65 | // New returns a new client 66 | func New(config *config.Config) (Client, error) { 67 | authProvider, err := auth.GetAuthProvider(config) 68 | if err != nil { 69 | return nil, err 70 | } 71 | return NewPulsarClientWithAuthProvider(config, authProvider) 72 | } 73 | 74 | // NewWithAuthProvider creates a client with auth provider. 75 | // Deprecated: Use NewPulsarClientWithAuthProvider instead. 76 | func NewWithAuthProvider(config *config.Config, authProvider auth.Provider) Client { 77 | client, err := NewPulsarClientWithAuthProvider(config, authProvider) 78 | if err != nil { 79 | panic(err) 80 | } 81 | return client 82 | } 83 | 84 | // NewPulsarClientWithAuthProvider create a client with auth provider. 85 | func NewPulsarClientWithAuthProvider(config *config.Config, authProvider auth.Provider) (Client, error) { 86 | if len(config.WebServiceURL) == 0 { 87 | config.WebServiceURL = DefaultWebServiceURL 88 | } 89 | 90 | return &pulsarClient{ 91 | APIVersion: config.PulsarAPIVersion, 92 | Client: &rest.Client{ 93 | ServiceURL: config.WebServiceURL, 94 | VersionInfo: ReleaseVersion, 95 | HTTPClient: &http.Client{ 96 | Timeout: DefaultHTTPTimeOutDuration, 97 | Transport: authProvider, 98 | }, 99 | }, 100 | }, nil 101 | } 102 | 103 | func (c *pulsarClient) endpoint(componentPath string, parts ...string) string { 104 | escapedParts := make([]string, len(parts)) 105 | for i, part := range parts { 106 | escapedParts[i] = url.PathEscape(part) 107 | } 108 | return path.Join( 109 | utils.MakeHTTPPath(c.APIVersion.String(), componentPath), 110 | path.Join(escapedParts...), 111 | ) 112 | } 113 | -------------------------------------------------------------------------------- /pkg/admin/admin_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "net/http" 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | "github.com/stretchr/testify/require" 24 | 25 | "github.com/streamnative/pulsar-admin-go/pkg/admin/auth" 26 | "github.com/streamnative/pulsar-admin-go/pkg/admin/config" 27 | ) 28 | 29 | func TestPulsarClientEndpointEscapes(t *testing.T) { 30 | client := pulsarClient{Client: nil, APIVersion: config.V2} 31 | actual := client.endpoint("/myendpoint", "abc%? /def", "ghi") 32 | expected := "/admin/v2/myendpoint/abc%25%3F%20%2Fdef/ghi" 33 | assert.Equal(t, expected, actual) 34 | } 35 | 36 | func TestNew(t *testing.T) { 37 | config := &config.Config{} 38 | admin, err := New(config) 39 | require.NoError(t, err) 40 | require.NotNil(t, admin) 41 | } 42 | 43 | func TestNewWithAuthProvider(t *testing.T) { 44 | config := &config.Config{} 45 | 46 | tokenAuth, err := auth.NewAuthenticationToken("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9."+ 47 | "eyJzdWIiOiJhZG1pbiIsImlhdCI6MTUxNjIzOTAyMn0.sVt6cyu3HKd89LcQvZVMNbqT0DTl3FvG9oYbj8hBDqU", nil) 48 | require.NoError(t, err) 49 | require.NotNil(t, tokenAuth) 50 | 51 | admin, err := NewPulsarClientWithAuthProvider(config, tokenAuth) 52 | require.NoError(t, err) 53 | require.NotNil(t, admin) 54 | } 55 | 56 | type customAuthProvider struct { 57 | transport http.RoundTripper 58 | } 59 | 60 | var _ auth.Provider = &customAuthProvider{} 61 | 62 | func (c *customAuthProvider) RoundTrip(req *http.Request) (*http.Response, error) { 63 | panic("implement me") 64 | } 65 | 66 | func (c *customAuthProvider) Transport() http.RoundTripper { 67 | return c.transport 68 | } 69 | 70 | func (c *customAuthProvider) WithTransport(transport http.RoundTripper) { 71 | c.transport = transport 72 | } 73 | 74 | func TestNewWithCustomAuthProviderWithTransport(t *testing.T) { 75 | config := &config.Config{} 76 | defaultTransport, err := auth.NewDefaultTransport(config) 77 | require.NoError(t, err) 78 | 79 | customAuthProvider := &customAuthProvider{ 80 | transport: defaultTransport, 81 | } 82 | 83 | admin, err := NewPulsarClientWithAuthProvider(config, customAuthProvider) 84 | require.NoError(t, err) 85 | require.NotNil(t, admin) 86 | 87 | // Expected the customAuthProvider will not be overwritten. 88 | require.Equal(t, customAuthProvider, admin.(*pulsarClient).Client.HTTPClient.Transport) 89 | } 90 | 91 | func TestNewWithTlsAllowInsecure(t *testing.T) { 92 | config := &config.Config{ 93 | TLSAllowInsecureConnection: true, 94 | } 95 | admin, err := New(config) 96 | require.NoError(t, err) 97 | require.NotNil(t, admin) 98 | 99 | pulsarClientS := admin.(*pulsarClient) 100 | require.NotNil(t, pulsarClientS.Client.HTTPClient.Transport) 101 | 102 | ap := pulsarClientS.Client.HTTPClient.Transport.(*auth.DefaultProvider) 103 | tr := ap.Transport().(*http.Transport) 104 | require.NotNil(t, tr) 105 | require.NotNil(t, tr.TLSClientConfig) 106 | require.True(t, tr.TLSClientConfig.InsecureSkipVerify) 107 | } 108 | -------------------------------------------------------------------------------- /pkg/utils/message.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | //nolint 19 | import ( 20 | "github.com/golang/protobuf/proto" 21 | ) 22 | 23 | type Message struct { 24 | MessageID MessageID 25 | Payload []byte 26 | Topic string 27 | Properties map[string]string 28 | } 29 | 30 | func NewMessage(topic string, id MessageID, payload []byte, properties map[string]string) *Message { 31 | return &Message{ 32 | MessageID: id, 33 | Payload: payload, 34 | Topic: topic, 35 | Properties: properties, 36 | } 37 | } 38 | 39 | func (m *Message) GetMessageID() MessageID { 40 | return m.MessageID 41 | } 42 | 43 | func (m *Message) GetProperties() map[string]string { 44 | return m.Properties 45 | } 46 | 47 | func (m *Message) GetPayload() []byte { 48 | return m.Payload 49 | } 50 | 51 | // nolint 52 | type SingleMessageMetadata struct { 53 | Properties []*KeyValue `protobuf:"bytes,1,rep,name=properties" json:"properties,omitempty"` 54 | PartitionKey *string `protobuf:"bytes,2,opt,name=partition_key,json=partitionKey" json:"partition_key,omitempty"` 55 | PayloadSize *int32 `protobuf:"varint,3,req,name=payload_size,json=payloadSize" json:"payload_size,omitempty"` 56 | CompactedOut *bool `protobuf:"varint,4,opt,name=compacted_out,json=compactedOut,def=0" json:"compacted_out,omitempty"` 57 | // the timestamp that this event occurs. it is typically set by applications. 58 | // if this field is omitted, `publish_time` can be used for the purpose of `event_time`. 59 | EventTime *uint64 `protobuf:"varint,5,opt,name=event_time,json=eventTime,def=0" json:"event_time,omitempty"` 60 | PartitionKeyB64Encoded *bool `protobuf:"varint,6,opt,name=partition_key_b64_encoded,json=partitionKeyB64Encoded,def=0" json:"partition_key_b64_encoded,omitempty"` 61 | // Specific a key to overwrite the message key which used for ordering dispatch in Key_Shared mode. 62 | OrderingKey []byte `protobuf:"bytes,7,opt,name=ordering_key,json=orderingKey" json:"ordering_key,omitempty"` 63 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 64 | XXX_unrecognized []byte `json:"-"` 65 | XXX_sizecache int32 `json:"-"` 66 | } 67 | 68 | func (m *SingleMessageMetadata) Reset() { *m = SingleMessageMetadata{} } 69 | func (m *SingleMessageMetadata) String() string { return proto.CompactTextString(m) } 70 | func (*SingleMessageMetadata) ProtoMessage() {} 71 | func (m *SingleMessageMetadata) GetPayloadSize() int32 { 72 | if m != nil && m.PayloadSize != nil { 73 | return *m.PayloadSize 74 | } 75 | return 0 76 | } 77 | 78 | // nolint 79 | type KeyValue struct { 80 | Key *string `protobuf:"bytes,1,req,name=key" json:"key,omitempty"` 81 | Value *string `protobuf:"bytes,2,req,name=value" json:"value,omitempty"` 82 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 83 | XXX_unrecognized []byte `json:"-"` 84 | XXX_sizecache int32 `json:"-"` 85 | } 86 | 87 | func (m *KeyValue) Reset() { *m = KeyValue{} } 88 | func (m *KeyValue) String() string { return proto.CompactTextString(m) } 89 | func (*KeyValue) ProtoMessage() {} 90 | -------------------------------------------------------------------------------- /pkg/utils/package_name.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "fmt" 20 | "strings" 21 | 22 | "github.com/pkg/errors" 23 | ) 24 | 25 | type PackageName struct { 26 | packageType PackageType 27 | namespace string 28 | tenant string 29 | name string 30 | version string 31 | completePackageName string 32 | completeName string 33 | } 34 | 35 | func invalidPackageNameError(completeName string) error { 36 | return errors.Errorf("Invalid package name '%s', it should be "+ 37 | "in the format of type://tenant/namespace/name@version", completeName) 38 | } 39 | 40 | func GetPackageNameWithComponents(packageType PackageType, 41 | tenant, namespace, name, version string) (*PackageName, error) { 42 | return GetPackageName(fmt.Sprintf("%s://%s/%s/%s@%s", packageType, tenant, namespace, name, version)) 43 | } 44 | 45 | func GetPackageName(completeName string) (*PackageName, error) { 46 | var packageName PackageName 47 | var err error 48 | if !strings.Contains(completeName, "://") { 49 | return nil, invalidPackageNameError(completeName) 50 | } 51 | parts := strings.Split(completeName, "://") 52 | if len(parts) != 2 { 53 | return nil, invalidPackageNameError(completeName) 54 | } 55 | packageName.packageType, err = parsePackageType(parts[0]) 56 | if err != nil { 57 | return nil, err 58 | } 59 | rest := parts[1] 60 | if !strings.Contains(rest, "@") { 61 | // if the package name does not contains '@', that means user does not set the version of package. 62 | // We will set the version to latest. 63 | rest += "@" 64 | } 65 | parts = strings.Split(rest, "@") 66 | if len(parts) != 2 { 67 | return nil, invalidPackageNameError(completeName) 68 | } 69 | partsWithoutVersion := strings.Split(parts[0], "/") 70 | if len(partsWithoutVersion) != 3 { 71 | return nil, invalidPackageNameError(completeName) 72 | } 73 | packageName.tenant = partsWithoutVersion[0] 74 | packageName.namespace = partsWithoutVersion[1] 75 | packageName.name = partsWithoutVersion[2] 76 | packageName.version = "latest" 77 | if parts[1] != "" { 78 | packageName.version = parts[1] 79 | } 80 | packageName.completeName = fmt.Sprintf("%s/%s/%s", 81 | packageName.tenant, packageName.namespace, packageName.name) 82 | packageName.completePackageName = fmt.Sprintf("%s://%s/%s/%s@%s", 83 | packageName.packageType, packageName.tenant, packageName.namespace, packageName.name, packageName.version) 84 | 85 | return &packageName, nil 86 | } 87 | 88 | func (p *PackageName) String() string { 89 | return p.completePackageName 90 | } 91 | 92 | func (p *PackageName) GetType() PackageType { 93 | return p.packageType 94 | } 95 | 96 | func (p *PackageName) GetTenant() string { 97 | return p.tenant 98 | } 99 | 100 | func (p *PackageName) GetNamespace() string { 101 | return p.namespace 102 | } 103 | 104 | func (p *PackageName) GetName() string { 105 | return p.name 106 | } 107 | 108 | func (p *PackageName) GetVersion() string { 109 | return p.version 110 | } 111 | 112 | func (p *PackageName) GetCompleteName() string { 113 | return p.completeName 114 | } 115 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 19 | 20 | # Contributing guidelines 21 | 22 | ## Project structure 23 | The overall project structure is illustrated below: 24 | 25 | ```shell 26 | ├── pkg/ 27 | │   ├── admin/ 28 | │   │   ├── auth/ 29 | │   │   ├── config/ 30 | │   ├── rest/ 31 | │   └── utils/ 32 | ├── alias.go 33 | ├── go.mod 34 | └── go.sum 35 | ``` 36 | 37 | - The `alias.go` file in the root defines `pulsaradmin` package scope, which contains shortcuts of some types and functions from the `pkg`. 38 | - The `pkg/admin` package contains all operations for pulsar admin resources. *Note: We should add a new file here if we wanna support a new resource.* 39 | - The `pkg/admin/config` package contains configuration options for constructing a pulsar admin client. 40 | - The `pkg/admin/auth` package contains auth providers which work in transport layer. 41 | - The `pkg/rest` package contains a wrapped HTTP client for requesting pulsar REST API. 42 | - The `pkg/utils` package contains common data structures and functions. 43 | 44 | ## Contributing steps 45 | 1. Submit an issue describing your proposed change. 46 | 2. Discuss and wait for proposal to be accepted. 47 | 3. Fork this repo, develop and test your code changes. 48 | 4. Submit a pull request. 49 | 50 | ## Conventions 51 | 52 | Please read through below conventions before contributions. 53 | 54 | ### PullRequest conventions 55 | 56 | - Use [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) to standardize PR title. 57 | 58 | ### Code conventions 59 | 60 | - [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) 61 | - [Effective Go](https://golang.org/doc/effective_go.html) 62 | - Know and avoid [Go landmines](https://gist.github.com/lavalamp/4bd23295a9f32706a48f) 63 | - Commenting 64 | - [Go's commenting conventions](http://blog.golang.org/godoc-documenting-go-code) 65 | - If reviewers ask questions about why the code is the way it is, that's a sign that comments might be helpful. 66 | - Naming 67 | - Please consider package name when selecting an interface name, and avoid redundancy. For example, `storage.Interface` is better than `storage.StorageInterface`. 68 | - Do not use uppercase characters, underscores, or dashes in package names. 69 | - Please consider parent directory name when choosing a package name. For example, `pkg/controllers/autoscaler/foo.go` should say `package autoscaler` not `package autoscalercontroller`. 70 | - Unless there's a good reason, the `package foo` line should match the name of the directory in which the `.go` file exists. 71 | - Importers can use a different name if they need to disambiguate. 72 | - Locks should be called `lock` and should never be embedded (always `lock sync.Mutex`). When multiple locks are present, give each lock a distinct name following Go conventions: `stateLock`, `mapLock` etc. 73 | 74 | ### Folder and file conventions 75 | 76 | - All filenames should be lowercase. 77 | - Go source files and directories use underscores, not dashes. 78 | - Package directories should generally avoid using separators as much as possible. When package names are multiple words, they usually should be in nested subdirectories. 79 | - Document directories and filenames should use dashes rather than underscores. 80 | - All source files should add a license at the beginning. -------------------------------------------------------------------------------- /pkg/utils/sink_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type SinkConfig struct { 19 | TopicsPattern *string `json:"topicsPattern,omitempty" yaml:"topicsPattern"` 20 | Resources *Resources `json:"resources,omitempty" yaml:"resources"` 21 | TimeoutMs *int64 `json:"timeoutMs,omitempty" yaml:"timeoutMs"` 22 | 23 | // Whether the subscriptions the functions created/used should be deleted when the functions is deleted 24 | CleanupSubscription bool `json:"cleanupSubscription" yaml:"cleanupSubscription"` 25 | 26 | RetainOrdering bool `json:"retainOrdering" yaml:"retainOrdering"` 27 | RetainKeyOrdering bool `json:"retainKeyOrdering" yaml:"retainKeyOrdering"` 28 | AutoAck bool `json:"autoAck" yaml:"autoAck"` 29 | Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"` 30 | Tenant string `json:"tenant,omitempty" yaml:"tenant"` 31 | Namespace string `json:"namespace,omitempty" yaml:"namespace"` 32 | Name string `json:"name,omitempty" yaml:"name"` 33 | ClassName string `json:"className,omitempty" yaml:"className"` 34 | 35 | SinkType string `json:"sinkType,omitempty" yaml:"sinkType"` 36 | Archive string `json:"archive,omitempty" yaml:"archive"` 37 | ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"` 38 | SourceSubscriptionName string `json:"sourceSubscriptionName,omitempty" yaml:"sourceSubscriptionName"` 39 | SourceSubscriptionPosition string `json:"sourceSubscriptionPosition,omitempty" yaml:"sourceSubscriptionPosition"` 40 | RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"` 41 | 42 | Inputs []string `json:"inputs,omitempty" yaml:"inputs"` 43 | TopicToSerdeClassName map[string]string `json:"topicToSerdeClassName,omitempty" yaml:"topicToSerdeClassName"` 44 | TopicToSchemaType map[string]string `json:"topicToSchemaType,omitempty" yaml:"topicToSchemaType"` 45 | InputSpecs map[string]ConsumerConfig `json:"inputSpecs,omitempty" yaml:"inputSpecs"` 46 | Configs map[string]interface{} `json:"configs,omitempty" yaml:"configs"` 47 | 48 | TopicToSchemaProperties map[string]string `json:"topicToSchemaProperties,omitempty" yaml:"topicToSchemaProperties"` 49 | 50 | CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"` 51 | 52 | // This is a map of secretName(aka how the secret is going to be 53 | // accessed in the function via context) to an object that 54 | // encapsulates how the secret is fetched by the underlying 55 | // secrets provider. The type of an value here can be found by the 56 | // SecretProviderConfigurator.getSecretObjectType() method. 57 | Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"` 58 | 59 | MaxMessageRetries int `json:"maxMessageRetries,omitempty" yaml:"maxMessageRetries"` 60 | DeadLetterTopic string `json:"deadLetterTopic,omitempty" yaml:"deadLetterTopic"` 61 | NegativeAckRedeliveryDelayMs int64 `json:"negativeAckRedeliveryDelayMs,omitempty" yaml:"negativeAckRedeliveryDelayMs"` 62 | TransformFunction string `json:"transformFunction,omitempty" yaml:"transformFunction"` 63 | TransformFunctionClassName string `json:"transformFunctionClassName,omitempty" yaml:"transformFunctionClassName"` 64 | TransformFunctionConfig string `json:"transformFunctionConfig,omitempty" yaml:"transformFunctionConfig"` 65 | } 66 | -------------------------------------------------------------------------------- /pkg/admin/auth/oauth2_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package auth 17 | 18 | import ( 19 | "fmt" 20 | "net/http" 21 | "net/http/httptest" 22 | "os" 23 | "testing" 24 | 25 | "github.com/apache/pulsar-client-go/oauth2" 26 | "github.com/apache/pulsar-client-go/oauth2/store" 27 | "github.com/pkg/errors" 28 | "github.com/stretchr/testify/assert" 29 | ) 30 | 31 | // mockOAuthServer will mock a oauth service for the tests 32 | func mockOAuthServer() *httptest.Server { 33 | // prepare a port for the mocked server 34 | server := httptest.NewUnstartedServer(http.DefaultServeMux) 35 | 36 | // mock the used REST path for the tests 37 | mockedHandler := http.NewServeMux() 38 | mockedHandler.HandleFunc("/.well-known/openid-configuration", func(writer http.ResponseWriter, request *http.Request) { 39 | s := fmt.Sprintf(`{ 40 | "issuer":"%s", 41 | "authorization_endpoint":"%s/authorize", 42 | "token_endpoint":"%s/oauth/token", 43 | "device_authorization_endpoint":"%s/oauth/device/code" 44 | }`, server.URL, server.URL, server.URL, server.URL) 45 | fmt.Fprintln(writer, s) 46 | }) 47 | mockedHandler.HandleFunc("/oauth/token", func(writer http.ResponseWriter, request *http.Request) { 48 | fmt.Fprintln(writer, "{\n \"access_token\": \"token-content\",\n \"token_type\": \"Bearer\"\n}") 49 | }) 50 | mockedHandler.HandleFunc("/authorize", func(writer http.ResponseWriter, request *http.Request) { 51 | fmt.Fprintln(writer, "true") 52 | }) 53 | 54 | server.Config.Handler = mockedHandler 55 | server.Start() 56 | 57 | return server 58 | } 59 | 60 | // mockKeyFile will mock a temp key file for testing. 61 | func mockKeyFile(server string) (string, error) { 62 | pwd, err := os.Getwd() 63 | if err != nil { 64 | return "", err 65 | } 66 | kf, err := os.CreateTemp(pwd, "test_oauth2") 67 | if err != nil { 68 | return "", err 69 | } 70 | _, err = kf.WriteString(fmt.Sprintf(`{ 71 | "type":"sn_service_account", 72 | "client_id":"client-id", 73 | "client_secret":"client-secret", 74 | "client_email":"oauth@test.org", 75 | "issuer_url":"%s" 76 | }`, server)) 77 | 78 | if err != nil { 79 | return "", err 80 | } 81 | 82 | return kf.Name(), nil 83 | } 84 | 85 | func TestOauth2(t *testing.T) { 86 | server := mockOAuthServer() 87 | defer server.Close() 88 | kf, err := mockKeyFile(server.URL) 89 | defer os.Remove(kf) 90 | if err != nil { 91 | t.Fatal(errors.Wrap(err, "create mocked key file failed")) 92 | } 93 | 94 | issuer := oauth2.Issuer{ 95 | IssuerEndpoint: server.URL, 96 | ClientID: "client-id", 97 | Audience: server.URL, 98 | } 99 | 100 | memoryStore := store.NewMemoryStore() 101 | err = saveGrant(memoryStore, kf, issuer.Audience) 102 | if err != nil { 103 | t.Fatal(err) 104 | } 105 | 106 | auth, err := NewAuthenticationOAuth2(issuer, memoryStore) 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | 111 | token, err := auth.source.Token() 112 | if err != nil { 113 | t.Fatal(err) 114 | } 115 | assert.Equal(t, "token-content", token.AccessToken) 116 | } 117 | 118 | func saveGrant(store store.Store, keyFile, audience string) error { 119 | flow, err := oauth2.NewDefaultClientCredentialsFlow(oauth2.ClientCredentialsFlowOptions{ 120 | KeyFile: keyFile, 121 | AdditionalScopes: nil, 122 | }) 123 | if err != nil { 124 | return err 125 | } 126 | 127 | grant, err := flow.Authorize(audience) 128 | if err != nil { 129 | return err 130 | } 131 | 132 | return store.SaveGrant(audience, *grant) 133 | } 134 | -------------------------------------------------------------------------------- /pkg/utils/policies.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | const ( 19 | FirstBoundary string = "0x00000000" 20 | LastBoundary string = "0xffffffff" 21 | ) 22 | 23 | type Policies struct { 24 | Bundles *BundlesData `json:"bundles"` 25 | Persistence *PersistencePolicies `json:"persistence"` 26 | RetentionPolicies *RetentionPolicies `json:"retention_policies"` 27 | SchemaValidationEnforced bool `json:"schema_validation_enforced"` 28 | DeduplicationEnabled *bool `json:"deduplicationEnabled"` 29 | Deleted bool `json:"deleted"` 30 | EncryptionRequired bool `json:"encryption_required"` 31 | MessageTTLInSeconds *int `json:"message_ttl_in_seconds"` 32 | MaxProducersPerTopic *int `json:"max_producers_per_topic"` 33 | MaxConsumersPerTopic *int `json:"max_consumers_per_topic"` 34 | MaxConsumersPerSubscription *int `json:"max_consumers_per_subscription"` 35 | CompactionThreshold *int64 `json:"compaction_threshold"` 36 | OffloadThreshold int64 `json:"offload_threshold"` 37 | OffloadDeletionLagMs *int64 `json:"offload_deletion_lag_ms"` 38 | AntiAffinityGroup string `json:"antiAffinityGroup"` 39 | ReplicationClusters []string `json:"replication_clusters"` 40 | LatencyStatsSampleRate map[string]int `json:"latency_stats_sample_rate"` 41 | BacklogQuotaMap map[BacklogQuotaType]BacklogQuota `json:"backlog_quota_map"` 42 | TopicDispatchRate map[string]DispatchRate `json:"topicDispatchRate"` 43 | SubscriptionDispatchRate map[string]DispatchRate `json:"subscriptionDispatchRate"` 44 | ReplicatorDispatchRate map[string]DispatchRate `json:"replicatorDispatchRate"` 45 | PublishMaxMessageRate map[string]PublishRate `json:"publishMaxMessageRate"` 46 | ClusterSubscribeRate map[string]SubscribeRate `json:"clusterSubscribeRate"` 47 | TopicAutoCreationConfig *TopicAutoCreationConfig `json:"autoTopicCreationOverride"` 48 | SchemaCompatibilityStrategy SchemaCompatibilityStrategy `json:"schema_auto_update_compatibility_strategy"` 49 | AuthPolicies AuthPolicies `json:"auth_policies"` 50 | SubscriptionAuthMode SubscriptionAuthMode `json:"subscription_auth_mode"` 51 | IsAllowAutoUpdateSchema *bool `json:"is_allow_auto_update_schema"` 52 | } 53 | 54 | func NewDefaultPolicies() *Policies { 55 | return &Policies{ 56 | AuthPolicies: *NewAuthPolicies(), 57 | ReplicationClusters: make([]string, 0, 10), 58 | BacklogQuotaMap: make(map[BacklogQuotaType]BacklogQuota), 59 | TopicDispatchRate: make(map[string]DispatchRate), 60 | SubscriptionDispatchRate: make(map[string]DispatchRate), 61 | ReplicatorDispatchRate: make(map[string]DispatchRate), 62 | PublishMaxMessageRate: make(map[string]PublishRate), 63 | ClusterSubscribeRate: make(map[string]SubscribeRate), 64 | LatencyStatsSampleRate: make(map[string]int), 65 | MessageTTLInSeconds: nil, 66 | Deleted: false, 67 | EncryptionRequired: false, 68 | SubscriptionAuthMode: None, 69 | MaxProducersPerTopic: nil, 70 | MaxConsumersPerSubscription: nil, 71 | MaxConsumersPerTopic: nil, 72 | CompactionThreshold: nil, 73 | OffloadThreshold: -1, 74 | SchemaCompatibilityStrategy: Full, 75 | SchemaValidationEnforced: false, 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /pkg/utils/topic_name.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "fmt" 20 | "net/url" 21 | "strconv" 22 | "strings" 23 | 24 | "github.com/pkg/errors" 25 | ) 26 | 27 | const ( 28 | PUBLICTENANT = "public" 29 | DEFAULTNAMESPACE = "default" 30 | PARTITIONEDTOPICSUFFIX = "-partition-" 31 | ) 32 | 33 | type TopicName struct { 34 | domain TopicDomain 35 | tenant string 36 | namespace string 37 | topic string 38 | partitionIndex int 39 | 40 | namespaceName *NameSpaceName 41 | } 42 | 43 | // The topic name can be in two different forms, one is fully qualified topic name, 44 | // the other one is short topic name 45 | func GetTopicName(completeName string) (*TopicName, error) { 46 | var topicName TopicName 47 | // The short topic name can be: 48 | // - 49 | // - // 50 | if !strings.Contains(completeName, "://") { 51 | parts := strings.Split(completeName, "/") 52 | switch len(parts) { 53 | case 3: 54 | completeName = persistent.String() + "://" + completeName 55 | case 1: 56 | completeName = persistent.String() + "://" + PUBLICTENANT + "/" + DEFAULTNAMESPACE + "/" + parts[0] 57 | default: 58 | return nil, errors.Errorf("Invalid short topic name '%s', it should be "+ 59 | "in the format of // or ", completeName) 60 | } 61 | } 62 | 63 | // The fully qualified topic name can be: 64 | // ://// 65 | 66 | parts := strings.SplitN(completeName, "://", 2) 67 | 68 | domain, err := ParseTopicDomain(parts[0]) 69 | if err != nil { 70 | return nil, err 71 | } 72 | topicName.domain = domain 73 | 74 | rest := parts[1] 75 | parts = strings.SplitN(rest, "/", 3) 76 | if len(parts) == 3 { 77 | topicName.tenant = parts[0] 78 | topicName.namespace = parts[1] 79 | topicName.topic = parts[2] 80 | topicName.partitionIndex = getPartitionIndex(completeName) 81 | } else { 82 | return nil, errors.Errorf("invalid topic name '%s', it should be in the format of "+ 83 | "//", rest) 84 | } 85 | 86 | if topicName.topic == "" { 87 | return nil, errors.New("topic name can not be empty") 88 | } 89 | 90 | n, err := GetNameSpaceName(topicName.tenant, topicName.namespace) 91 | if err != nil { 92 | return nil, err 93 | } 94 | topicName.namespaceName = n 95 | 96 | return &topicName, nil 97 | } 98 | 99 | func (t *TopicName) String() string { 100 | return fmt.Sprintf("%s://%s/%s/%s", t.domain, t.tenant, t.namespace, t.topic) 101 | } 102 | 103 | func (t *TopicName) GetDomain() TopicDomain { 104 | return t.domain 105 | } 106 | 107 | func (t *TopicName) GetTenant() string { 108 | return t.tenant 109 | } 110 | 111 | func (t *TopicName) GetNamespace() string { 112 | return t.namespace 113 | } 114 | 115 | func (t *TopicName) IsPersistent() bool { 116 | return t.domain == persistent 117 | } 118 | 119 | func (t *TopicName) GetRestPath() string { 120 | return fmt.Sprintf("%s/%s/%s/%s", t.domain, t.tenant, t.namespace, t.topic) 121 | } 122 | 123 | func (t *TopicName) GetEncodedTopic() string { 124 | return url.QueryEscape(t.topic) 125 | } 126 | 127 | func (t *TopicName) GetLocalName() string { 128 | return t.topic 129 | } 130 | 131 | func (t *TopicName) GetPartition(index int) (*TopicName, error) { 132 | if index < 0 { 133 | return nil, errors.New("invalid partition index number") 134 | } 135 | 136 | if strings.Contains(t.String(), PARTITIONEDTOPICSUFFIX) { 137 | return t, nil 138 | } 139 | 140 | topicNameWithPartition := t.String() + PARTITIONEDTOPICSUFFIX + strconv.Itoa(index) 141 | return GetTopicName(topicNameWithPartition) 142 | } 143 | 144 | func getPartitionIndex(topic string) int { 145 | if strings.Contains(topic, PARTITIONEDTOPICSUFFIX) { 146 | parts := strings.Split(topic, "-") 147 | index, err := strconv.Atoi(parts[len(parts)-1]) 148 | if err == nil { 149 | return index 150 | } 151 | } 152 | return -1 153 | } 154 | -------------------------------------------------------------------------------- /pkg/admin/schema.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "fmt" 20 | "strconv" 21 | 22 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 23 | ) 24 | 25 | // Schema is admin interface for schema management 26 | type Schema interface { 27 | // GetSchemaInfo retrieves the latest schema of a topic 28 | GetSchemaInfo(topic string) (*utils.SchemaInfo, error) 29 | 30 | // GetSchemaInfoWithVersion retrieves the latest schema with version of a topic 31 | GetSchemaInfoWithVersion(topic string) (*utils.SchemaInfoWithVersion, error) 32 | 33 | // GetSchemaInfoByVersion retrieves the schema of a topic at a given version 34 | GetSchemaInfoByVersion(topic string, version int64) (*utils.SchemaInfo, error) 35 | 36 | // DeleteSchema deletes the schema associated with a given topic 37 | DeleteSchema(topic string) error 38 | 39 | // CreateSchemaByPayload creates a schema for a given topic 40 | CreateSchemaByPayload(topic string, schemaPayload utils.PostSchemaPayload) error 41 | } 42 | 43 | type schemas struct { 44 | pulsar *pulsarClient 45 | basePath string 46 | } 47 | 48 | // Schemas is used to access the schemas endpoints 49 | func (c *pulsarClient) Schemas() Schema { 50 | return &schemas{ 51 | pulsar: c, 52 | basePath: "/schemas", 53 | } 54 | } 55 | 56 | func (s *schemas) GetSchemaInfo(topic string) (*utils.SchemaInfo, error) { 57 | topicName, err := utils.GetTopicName(topic) 58 | if err != nil { 59 | return nil, err 60 | } 61 | var response utils.GetSchemaResponse 62 | endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), 63 | topicName.GetLocalName(), "schema") 64 | 65 | err = s.pulsar.Client.Get(endpoint, &response) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | info := utils.ConvertGetSchemaResponseToSchemaInfo(topicName, response) 71 | return info, nil 72 | } 73 | 74 | func (s *schemas) GetSchemaInfoWithVersion(topic string) (*utils.SchemaInfoWithVersion, error) { 75 | topicName, err := utils.GetTopicName(topic) 76 | if err != nil { 77 | return nil, err 78 | } 79 | var response utils.GetSchemaResponse 80 | endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), 81 | topicName.GetLocalName(), "schema") 82 | 83 | err = s.pulsar.Client.Get(endpoint, &response) 84 | if err != nil { 85 | fmt.Println("err:", err.Error()) 86 | return nil, err 87 | } 88 | 89 | info := utils.ConvertGetSchemaResponseToSchemaInfoWithVersion(topicName, response) 90 | return info, nil 91 | } 92 | 93 | func (s *schemas) GetSchemaInfoByVersion(topic string, version int64) (*utils.SchemaInfo, error) { 94 | topicName, err := utils.GetTopicName(topic) 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | var response utils.GetSchemaResponse 100 | endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), topicName.GetLocalName(), 101 | "schema", strconv.FormatInt(version, 10)) 102 | 103 | err = s.pulsar.Client.Get(endpoint, &response) 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | info := utils.ConvertGetSchemaResponseToSchemaInfo(topicName, response) 109 | return info, nil 110 | } 111 | 112 | func (s *schemas) DeleteSchema(topic string) error { 113 | topicName, err := utils.GetTopicName(topic) 114 | if err != nil { 115 | return err 116 | } 117 | 118 | endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), 119 | topicName.GetLocalName(), "schema") 120 | 121 | fmt.Println(endpoint) 122 | 123 | return s.pulsar.Client.Delete(endpoint) 124 | } 125 | 126 | func (s *schemas) CreateSchemaByPayload(topic string, schemaPayload utils.PostSchemaPayload) error { 127 | topicName, err := utils.GetTopicName(topic) 128 | if err != nil { 129 | return err 130 | } 131 | 132 | endpoint := s.pulsar.endpoint(s.basePath, topicName.GetTenant(), topicName.GetNamespace(), 133 | topicName.GetLocalName(), "schema") 134 | 135 | return s.pulsar.Client.Post(endpoint, &schemaPayload) 136 | } 137 | -------------------------------------------------------------------------------- /pkg/admin/ns_isolation_policy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 20 | ) 21 | 22 | type NsIsolationPolicy interface { 23 | // Create a namespace isolation policy for a cluster 24 | CreateNamespaceIsolationPolicy(cluster, policyName string, namespaceIsolationData utils.NamespaceIsolationData) error 25 | 26 | // Delete a namespace isolation policy for a cluster 27 | DeleteNamespaceIsolationPolicy(cluster, policyName string) error 28 | 29 | // Get a single namespace isolation policy for a cluster 30 | GetNamespaceIsolationPolicy(cluster, policyName string) (*utils.NamespaceIsolationData, error) 31 | 32 | // Get the namespace isolation policies of a cluster 33 | GetNamespaceIsolationPolicies(cluster string) (map[string]utils.NamespaceIsolationData, error) 34 | 35 | // Returns list of active brokers with namespace-isolation policies attached to it. 36 | GetBrokersWithNamespaceIsolationPolicy(cluster string) ([]utils.BrokerNamespaceIsolationData, error) 37 | 38 | // Returns active broker with namespace-isolation policies attached to it. 39 | GetBrokerWithNamespaceIsolationPolicy(cluster, broker string) (*utils.BrokerNamespaceIsolationData, error) 40 | } 41 | 42 | type nsIsolationPolicy struct { 43 | pulsar *pulsarClient 44 | basePath string 45 | } 46 | 47 | func (c *pulsarClient) NsIsolationPolicy() NsIsolationPolicy { 48 | return &nsIsolationPolicy{ 49 | pulsar: c, 50 | basePath: "/clusters", 51 | } 52 | } 53 | 54 | func (n *nsIsolationPolicy) CreateNamespaceIsolationPolicy(cluster, policyName string, 55 | namespaceIsolationData utils.NamespaceIsolationData) error { 56 | return n.setNamespaceIsolationPolicy(cluster, policyName, namespaceIsolationData) 57 | } 58 | 59 | func (n *nsIsolationPolicy) setNamespaceIsolationPolicy(cluster, policyName string, 60 | namespaceIsolationData utils.NamespaceIsolationData) error { 61 | endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) 62 | return n.pulsar.Client.Post(endpoint, &namespaceIsolationData) 63 | } 64 | 65 | func (n *nsIsolationPolicy) DeleteNamespaceIsolationPolicy(cluster, policyName string) error { 66 | endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) 67 | return n.pulsar.Client.Delete(endpoint) 68 | } 69 | 70 | func (n *nsIsolationPolicy) GetNamespaceIsolationPolicy(cluster, policyName string) ( 71 | *utils.NamespaceIsolationData, error) { 72 | endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", policyName) 73 | var nsIsolationData utils.NamespaceIsolationData 74 | err := n.pulsar.Client.Get(endpoint, &nsIsolationData) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return &nsIsolationData, nil 79 | } 80 | 81 | func (n *nsIsolationPolicy) GetNamespaceIsolationPolicies(cluster string) ( 82 | map[string]utils.NamespaceIsolationData, error) { 83 | endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies") 84 | var tmpMap map[string]utils.NamespaceIsolationData 85 | err := n.pulsar.Client.Get(endpoint, &tmpMap) 86 | if err != nil { 87 | return nil, err 88 | } 89 | return tmpMap, nil 90 | } 91 | 92 | func (n *nsIsolationPolicy) GetBrokersWithNamespaceIsolationPolicy(cluster string) ( 93 | []utils.BrokerNamespaceIsolationData, error) { 94 | endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", "brokers") 95 | var res []utils.BrokerNamespaceIsolationData 96 | err := n.pulsar.Client.Get(endpoint, &res) 97 | if err != nil { 98 | return nil, err 99 | } 100 | return res, nil 101 | } 102 | 103 | func (n *nsIsolationPolicy) GetBrokerWithNamespaceIsolationPolicy(cluster, 104 | broker string) (*utils.BrokerNamespaceIsolationData, error) { 105 | endpoint := n.pulsar.endpoint(n.basePath, cluster, "namespaceIsolationPolicies", "brokers", broker) 106 | var brokerNamespaceIsolationData utils.BrokerNamespaceIsolationData 107 | err := n.pulsar.Client.Get(endpoint, &brokerNamespaceIsolationData) 108 | if err != nil { 109 | return nil, err 110 | } 111 | return &brokerNamespaceIsolationData, nil 112 | } 113 | -------------------------------------------------------------------------------- /pkg/admin/cluster.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 20 | ) 21 | 22 | // Clusters is admin interface for clusters management 23 | type Clusters interface { 24 | // List returns the list of clusters 25 | List() ([]string, error) 26 | 27 | // Get the configuration data for the specified cluster 28 | Get(string) (utils.ClusterData, error) 29 | 30 | // Create a new cluster 31 | Create(utils.ClusterData) error 32 | 33 | // Delete an existing cluster 34 | Delete(string) error 35 | 36 | // Update the configuration for a cluster 37 | Update(utils.ClusterData) error 38 | 39 | // UpdatePeerClusters updates peer cluster names. 40 | UpdatePeerClusters(string, []string) error 41 | 42 | // GetPeerClusters returns peer-cluster names 43 | GetPeerClusters(string) ([]string, error) 44 | 45 | // CreateFailureDomain creates a domain into cluster 46 | CreateFailureDomain(utils.FailureDomainData) error 47 | 48 | // GetFailureDomain returns the domain registered into a cluster 49 | GetFailureDomain(clusterName, domainName string) (utils.FailureDomainData, error) 50 | 51 | // ListFailureDomains returns all registered domains in cluster 52 | ListFailureDomains(string) (utils.FailureDomainMap, error) 53 | 54 | // DeleteFailureDomain deletes a domain in cluster 55 | DeleteFailureDomain(utils.FailureDomainData) error 56 | 57 | // UpdateFailureDomain updates a domain into cluster 58 | UpdateFailureDomain(utils.FailureDomainData) error 59 | } 60 | 61 | type clusters struct { 62 | pulsar *pulsarClient 63 | basePath string 64 | } 65 | 66 | // Clusters is used to access the cluster endpoints. 67 | func (c *pulsarClient) Clusters() Clusters { 68 | return &clusters{ 69 | pulsar: c, 70 | basePath: "/clusters", 71 | } 72 | } 73 | 74 | func (c *clusters) List() ([]string, error) { 75 | var clusters []string 76 | err := c.pulsar.Client.Get(c.pulsar.endpoint(c.basePath), &clusters) 77 | return clusters, err 78 | } 79 | 80 | func (c *clusters) Get(name string) (utils.ClusterData, error) { 81 | cdata := utils.ClusterData{} 82 | endpoint := c.pulsar.endpoint(c.basePath, name) 83 | err := c.pulsar.Client.Get(endpoint, &cdata) 84 | return cdata, err 85 | } 86 | 87 | func (c *clusters) Create(cdata utils.ClusterData) error { 88 | endpoint := c.pulsar.endpoint(c.basePath, cdata.Name) 89 | return c.pulsar.Client.Put(endpoint, &cdata) 90 | } 91 | 92 | func (c *clusters) Delete(name string) error { 93 | endpoint := c.pulsar.endpoint(c.basePath, name) 94 | return c.pulsar.Client.Delete(endpoint) 95 | } 96 | 97 | func (c *clusters) Update(cdata utils.ClusterData) error { 98 | endpoint := c.pulsar.endpoint(c.basePath, cdata.Name) 99 | return c.pulsar.Client.Post(endpoint, &cdata) 100 | } 101 | 102 | func (c *clusters) GetPeerClusters(name string) ([]string, error) { 103 | var peerClusters []string 104 | endpoint := c.pulsar.endpoint(c.basePath, name, "peers") 105 | err := c.pulsar.Client.Get(endpoint, &peerClusters) 106 | return peerClusters, err 107 | } 108 | 109 | func (c *clusters) UpdatePeerClusters(cluster string, peerClusters []string) error { 110 | endpoint := c.pulsar.endpoint(c.basePath, cluster, "peers") 111 | return c.pulsar.Client.Post(endpoint, peerClusters) 112 | } 113 | 114 | func (c *clusters) CreateFailureDomain(data utils.FailureDomainData) error { 115 | endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) 116 | return c.pulsar.Client.Post(endpoint, &data) 117 | } 118 | 119 | func (c *clusters) GetFailureDomain(clusterName string, domainName string) (utils.FailureDomainData, error) { 120 | var res utils.FailureDomainData 121 | endpoint := c.pulsar.endpoint(c.basePath, clusterName, "failureDomains", domainName) 122 | err := c.pulsar.Client.Get(endpoint, &res) 123 | return res, err 124 | } 125 | 126 | func (c *clusters) ListFailureDomains(clusterName string) (utils.FailureDomainMap, error) { 127 | var domainData utils.FailureDomainMap 128 | endpoint := c.pulsar.endpoint(c.basePath, clusterName, "failureDomains") 129 | err := c.pulsar.Client.Get(endpoint, &domainData) 130 | return domainData, err 131 | } 132 | 133 | func (c *clusters) DeleteFailureDomain(data utils.FailureDomainData) error { 134 | endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) 135 | return c.pulsar.Client.Delete(endpoint) 136 | } 137 | func (c *clusters) UpdateFailureDomain(data utils.FailureDomainData) error { 138 | endpoint := c.pulsar.endpoint(c.basePath, data.ClusterName, "failureDomains", data.DomainName) 139 | return c.pulsar.Client.Post(endpoint, &data) 140 | } 141 | -------------------------------------------------------------------------------- /pkg/utils/functions_stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | type FunctionStats struct { 19 | // Overall total number of records function received from source 20 | ReceivedTotal int64 `json:"receivedTotal"` 21 | 22 | // Overall total number of records successfully processed by user function 23 | ProcessedSuccessfullyTotal int64 `json:"processedSuccessfullyTotal"` 24 | 25 | // Overall total number of system exceptions thrown 26 | SystemExceptionsTotal int64 `json:"systemExceptionsTotal"` 27 | 28 | // Overall total number of user exceptions thrown 29 | UserExceptionsTotal int64 `json:"userExceptionsTotal"` 30 | 31 | // Average process latency for function 32 | AvgProcessLatency float64 `json:"avgProcessLatency"` 33 | 34 | // Timestamp of when the function was last invoked by any instance 35 | LastInvocation int64 `json:"lastInvocation"` 36 | 37 | OneMin FunctionInstanceStatsDataBase `json:"oneMin"` 38 | 39 | Instances []FunctionInstanceStats `json:"instances"` 40 | 41 | FunctionInstanceStats 42 | } 43 | 44 | type FunctionInstanceStats struct { 45 | FunctionInstanceStatsDataBase 46 | 47 | InstanceID int64 `json:"instanceId"` 48 | 49 | Metrics FunctionInstanceStatsData `json:"metrics"` 50 | } 51 | 52 | type FunctionInstanceStatsDataBase struct { 53 | // Total number of records function received from source for instance 54 | ReceivedTotal int64 `json:"receivedTotal"` 55 | 56 | // Total number of records successfully processed by user function for instance 57 | ProcessedSuccessfullyTotal int64 `json:"processedSuccessfullyTotal"` 58 | 59 | // Total number of system exceptions thrown for instance 60 | SystemExceptionsTotal int64 `json:"systemExceptionsTotal"` 61 | 62 | // Total number of user exceptions thrown for instance 63 | UserExceptionsTotal int64 `json:"userExceptionsTotal"` 64 | 65 | // Average process latency for function for instance 66 | AvgProcessLatency float64 `json:"avgProcessLatency"` 67 | } 68 | 69 | type FunctionInstanceStatsData struct { 70 | OneMin FunctionInstanceStatsDataBase `json:"oneMin"` 71 | 72 | // Timestamp of when the function was last invoked for instance 73 | LastInvocation int64 `json:"lastInvocation"` 74 | 75 | // Map of user defined metrics 76 | UserMetrics map[string]float64 `json:"userMetrics"` 77 | 78 | FunctionInstanceStatsDataBase 79 | } 80 | 81 | func (fs *FunctionStats) AddInstance(functionInstanceStats FunctionInstanceStats) { 82 | fs.Instances = append(fs.Instances, functionInstanceStats) 83 | } 84 | 85 | func (fs *FunctionStats) CalculateOverall() *FunctionStats { 86 | var ( 87 | nonNullInstances int 88 | nonNullInstancesOneMin int 89 | ) 90 | 91 | for _, functionInstanceStats := range fs.Instances { 92 | functionInstanceStatsData := functionInstanceStats.Metrics 93 | fs.ReceivedTotal += functionInstanceStatsData.ReceivedTotal 94 | fs.ProcessedSuccessfullyTotal += functionInstanceStatsData.ProcessedSuccessfullyTotal 95 | fs.SystemExceptionsTotal += functionInstanceStatsData.SystemExceptionsTotal 96 | fs.UserExceptionsTotal += functionInstanceStatsData.UserExceptionsTotal 97 | 98 | if functionInstanceStatsData.AvgProcessLatency != 0 { 99 | if fs.AvgProcessLatency == 0 { 100 | fs.AvgProcessLatency = 0.0 101 | } 102 | 103 | fs.AvgProcessLatency += functionInstanceStatsData.AvgProcessLatency 104 | nonNullInstances++ 105 | } 106 | 107 | fs.OneMin.ReceivedTotal += functionInstanceStatsData.OneMin.ReceivedTotal 108 | fs.OneMin.ProcessedSuccessfullyTotal += functionInstanceStatsData.OneMin.ProcessedSuccessfullyTotal 109 | fs.OneMin.SystemExceptionsTotal += functionInstanceStatsData.OneMin.SystemExceptionsTotal 110 | fs.OneMin.UserExceptionsTotal += functionInstanceStatsData.OneMin.UserExceptionsTotal 111 | 112 | if functionInstanceStatsData.OneMin.AvgProcessLatency != 0 { 113 | if fs.OneMin.AvgProcessLatency == 0 { 114 | fs.OneMin.AvgProcessLatency = 0.0 115 | } 116 | 117 | fs.OneMin.AvgProcessLatency += functionInstanceStatsData.OneMin.AvgProcessLatency 118 | nonNullInstancesOneMin++ 119 | } 120 | 121 | if functionInstanceStatsData.LastInvocation != 0 { 122 | if fs.LastInvocation == 0 || functionInstanceStatsData.LastInvocation > fs.LastInvocation { 123 | fs.LastInvocation = functionInstanceStatsData.LastInvocation 124 | } 125 | } 126 | } 127 | 128 | // calculate average from sum 129 | if nonNullInstances > 0 { 130 | fs.AvgProcessLatency /= float64(nonNullInstances) 131 | } else { 132 | fs.AvgProcessLatency = 0 133 | } 134 | 135 | // calculate 1min average from sum 136 | if nonNullInstancesOneMin > 0 { 137 | fs.OneMin.AvgProcessLatency /= float64(nonNullInstancesOneMin) 138 | } else { 139 | fs.AvgProcessLatency = 0 140 | } 141 | 142 | return fs 143 | } 144 | -------------------------------------------------------------------------------- /pkg/utils/function_confg.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | const ( 19 | JavaRuntime = "JAVA" 20 | PythonRuntime = "PYTHON" 21 | GoRuntime = "GO" 22 | ) 23 | 24 | type FunctionConfig struct { 25 | TimeoutMs *int64 `json:"timeoutMs,omitempty" yaml:"timeoutMs"` 26 | TopicsPattern *string `json:"topicsPattern,omitempty" yaml:"topicsPattern"` 27 | // Whether the subscriptions the functions created/used should be deleted when the functions is deleted 28 | CleanupSubscription bool `json:"cleanupSubscription" yaml:"cleanupSubscription"` 29 | RetainOrdering bool `json:"retainOrdering" yaml:"retainOrdering"` 30 | RetainKeyOrdering bool `json:"retainKeyOrdering" yaml:"retainKeyOrdering"` 31 | BatchBuilder string `json:"batchBuilder,omitempty" yaml:"batchBuilder"` 32 | ForwardSourceMessageProperty bool `json:"forwardSourceMessageProperty" yaml:"forwardSourceMessageProperty"` 33 | AutoAck bool `json:"autoAck" yaml:"autoAck"` 34 | Parallelism int `json:"parallelism,omitempty" yaml:"parallelism"` 35 | MaxMessageRetries *int `json:"maxMessageRetries,omitempty" yaml:"maxMessageRetries"` 36 | 37 | Output string `json:"output,omitempty" yaml:"output"` 38 | 39 | ProducerConfig *ProducerConfig `json:"producerConfig,omitempty" yaml:"producerConfig"` 40 | CustomSchemaOutputs map[string]string `json:"customSchemaOutputs,omitempty" yaml:"customSchemaOutputs"` 41 | 42 | OutputSerdeClassName string `json:"outputSerdeClassName,omitempty" yaml:"outputSerdeClassName"` 43 | LogTopic string `json:"logTopic,omitempty" yaml:"logTopic"` 44 | ProcessingGuarantees string `json:"processingGuarantees,omitempty" yaml:"processingGuarantees"` 45 | 46 | // Represents either a builtin schema type (eg: 'avro', 'json', etc) or the class name for a Schema implementation 47 | OutputSchemaType string `json:"outputSchemaType,omitempty" yaml:"outputSchemaType"` 48 | OutputTypeClassName string `json:"outputTypeClassName,omitempty" yaml:"outputTypeClassName"` 49 | 50 | Runtime string `json:"runtime,omitempty" yaml:"runtime"` 51 | DeadLetterTopic string `json:"deadLetterTopic,omitempty" yaml:"deadLetterTopic"` 52 | SubName string `json:"subName,omitempty" yaml:"subName"` 53 | FQFN string `json:"fqfn,omitempty" yaml:"fqfn"` 54 | Jar *string `json:"jar,omitempty" yaml:"jar"` 55 | Py *string `json:"py,omitempty" yaml:"py"` 56 | Go *string `json:"go,omitempty" yaml:"go"` 57 | FunctionType *string `json:"functionType,omitempty" yaml:"functionType"` 58 | // Any flags that you want to pass to the runtime. 59 | // note that in thread mode, these flags will have no impact 60 | RuntimeFlags string `json:"runtimeFlags,omitempty" yaml:"runtimeFlags"` 61 | 62 | Tenant string `json:"tenant,omitempty" yaml:"tenant"` 63 | Namespace string `json:"namespace,omitempty" yaml:"namespace"` 64 | Name string `json:"name,omitempty" yaml:"name"` 65 | ClassName string `json:"className,omitempty" yaml:"className"` 66 | 67 | Resources *Resources `json:"resources,omitempty" yaml:"resources"` 68 | WindowConfig *WindowConfig `json:"windowConfig,omitempty" yaml:"windowConfig"` 69 | Inputs []string `json:"inputs,omitempty" yaml:"inputs"` 70 | UserConfig map[string]interface{} `json:"userConfig,omitempty" yaml:"userConfig"` 71 | CustomSerdeInputs map[string]string `json:"customSerdeInputs,omitempty" yaml:"customSerdeInputs"` 72 | CustomSchemaInputs map[string]string `json:"customSchemaInputs,omitempty" yaml:"customSchemaInputs"` 73 | 74 | // A generalized way of specifying inputs 75 | InputSpecs map[string]ConsumerConfig `json:"inputSpecs,omitempty" yaml:"inputSpecs"` 76 | InputTypeClassName string `json:"inputTypeClassName,omitempty" yaml:"inputTypeClassName"` 77 | 78 | CustomRuntimeOptions string `json:"customRuntimeOptions,omitempty" yaml:"customRuntimeOptions"` 79 | 80 | // This is a map of secretName(aka how the secret is going to be 81 | // accessed in the function via context) to an object that 82 | // encapsulates how the secret is fetched by the underlying 83 | // secrets provider. The type of an value here can be found by the 84 | // SecretProviderConfigurator.getSecretObjectType() method. 85 | Secrets map[string]interface{} `json:"secrets,omitempty" yaml:"secrets"` 86 | 87 | MaxPendingAsyncRequests int `json:"maxPendingAsyncRequests,omitempty" yaml:"maxPendingAsyncRequests"` 88 | //nolint 89 | ExposePulsarAdminClientEnabled bool `json:"exposePulsarAdminClientEnabled" yaml:"exposePulsarAdminClientEnabled"` 90 | SkipToLatest bool `json:"skipToLatest" yaml:"skipToLatest"` 91 | SubscriptionPosition string `json:"subscriptionPosition,omitempty" yaml:"subscriptionPosition"` 92 | } 93 | -------------------------------------------------------------------------------- /pkg/admin/brokers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "fmt" 20 | "net/url" 21 | "strings" 22 | 23 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 24 | ) 25 | 26 | // Brokers is admin interface for brokers management 27 | type Brokers interface { 28 | // GetActiveBrokers returns the list of active brokers in the cluster. 29 | GetActiveBrokers(cluster string) ([]string, error) 30 | 31 | // GetDynamicConfigurationNames returns list of updatable configuration name 32 | GetDynamicConfigurationNames() ([]string, error) 33 | 34 | // GetOwnedNamespaces returns the map of owned namespaces and their status from a single broker in the cluster 35 | GetOwnedNamespaces(cluster, brokerURL string) (map[string]utils.NamespaceOwnershipStatus, error) 36 | 37 | // UpdateDynamicConfiguration updates dynamic configuration value in to Zk that triggers watch on 38 | // brokers and all brokers can update {@link ServiceConfiguration} value locally 39 | UpdateDynamicConfiguration(configName, configValue string) error 40 | 41 | // DeleteDynamicConfiguration deletes dynamic configuration value in to Zk. It will not impact current value 42 | // in broker but next time when broker restarts, it applies value from configuration file only. 43 | DeleteDynamicConfiguration(configName string) error 44 | 45 | // GetRuntimeConfigurations returns values of runtime configuration 46 | GetRuntimeConfigurations() (map[string]string, error) 47 | 48 | // GetInternalConfigurationData returns the internal configuration data 49 | GetInternalConfigurationData() (*utils.InternalConfigurationData, error) 50 | 51 | // GetAllDynamicConfigurations returns values of all overridden dynamic-configs 52 | GetAllDynamicConfigurations() (map[string]string, error) 53 | 54 | // HealthCheck run a health check on the broker 55 | HealthCheck() error 56 | } 57 | 58 | type broker struct { 59 | pulsar *pulsarClient 60 | basePath string 61 | } 62 | 63 | // Brokers is used to access the brokers endpoints 64 | func (c *pulsarClient) Brokers() Brokers { 65 | return &broker{ 66 | pulsar: c, 67 | basePath: "/brokers", 68 | } 69 | } 70 | 71 | func (b *broker) GetActiveBrokers(cluster string) ([]string, error) { 72 | endpoint := b.pulsar.endpoint(b.basePath, cluster) 73 | var res []string 74 | err := b.pulsar.Client.Get(endpoint, &res) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return res, nil 79 | } 80 | 81 | func (b *broker) GetDynamicConfigurationNames() ([]string, error) { 82 | endpoint := b.pulsar.endpoint(b.basePath, "/configuration/") 83 | var res []string 84 | err := b.pulsar.Client.Get(endpoint, &res) 85 | if err != nil { 86 | return nil, err 87 | } 88 | return res, nil 89 | } 90 | 91 | func (b *broker) GetOwnedNamespaces(cluster, brokerURL string) (map[string]utils.NamespaceOwnershipStatus, error) { 92 | endpoint := b.pulsar.endpoint(b.basePath, cluster, brokerURL, "ownedNamespaces") 93 | var res map[string]utils.NamespaceOwnershipStatus 94 | err := b.pulsar.Client.Get(endpoint, &res) 95 | if err != nil { 96 | return nil, err 97 | } 98 | return res, nil 99 | } 100 | 101 | func (b *broker) UpdateDynamicConfiguration(configName, configValue string) error { 102 | value := url.QueryEscape(configValue) 103 | endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", configName, value) 104 | return b.pulsar.Client.Post(endpoint, nil) 105 | } 106 | 107 | func (b *broker) DeleteDynamicConfiguration(configName string) error { 108 | endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", configName) 109 | return b.pulsar.Client.Delete(endpoint) 110 | } 111 | 112 | func (b *broker) GetRuntimeConfigurations() (map[string]string, error) { 113 | endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", "runtime") 114 | var res map[string]string 115 | err := b.pulsar.Client.Get(endpoint, &res) 116 | if err != nil { 117 | return nil, err 118 | } 119 | return res, nil 120 | } 121 | 122 | func (b *broker) GetInternalConfigurationData() (*utils.InternalConfigurationData, error) { 123 | endpoint := b.pulsar.endpoint(b.basePath, "/internal-configuration") 124 | var res utils.InternalConfigurationData 125 | err := b.pulsar.Client.Get(endpoint, &res) 126 | if err != nil { 127 | return nil, err 128 | } 129 | return &res, nil 130 | } 131 | 132 | func (b *broker) GetAllDynamicConfigurations() (map[string]string, error) { 133 | endpoint := b.pulsar.endpoint(b.basePath, "/configuration/", "values") 134 | var res map[string]string 135 | err := b.pulsar.Client.Get(endpoint, &res) 136 | if err != nil { 137 | return nil, err 138 | } 139 | return res, nil 140 | } 141 | 142 | func (b *broker) HealthCheck() error { 143 | endpoint := b.pulsar.endpoint(b.basePath, "/health") 144 | 145 | buf, err := b.pulsar.Client.GetWithQueryParams(endpoint, nil, nil, false) 146 | if err != nil { 147 | return err 148 | } 149 | 150 | if !strings.EqualFold(string(buf), "ok") { 151 | return fmt.Errorf("health check returned unexpected result: %s", string(buf)) 152 | } 153 | return nil 154 | } 155 | -------------------------------------------------------------------------------- /pkg/utils/load_manage_report.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package utils 17 | 18 | import ( 19 | "math" 20 | ) 21 | 22 | type LocalBrokerData struct { 23 | // URLs to satisfy contract of ServiceLookupData (used by NamespaceService). 24 | WebServiceURL string `json:"webServiceUrl"` 25 | WebServiceURLTLS string `json:"webServiceUrlTls"` 26 | PulsarServiceURL string `json:"pulsarServiceUrl"` 27 | PulsarServiceURLTLS string `json:"pulsarServiceUrlTls"` 28 | PersistentTopicsEnabled bool `json:"persistentTopicsEnabled"` 29 | NonPersistentTopicsEnabled bool `json:"nonPersistentTopicsEnabled"` 30 | 31 | // Most recently available system resource usage. 32 | CPU ResourceUsage `json:"cpu"` 33 | Memory ResourceUsage `json:"memory"` 34 | DirectMemory ResourceUsage `json:"directMemory"` 35 | BandwidthIn ResourceUsage `json:"bandwidthIn"` 36 | BandwidthOut ResourceUsage `json:"bandwidthOut"` 37 | 38 | // Message data from the most recent namespace bundle stats. 39 | MsgThroughputIn float64 `json:"msgThroughputIn"` 40 | MsgThroughputOut float64 `json:"msgThroughputOut"` 41 | MsgRateIn float64 `json:"msgRateIn"` 42 | MsgRateOut float64 `json:"msgRateOut"` 43 | 44 | // Timestamp of last update. 45 | LastUpdate int64 `json:"lastUpdate"` 46 | 47 | // The stats given in the most recent invocation of update. 48 | LastStats map[string]*NamespaceBundleStats `json:"lastStats"` 49 | NumTopics int `json:"numTopics"` 50 | NumBundles int `json:"numBundles"` 51 | NumConsumers int `json:"numConsumers"` 52 | NumProducers int `json:"numProducers"` 53 | 54 | // All bundles belonging to this broker. 55 | Bundles []string `json:"bundles"` 56 | 57 | // The bundles gained since the last invocation of update. 58 | LastBundleGains []string `json:"lastBundleGains"` 59 | 60 | // The bundles lost since the last invocation of update. 61 | LastBundleLosses []string `json:"lastBundleLosses"` 62 | 63 | // The version string that this broker is running, obtained from the Maven build artifact in the POM 64 | BrokerVersionString string `json:"brokerVersionString"` 65 | 66 | // This place-holder requires to identify correct LoadManagerReport type while deserializing 67 | LoadReportType string `json:"loadReportType"` 68 | 69 | // the external protocol data advertised by protocol handlers. 70 | Protocols map[string]string `json:"protocols"` 71 | } 72 | 73 | func NewLocalBrokerData() LocalBrokerData { 74 | lastStats := make(map[string]*NamespaceBundleStats) 75 | lastStats[""] = NewNamespaceBundleStats() 76 | return LocalBrokerData{ 77 | LastStats: lastStats, 78 | } 79 | } 80 | 81 | type NamespaceBundleStats struct { 82 | MsgRateIn float64 `json:"msgRateIn"` 83 | MsgThroughputIn float64 `json:"msgThroughputIn"` 84 | MsgRateOut float64 `json:"msgRateOut"` 85 | MsgThroughputOut float64 `json:"msgThroughputOut"` 86 | ConsumerCount int `json:"consumerCount"` 87 | ProducerCount int `json:"producerCount"` 88 | TopicsNum int64 `json:"topics"` 89 | CacheSize int64 `json:"cacheSize"` 90 | 91 | // Consider the throughput equal if difference is less than 100 KB/s 92 | ThroughputDifferenceThreshold float64 `json:"throughputDifferenceThreshold"` 93 | // Consider the msgRate equal if the difference is less than 100 94 | MsgRateDifferenceThreshold float64 `json:"msgRateDifferenceThreshold"` 95 | // Consider the total topics/producers/consumers equal if the difference is less than 500 96 | TopicConnectionDifferenceThreshold int64 `json:"topicConnectionDifferenceThreshold"` 97 | // Consider the cache size equal if the difference is less than 100 kb 98 | CacheSizeDifferenceThreshold int64 `json:"cacheSizeDifferenceThreshold"` 99 | } 100 | 101 | func NewNamespaceBundleStats() *NamespaceBundleStats { 102 | return &NamespaceBundleStats{ 103 | ThroughputDifferenceThreshold: 1e5, 104 | MsgRateDifferenceThreshold: 100, 105 | TopicConnectionDifferenceThreshold: 500, 106 | CacheSizeDifferenceThreshold: 100000, 107 | } 108 | } 109 | 110 | type ResourceUsage struct { 111 | Usage float64 `json:"usage"` 112 | Limit float64 `json:"limit"` 113 | } 114 | 115 | func (ru *ResourceUsage) Reset() { 116 | ru.Usage = -1 117 | ru.Limit = -1 118 | } 119 | 120 | func (ru *ResourceUsage) CompareTo(o *ResourceUsage) int { 121 | required := o.Limit - o.Usage 122 | available := ru.Limit - ru.Usage 123 | return compare(required, available) 124 | } 125 | 126 | func (ru *ResourceUsage) PercentUsage() float32 { 127 | var proportion float32 128 | if ru.Limit > 0 { 129 | proportion = float32(ru.Usage) / float32(ru.Limit) 130 | } 131 | return proportion * 100 132 | } 133 | 134 | func compare(val1, val2 float64) int { 135 | if val1 < val2 { 136 | return -1 137 | } 138 | 139 | if val1 < val2 { 140 | return 1 141 | } 142 | 143 | thisBits := math.Float64bits(val1) 144 | anotherBits := math.Float64bits(val2) 145 | 146 | if thisBits == anotherBits { 147 | return 0 148 | } 149 | 150 | if thisBits < anotherBits { 151 | return -1 152 | } 153 | return 1 154 | } 155 | -------------------------------------------------------------------------------- /pkg/admin/auth/oauth2.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package auth 17 | 18 | import ( 19 | "encoding/json" 20 | "net/http" 21 | "path/filepath" 22 | 23 | "github.com/99designs/keyring" 24 | "github.com/apache/pulsar-client-go/oauth2" 25 | "github.com/apache/pulsar-client-go/oauth2/cache" 26 | clock2 "github.com/apache/pulsar-client-go/oauth2/clock" 27 | "github.com/apache/pulsar-client-go/oauth2/store" 28 | "github.com/pkg/errors" 29 | xoauth2 "golang.org/x/oauth2" 30 | ) 31 | 32 | const ( 33 | OAuth2PluginName = "org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2" 34 | OAuth2PluginShortName = "oauth2" 35 | ) 36 | 37 | type OAuth2ClientCredentials struct { 38 | IssuerURL string `json:"issuerUrl,omitempty"` 39 | Audience string `json:"audience,omitempty"` 40 | Scope string `json:"scope,omitempty"` 41 | PrivateKey string `json:"privateKey,omitempty"` 42 | ClientID string `json:"clientId,omitempty"` 43 | } 44 | 45 | type OAuth2Provider struct { 46 | clock clock2.RealClock 47 | issuer oauth2.Issuer 48 | store store.Store 49 | source cache.CachingTokenSource 50 | defaultTransport http.RoundTripper 51 | tokenTransport *transport 52 | } 53 | 54 | func NewAuthenticationOAuth2(issuer oauth2.Issuer, store store.Store) (*OAuth2Provider, error) { 55 | p := &OAuth2Provider{ 56 | clock: clock2.RealClock{}, 57 | issuer: issuer, 58 | store: store, 59 | } 60 | 61 | err := p.loadGrant() 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | return p, nil 67 | } 68 | 69 | // NewAuthenticationOAuth2WithDefaultFlow uses memory to save the grant 70 | func NewAuthenticationOAuth2WithDefaultFlow(issuer oauth2.Issuer, keyFile string) (Provider, error) { 71 | st := store.NewMemoryStore() 72 | flow, err := oauth2.NewDefaultClientCredentialsFlow(oauth2.ClientCredentialsFlowOptions{ 73 | KeyFile: keyFile, 74 | }) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | grant, err := flow.Authorize(issuer.Audience) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | err = st.SaveGrant(issuer.Audience, *grant) 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | p := &OAuth2Provider{ 90 | clock: clock2.RealClock{}, 91 | issuer: issuer, 92 | store: st, 93 | } 94 | 95 | return p, p.loadGrant() 96 | } 97 | 98 | func NewAuthenticationOAuth2FromAuthParams(encodedAuthParam string, 99 | transport http.RoundTripper) (*OAuth2Provider, error) { 100 | 101 | var paramsJSON OAuth2ClientCredentials 102 | err := json.Unmarshal([]byte(encodedAuthParam), ¶msJSON) 103 | if err != nil { 104 | return nil, err 105 | } 106 | return NewAuthenticationOAuth2WithParams(paramsJSON.IssuerURL, paramsJSON.ClientID, paramsJSON.Audience, 107 | paramsJSON.Scope, transport) 108 | } 109 | 110 | func NewAuthenticationOAuth2WithParams( 111 | issuerEndpoint, 112 | clientID, 113 | audience string, 114 | scope string, 115 | transport http.RoundTripper) (*OAuth2Provider, error) { 116 | 117 | issuer := oauth2.Issuer{ 118 | IssuerEndpoint: issuerEndpoint, 119 | ClientID: clientID, 120 | Audience: audience, 121 | } 122 | 123 | keyringStore, err := MakeKeyringStore() 124 | if err != nil { 125 | return nil, err 126 | } 127 | 128 | p := &OAuth2Provider{ 129 | clock: clock2.RealClock{}, 130 | issuer: issuer, 131 | store: keyringStore, 132 | defaultTransport: transport, 133 | } 134 | 135 | err = p.loadGrant() 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | return p, nil 141 | } 142 | 143 | func (o *OAuth2Provider) loadGrant() error { 144 | grant, err := o.store.LoadGrant(o.issuer.Audience) 145 | if err != nil { 146 | if err == store.ErrNoAuthenticationData { 147 | return errors.New("oauth2 login required") 148 | } 149 | return err 150 | } 151 | return o.initCache(grant) 152 | } 153 | 154 | func (o *OAuth2Provider) initCache(grant *oauth2.AuthorizationGrant) error { 155 | refresher, err := o.getRefresher(grant.Type) 156 | if err != nil { 157 | return err 158 | } 159 | 160 | source, err := cache.NewDefaultTokenCache(o.store, o.issuer.Audience, refresher) 161 | if err != nil { 162 | return err 163 | } 164 | o.source = source 165 | o.tokenTransport = &transport{ 166 | source: o.source, 167 | wrapped: &xoauth2.Transport{ 168 | Source: o.source, 169 | Base: o.defaultTransport, 170 | }, 171 | } 172 | return nil 173 | } 174 | 175 | func (o *OAuth2Provider) RoundTrip(req *http.Request) (*http.Response, error) { 176 | return o.tokenTransport.RoundTrip(req) 177 | } 178 | 179 | func (o *OAuth2Provider) WithTransport(tripper http.RoundTripper) { 180 | o.defaultTransport = tripper 181 | } 182 | 183 | func (o *OAuth2Provider) Transport() http.RoundTripper { 184 | return o.tokenTransport 185 | } 186 | 187 | func (o *OAuth2Provider) getRefresher(t oauth2.AuthorizationGrantType) (oauth2.AuthorizationGrantRefresher, error) { 188 | switch t { 189 | case oauth2.GrantTypeClientCredentials: 190 | return oauth2.NewDefaultClientCredentialsGrantRefresher(o.clock) 191 | case oauth2.GrantTypeDeviceCode: 192 | return oauth2.NewDefaultDeviceAuthorizationGrantRefresher(o.clock) 193 | default: 194 | return nil, store.ErrUnsupportedAuthData 195 | } 196 | } 197 | 198 | type transport struct { 199 | source cache.CachingTokenSource 200 | wrapped *xoauth2.Transport 201 | } 202 | 203 | func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { 204 | if len(req.Header.Get("Authorization")) != 0 { 205 | return t.wrapped.Base.RoundTrip(req) 206 | } 207 | 208 | res, err := t.wrapped.RoundTrip(req) 209 | if err != nil { 210 | return nil, err 211 | } 212 | 213 | if res.StatusCode == 401 { 214 | err := t.source.InvalidateToken() 215 | if err != nil { 216 | return nil, err 217 | } 218 | } 219 | 220 | return res, nil 221 | } 222 | 223 | func (t *transport) WrappedRoundTripper() http.RoundTripper { return t.wrapped.Base } 224 | 225 | const ( 226 | serviceName = "pulsar" 227 | keyChainName = "pulsarctl" 228 | ) 229 | 230 | func MakeKeyringStore() (store.Store, error) { 231 | kr, err := makeKeyring() 232 | if err != nil { 233 | return nil, err 234 | } 235 | return store.NewKeyringStore(kr) 236 | } 237 | 238 | func makeKeyring() (keyring.Keyring, error) { 239 | return keyring.Open(keyring.Config{ 240 | AllowedBackends: keyring.AvailableBackends(), 241 | ServiceName: serviceName, 242 | KeychainName: keyChainName, 243 | KeychainTrustApplication: true, 244 | FileDir: filepath.Join("~/.config/pulsar", "credentials"), 245 | FilePasswordFunc: keyringPrompt, 246 | }) 247 | } 248 | 249 | func keyringPrompt(prompt string) (string, error) { 250 | return "", nil 251 | } 252 | -------------------------------------------------------------------------------- /pkg/admin/packages.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 StreamNative, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, 10 | // software distributed under the License is distributed on an 11 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 12 | // KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations 14 | // under the License. 15 | 16 | package admin 17 | 18 | import ( 19 | "bytes" 20 | "encoding/json" 21 | "fmt" 22 | "io" 23 | "mime/multipart" 24 | "net/textproto" 25 | "os" 26 | "path" 27 | "path/filepath" 28 | "strings" 29 | 30 | "github.com/pkg/errors" 31 | 32 | "github.com/streamnative/pulsar-admin-go/pkg/utils" 33 | ) 34 | 35 | // Packages is admin interface for functions management 36 | type Packages interface { 37 | // Download Function/Connector Package 38 | // @param destinationFile 39 | // file where data should be downloaded to 40 | // @param packageURL 41 | // the package URL 42 | Download(packageURL, destinationFile string) error 43 | 44 | // Upload Function/Connector Package 45 | // @param filePath 46 | // file where data should be uploaded to 47 | // @param packageURL 48 | // type://tenant/namespace/packageName@version 49 | // @param description 50 | // descriptions of a package 51 | // @param contact 52 | // contact information of a package 53 | // @param properties 54 | // external infromations of a package 55 | Upload(packageURL, filePath, description, contact string, properties map[string]string) error 56 | 57 | // List all the packages with the given type in a namespace 58 | List(typeName, namespace string) ([]string, error) 59 | 60 | // ListVersions list all the versions of a package 61 | ListVersions(packageURL string) ([]string, error) 62 | 63 | // Delete the specified package 64 | Delete(packageURL string) error 65 | 66 | // GetMetadata get a package metadata information 67 | GetMetadata(packageURL string) (utils.PackageMetadata, error) 68 | 69 | // UpdateMetadata update a package metadata information 70 | UpdateMetadata(packageURL, description, contact string, properties map[string]string) error 71 | } 72 | 73 | type packages struct { 74 | pulsar *pulsarClient 75 | basePath string 76 | } 77 | 78 | func (p *packages) createStringFromField(w *multipart.Writer, value string) (io.Writer, error) { 79 | h := make(textproto.MIMEHeader) 80 | h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s" `, value)) 81 | h.Set("Content-Type", "application/json") 82 | return w.CreatePart(h) 83 | } 84 | 85 | // Packages is used to access the functions endpoints 86 | func (c *pulsarClient) Packages() Packages { 87 | return &packages{ 88 | pulsar: c, 89 | basePath: "/packages", 90 | } 91 | } 92 | 93 | func (p packages) Download(packageURL, destinationFile string) error { 94 | packageName, err := utils.GetPackageName(packageURL) 95 | if err != nil { 96 | return err 97 | } 98 | endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), 99 | packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) 100 | 101 | parent := path.Dir(destinationFile) 102 | if parent != "." { 103 | err = os.MkdirAll(parent, 0755) 104 | if err != nil { 105 | return fmt.Errorf("failed to create parent directory %s: %w", parent, err) 106 | } 107 | } 108 | 109 | _, err = os.Open(destinationFile) 110 | if err != nil { 111 | if !os.IsNotExist(err) { 112 | return fmt.Errorf("file %s already exists, please delete "+ 113 | "the file first or change the file name", destinationFile) 114 | } 115 | } 116 | file, err := os.Create(destinationFile) 117 | if err != nil { 118 | return err 119 | } 120 | 121 | _, err = p.pulsar.Client.GetWithOptions(endpoint, nil, nil, false, file) 122 | if err != nil { 123 | return err 124 | } 125 | return nil 126 | } 127 | 128 | func (p packages) Upload(packageURL, filePath, description, contact string, properties map[string]string) error { 129 | if strings.TrimSpace(filePath) == "" { 130 | return errors.New("file path is empty") 131 | } 132 | if strings.TrimSpace(packageURL) == "" { 133 | return errors.New("package URL is empty") 134 | } 135 | packageName, err := utils.GetPackageName(packageURL) 136 | if err != nil { 137 | return err 138 | } 139 | endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), 140 | packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) 141 | metadata := utils.PackageMetadata{ 142 | Description: description, 143 | Contact: contact, 144 | Properties: properties, 145 | } 146 | // buffer to store our request as bytes 147 | bodyBuf := bytes.NewBufferString("") 148 | 149 | multiPartWriter := multipart.NewWriter(bodyBuf) 150 | 151 | metadataJSON, err := json.Marshal(metadata) 152 | if err != nil { 153 | return err 154 | } 155 | 156 | stringWriter, err := p.createStringFromField(multiPartWriter, "metadata") 157 | if err != nil { 158 | return err 159 | } 160 | 161 | _, err = stringWriter.Write(metadataJSON) 162 | if err != nil { 163 | return err 164 | } 165 | 166 | file, err := os.Open(filePath) 167 | if err != nil { 168 | return err 169 | } 170 | defer file.Close() 171 | 172 | part, err := multiPartWriter.CreateFormFile("file", filepath.Base(file.Name())) 173 | 174 | if err != nil { 175 | return err 176 | } 177 | 178 | // copy the actual file content to the filed's writer 179 | _, err = io.Copy(part, file) 180 | if err != nil { 181 | return err 182 | } 183 | 184 | if err = multiPartWriter.Close(); err != nil { 185 | return err 186 | } 187 | 188 | contentType := multiPartWriter.FormDataContentType() 189 | err = p.pulsar.Client.PostWithMultiPart(endpoint, nil, bodyBuf, contentType) 190 | if err != nil { 191 | return err 192 | } 193 | 194 | return nil 195 | } 196 | 197 | func (p packages) List(typeName, namespace string) ([]string, error) { 198 | var packageList []string 199 | endpoint := p.pulsar.endpoint(p.basePath, typeName, namespace) 200 | err := p.pulsar.Client.Get(endpoint, &packageList) 201 | return packageList, err 202 | } 203 | 204 | func (p packages) ListVersions(packageURL string) ([]string, error) { 205 | var versionList []string 206 | packageName, err := utils.GetPackageName(packageURL) 207 | if err != nil { 208 | return versionList, err 209 | } 210 | endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), 211 | packageName.GetNamespace(), packageName.GetName()) 212 | err = p.pulsar.Client.Get(endpoint, &versionList) 213 | return versionList, err 214 | } 215 | 216 | func (p packages) Delete(packageURL string) error { 217 | packageName, err := utils.GetPackageName(packageURL) 218 | if err != nil { 219 | return err 220 | } 221 | endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), 222 | packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion()) 223 | 224 | return p.pulsar.Client.Delete(endpoint) 225 | } 226 | 227 | func (p packages) GetMetadata(packageURL string) (utils.PackageMetadata, error) { 228 | var metadata utils.PackageMetadata 229 | packageName, err := utils.GetPackageName(packageURL) 230 | if err != nil { 231 | return metadata, err 232 | } 233 | endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), 234 | packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion(), "metadata") 235 | err = p.pulsar.Client.Get(endpoint, &metadata) 236 | return metadata, err 237 | } 238 | 239 | func (p packages) UpdateMetadata(packageURL, description, contact string, properties map[string]string) error { 240 | metadata := utils.PackageMetadata{ 241 | Description: description, 242 | Contact: contact, 243 | Properties: properties, 244 | } 245 | packageName, err := utils.GetPackageName(packageURL) 246 | if err != nil { 247 | return err 248 | } 249 | endpoint := p.pulsar.endpoint(p.basePath, string(packageName.GetType()), packageName.GetTenant(), 250 | packageName.GetNamespace(), packageName.GetName(), packageName.GetVersion(), "metadata") 251 | 252 | return p.pulsar.Client.Put(endpoint, &metadata) 253 | } 254 | --------------------------------------------------------------------------------