├── .gitignore ├── LICENSE ├── README.md ├── nats.conf ├── nats.yml ├── route_checker └── main.go └── ssl.cnf /.gitignore: -------------------------------------------------------------------------------- 1 | *.pem 2 | *.csr 3 | *.srl 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kubernetes-nats-cluster 2 | NATS cluster on top of Kubernetes made easy. 3 | 4 | **THIS PROJECT HAS BEEN ARCHIVED. SEE https://github.com/nats-io/nats-operator** 5 | 6 | **NOTE:** This repository provides a configurable way to deploy secure, available 7 | and scalable NATS clusters. However, [a _smarter_ solution](https://github.com/pires/nats-operator) 8 | in on the way (see [#5](https://github.com/pires/kubernetes-nats-cluster/issues/5)). 9 | 10 | ## Pre-requisites 11 | 12 | * Kubernetes cluster v1.8+ - tested with v1.9.0 on top of [Vagrant + CoreOS](https://github.com/pires/kubernetes-vagrant-coreos-cluster) 13 | * At least 3 nodes available (see [Pod anti-affinity](#pod-anti-affinity)) 14 | * `kubectl` configured to access your cluster master API Server 15 | * `openssl` for TLS certificate generation 16 | 17 | ## Deploy 18 | 19 | We will be deploying a cluster of 3 NATS instances, with the following set-up: 20 | - TLS on for clients, but not clustering because peer-auth requires real SANS DNS in certificate 21 | - NATS client credentials: `nats_client_user:nats_client_pwd` 22 | - NATS route/cluster credentials: `nats_route_user:nats_route_pwd` 23 | - Logging: `debug:false`, `trace:true`, `logtime:true` 24 | 25 | First, make sure to change `nats.conf` according to your needs. 26 | Then create a Kubernetes configmap to store it: 27 | ```bash 28 | kubectl create configmap nats-config --from-file nats.conf 29 | ``` 30 | 31 | Next, we need to generate valid TLS artifacts: 32 | ```bash 33 | openssl genrsa -out ca-key.pem 2048 34 | openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" 35 | openssl genrsa -out nats-key.pem 2048 36 | openssl req -new -key nats-key.pem -out nats.csr -subj "/CN=kube-nats" -config ssl.cnf 37 | openssl x509 -req -in nats.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out nats.pem -days 3650 -extensions v3_req -extfile ssl.cnf 38 | ``` 39 | 40 | Then, it's time to create a couple Kubernetes secrets to store the TLS artifacts: 41 | - `tls-nats-server` for the NATS server TLS setup 42 | - `tls-nats-client` for NATS client apps setup - one will need it to validate the self-signed certificate 43 | used to secure NATS server 44 | ```bash 45 | kubectl create secret generic tls-nats-server --from-file nats.pem --from-file nats-key.pem --from-file ca.pem 46 | kubectl create secret generic tls-nats-client --from-file ca.pem 47 | ``` 48 | 49 | **ATTENTION:** Both using self-signed certificates and using the same certificates for securing 50 | client and cluster connections is a significant security compromise. But for the sake of showing 51 | how it can be done, I'm fine with doing just that. 52 | In an ideal scenario, there should be: 53 | - One centralized PKI/CA 54 | - One certificate for securing NATS route/cluster connections 55 | - One certificate for securing NATS client connections 56 | - TLS route/cluster authentication should be enforced, so one TLS certificate per route/cluster peer 57 | - TLS client authentication should be enforced, so one TLS certificate per client 58 | 59 | And finally, we deploy NATS: 60 | ```bash 61 | kubectl create -f nats.yml 62 | ``` 63 | 64 | Logs should be enough to make sure everything is working as expected: 65 | ``` 66 | $ kubectl logs -f nats-0 67 | [1] 2017/12/17 12:38:37.801139 [INF] Starting nats-server version 1.0.4 68 | [1] 2017/12/17 12:38:37.801449 [INF] Starting http monitor on 0.0.0.0:8222 69 | [1] 2017/12/17 12:38:37.801580 [INF] Listening for client connections on 0.0.0.0:4242 70 | [1] 2017/12/17 12:38:37.801772 [INF] TLS required for client connections 71 | [1] 2017/12/17 12:38:37.801778 [INF] Server is ready 72 | [1] 2017/12/17 12:38:37.802078 [INF] Listening for route connections on 0.0.0.0:6222 73 | [1] 2017/12/17 12:38:38.874497 [TRC] 10.244.1.3:33494 - rid:1 - ->> [CONNECT {"verbose":false,"pedantic":false,"user":"nats_route_user","pass":"nats_route_pwd","tls_required":true,"name":"KGMPnL89We3gFLEjmp8S5J"}] 74 | [1] 2017/12/17 12:38:38.956806 [TRC] 10.244.74.2:46018 - rid:3 - ->> [CONNECT {"verbose":false,"pedantic":false,"user":"nats_route_user","pass":"nats_route_pwd","tls_required":true,"name":"Skc5mx9enWrGPIQhyE7uzR"}] 75 | [1] 2017/12/17 12:38:39.951160 [TRC] 10.244.1.4:46242 - rid:4 - ->> [CONNECT {"verbose":false,"pedantic":false,"user":"nats_route_user","pass":"nats_route_pwd","tls_required":true,"name":"0kaCfF3BU8g92snOe34251"}] 76 | [1] 2017/12/17 12:40:38.956203 [TRC] 10.244.74.2:46018 - rid:3 - <<- [PING] 77 | [1] 2017/12/17 12:40:38.958279 [TRC] 10.244.74.2:46018 - rid:3 - ->> [PING] 78 | [1] 2017/12/17 12:40:38.958300 [TRC] 10.244.74.2:46018 - rid:3 - <<- [PONG] 79 | [1] 2017/12/17 12:40:38.961791 [TRC] 10.244.74.2:46018 - rid:3 - ->> [PONG] 80 | [1] 2017/12/17 12:40:39.951421 [TRC] 10.244.1.4:46242 - rid:4 - <<- [PING] 81 | [1] 2017/12/17 12:40:39.952578 [TRC] 10.244.1.4:46242 - rid:4 - ->> [PONG] 82 | [1] 2017/12/17 12:40:39.952594 [TRC] 10.244.1.4:46242 - rid:4 - ->> [PING] 83 | [1] 2017/12/17 12:40:39.952598 [TRC] 10.244.1.4:46242 - rid:4 - <<- [PONG] 84 | ``` 85 | 86 | ## Scale 87 | 88 | **WARNING:** Due to the [Pod anti-affinity](#pod-anti-affinity) rule, for scaling up to _n_ NATS 89 | instances, one needs _n_ available Kubernetes nodes. 90 | 91 | ``` 92 | kubectl scale statefulsets nats --replicas 5 93 | ``` 94 | 95 | Did it work? 96 | 97 | ``` 98 | NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE 99 | svc/kubernetes ClusterIP 10.100.0.1 443/TCP 1h 100 | svc/nats ClusterIP None 4222/TCP,6222/TCP,8222/TCP 4m 101 | 102 | NAME READY STATUS RESTARTS AGE 103 | po/nats-0 1/1 Running 0 4m 104 | po/nats-1 1/1 Running 0 4m 105 | po/nats-2 1/1 Running 0 4m 106 | po/nats-3 1/1 Running 0 7s 107 | po/nats-4 1/1 Running 0 6s 108 | ``` 109 | 110 | ## Access the service 111 | 112 | *Don't forget* that services in Kubernetes are only acessible from containers in the cluster. 113 | 114 | In this case, we're using a [`headless service`](http://kubernetes.io/v1.1/docs/user-guide/services.html#headless-services). 115 | 116 | Just point your client apps to: 117 | ``` 118 | nats:4222 119 | ``` 120 | 121 | 122 | 123 | ## Pod anti-affinity 124 | 125 | 126 | One of the main advantages of running NATS on top of Kubernetes is how resilient the cluster becomes, 127 | particularly during node restarts. However if all NATS pods are scheduled onto the same node(s), this 128 | advantage decreases significantly and may even result in service downtime. 129 | 130 | It is then **highly recommended** that one adopts [pod anti-affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity-beta-feature) 131 | in order to increase availability. This is enabled by default (see `nats.yml`). 132 | -------------------------------------------------------------------------------- /nats.conf: -------------------------------------------------------------------------------- 1 | listen: 0.0.0.0:4222 2 | http: 0.0.0.0:8222 3 | 4 | tls { 5 | cert_file: "/etc/nats/tls/nats.pem" 6 | key_file: "/etc/nats/tls/nats-key.pem" 7 | timeout: 2 8 | } 9 | 10 | # Authorization for client connections 11 | authorization { 12 | user: nats_client_user 13 | # ./util/mkpasswd -p T0pS3cr3t 14 | password: nats_client_pwd 15 | timeout: 1 16 | } 17 | 18 | # Cluster definition 19 | cluster { 20 | listen: 0.0.0.0:6222 21 | 22 | # Disabled this because peer-auth requires real SANS DNS in certificate 23 | #tls { 24 | # cert_file: "/etc/nats/tls/nats.pem" 25 | # key_file: "/etc/nats/tls/nats-key.pem" 26 | # ca_file: "/etc/nats/tls/ca.pem" 27 | # timeout: 5 28 | #} 29 | 30 | # Authorization for route connections 31 | authorization { 32 | user: nats_route_user 33 | password: nats_route_pwd 34 | timeout: 0.5 35 | } 36 | 37 | # Routes are actively solicited and connected to from this server. 38 | # Other servers can connect to us if they supply the correct credentials 39 | # in their routes definitions from above. 40 | routes = [ 41 | nats://nats_route_user:nats_route_pwd@nats:6222 42 | ] 43 | } 44 | 45 | # logging options 46 | debug: false 47 | trace: true 48 | logtime: true 49 | 50 | # Some system overides 51 | 52 | # max_connections 53 | max_connections: 100 54 | 55 | # maximum protocol control line 56 | max_control_line: 512 57 | 58 | # maximum payload 59 | max_payload: 65536 60 | 61 | # Duration the server can block on a socket write to a client. Exceeding the 62 | # deadline will designate a client as a slow consumer. 63 | write_deadline: "2s" -------------------------------------------------------------------------------- /nats.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: nats 5 | labels: 6 | component: nats 7 | spec: 8 | selector: 9 | component: nats 10 | clusterIP: None 11 | ports: 12 | - name: client 13 | port: 4222 14 | - name: cluster 15 | port: 6222 16 | - name: monitor 17 | port: 8222 18 | --- 19 | apiVersion: apps/v1beta1 20 | kind: StatefulSet 21 | metadata: 22 | name: nats 23 | labels: 24 | component: nats 25 | spec: 26 | serviceName: nats 27 | replicas: 3 28 | template: 29 | metadata: 30 | labels: 31 | component: nats 32 | spec: 33 | affinity: 34 | podAntiAffinity: 35 | requiredDuringSchedulingIgnoredDuringExecution: 36 | - labelSelector: 37 | matchExpressions: 38 | - key: component 39 | operator: In 40 | values: 41 | - nats 42 | topologyKey: kubernetes.io/hostname 43 | containers: 44 | - name: nats 45 | image: nats:1.0.4 46 | args: [ "--config", "/etc/nats/config/nats.conf"] 47 | volumeMounts: 48 | - name: tls-volume 49 | mountPath: /etc/nats/tls 50 | - name: config-volume 51 | mountPath: /etc/nats/config 52 | ports: 53 | - containerPort: 4222 54 | name: client 55 | - containerPort: 6222 56 | name: cluster 57 | - containerPort: 8222 58 | name: monitor 59 | livenessProbe: 60 | httpGet: 61 | path: / 62 | port: 8222 63 | initialDelaySeconds: 10 64 | timeoutSeconds: 5 65 | volumes: 66 | - name: tls-volume 67 | secret: 68 | secretName: tls-nats-server 69 | - name: config-volume 70 | configMap: 71 | name: nats-config 72 | -------------------------------------------------------------------------------- /route_checker/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "io/ioutil" 7 | "log" 8 | "net" 9 | "net/http" 10 | ) 11 | 12 | var ( 13 | lookupName = flag.String("lookup", "nats", "Lookup name") 14 | server = flag.String("server", "http://localhost:8222", "NATS URL to query") 15 | ) 16 | 17 | type routez struct { 18 | RoutesCount int `json:"num_routes"` 19 | } 20 | 21 | func countInstancesInCluster(lookupName string) (int, error) { 22 | _, srvRecords, err := net.LookupSRV("", "", lookupName) 23 | if err != nil { 24 | return 0, err 25 | } 26 | 27 | return len(srvRecords), nil 28 | } 29 | 30 | func main() { 31 | flag.Parse() 32 | 33 | // if we're the only instance in the cluster, we're good to go 34 | // otherwise, we need to check if there are any routes established 35 | // and if not, exit with error. 36 | instances, err := countInstancesInCluster(*lookupName) 37 | if err != nil { 38 | log.Fatalf("There was an error while verifiying if this is the first instance in the cluster: %s\n", err.Error()) 39 | } else { 40 | log.Printf("There are %d instances in the cluster.", instances) 41 | } 42 | 43 | if instances > 1 { 44 | // query NATS monitoring endpoint 45 | url := *server + "/routez" 46 | log.Printf("Querying server %s..\n", url) 47 | res, err := http.Get(url) 48 | if err != nil { 49 | log.Fatalf("There was an error while querying %s: %s\n", url, err.Error()) 50 | } 51 | 52 | body, err := ioutil.ReadAll(res.Body) 53 | if err != nil { 54 | log.Fatalf("There was an error while reading response: %s\n", err.Error()) 55 | } 56 | 57 | var data routez 58 | if err := json.Unmarshal(body, &data); err != nil { 59 | log.Fatalf("There was an error while decoding response: %s\n", err.Error()) 60 | } 61 | 62 | // if enough routes aren't established, exit with error 63 | if data.RoutesCount < instances-1 { 64 | log.Fatalf("There aren't enough routes established. Only %d out of %d.", data.RoutesCount, instances-1) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /ssl.cnf: -------------------------------------------------------------------------------- 1 | [req] 2 | req_extensions = v3_req 3 | distinguished_name = req_distinguished_name 4 | [req_distinguished_name] 5 | [ v3_req ] 6 | basicConstraints = CA:FALSE 7 | keyUsage = nonRepudiation, digitalSignature, keyEncipherment 8 | subjectAltName = @alt_names 9 | [alt_names] 10 | DNS.1 = nats 11 | 12 | --------------------------------------------------------------------------------