Skip to content

Commit 8014c81

Browse files
bruthbruth
committed
Add multi-region example
Signed-off-by: Byron Ruth <[email protected]>
1 parent fb5cae0 commit 8014c81

17 files changed

+788
-0
lines changed

‎examples/topologies/multi-region/cli/Dockerfile‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM golang:1.19-alpine3.17 AS build
2+
3+
RUN apk update && apk add git
4+
5+
RUN go install github.com/nats-io/nats-server/[email protected]
6+
RUN go install github.com/nats-io/natscli/nats@main
7+
RUN go install github.com/nats-io/nsc/[email protected]
8+
9+
FROM alpine:3.17
10+
11+
RUN apk add bash curl
12+
13+
COPY --from=build /go/bin/nats-server /usr/local/bin/
14+
COPY --from=build /go/bin/nats /usr/local/bin/
15+
COPY --from=build /go/bin/nsc /usr/local/bin/
16+
17+
WORKDIR /app
18+
19+
COPY . .
20+
21+
ENTRYPOINT ["bash"]
22+
23+
CMD ["main.sh"]

‎examples/topologies/multi-region/cli/GLOBAL.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"name": "GLOBAL",
3+
"subjects": [
4+
"js.in.global.>"
5+
],
6+
"retention": "limits",
7+
"max_consumers": -1,
8+
"max_msgs_per_subject": -1,
9+
"max_msgs": -1,
10+
"max_bytes": -1,
11+
"max_age": 0,
12+
"max_msg_size": -1,
13+
"storage": "file",
14+
"discard": "old",
15+
"num_replicas": 3,
16+
"duplicate_window": 120000000000,
17+
"sealed": false,
18+
"deny_delete": false,
19+
"deny_purge": false,
20+
"allow_rollup_hdrs": false,
21+
"allow_direct": false,
22+
"mirror_direct": false
23+
}

‎examples/topologies/multi-region/cli/ORDERS_CENTRAL.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"name": "ORDERS_CENTRAL",
3+
"subjects": [
4+
"js.in.orders_central"
5+
],
6+
"retention": "limits",
7+
"max_consumers": -1,
8+
"max_msgs_per_subject": -1,
9+
"max_msgs": -1,
10+
"max_bytes": -1,
11+
"max_age": 0,
12+
"max_msg_size": -1,
13+
"storage": "file",
14+
"discard": "old",
15+
"num_replicas": 3,
16+
"duplicate_window": 120000000000,
17+
"placement": {
18+
"cluster": "",
19+
"tags": [
20+
"region:central"
21+
]
22+
},
23+
"sources": [
24+
{
25+
"name": "ORDERS_EAST",
26+
"filter_subject": "js.in.orders_east"
27+
},
28+
{
29+
"name": "ORDERS_WEST",
30+
"filter_subject": "js.in.orders_west"
31+
}
32+
],
33+
"sealed": false,
34+
"deny_delete": false,
35+
"deny_purge": false,
36+
"allow_rollup_hdrs": false,
37+
"allow_direct": false,
38+
"mirror_direct": false
39+
}

‎examples/topologies/multi-region/cli/ORDERS_EAST.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"name": "ORDERS_EAST",
3+
"subjects": [
4+
"js.in.orders_east"
5+
],
6+
"retention": "limits",
7+
"max_consumers": -1,
8+
"max_msgs_per_subject": -1,
9+
"max_msgs": -1,
10+
"max_bytes": -1,
11+
"max_age": 0,
12+
"max_msg_size": -1,
13+
"storage": "file",
14+
"discard": "old",
15+
"num_replicas": 3,
16+
"duplicate_window": 120000000000,
17+
"placement": {
18+
"cluster": "",
19+
"tags": [
20+
"region:east"
21+
]
22+
},
23+
"sources": [
24+
{
25+
"name": "ORDERS_WEST",
26+
"filter_subject": "js.in.orders_west"
27+
},
28+
{
29+
"name": "ORDERS_CENTRAL",
30+
"filter_subject": "js.in.orders_central"
31+
}
32+
],
33+
"sealed": false,
34+
"deny_delete": false,
35+
"deny_purge": false,
36+
"allow_rollup_hdrs": false,
37+
"allow_direct": false,
38+
"mirror_direct": false
39+
}

‎examples/topologies/multi-region/cli/ORDERS_WEST.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"name": "ORDERS_WEST",
3+
"subjects": [
4+
"js.in.orders_west"
5+
],
6+
"retention": "limits",
7+
"max_consumers": -1,
8+
"max_msgs_per_subject": -1,
9+
"max_msgs": -1,
10+
"max_bytes": -1,
11+
"max_age": 0,
12+
"max_msg_size": -1,
13+
"storage": "file",
14+
"discard": "old",
15+
"num_replicas": 3,
16+
"duplicate_window": 120000000000,
17+
"placement": {
18+
"cluster": "",
19+
"tags": [
20+
"region:west"
21+
]
22+
},
23+
"sources": [
24+
{
25+
"name": "ORDERS_EAST",
26+
"filter_subject": "js.in.orders_east"
27+
},
28+
{
29+
"name": "ORDERS_CENTRAL",
30+
"filter_subject": "js.in.orders_central"
31+
}
32+
],
33+
"sealed": false,
34+
"deny_delete": false,
35+
"deny_purge": false,
36+
"allow_rollup_hdrs": false,
37+
"allow_direct": false,
38+
"mirror_direct": false
39+
}

‎examples/topologies/multi-region/cli/README.md‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Multi Region NATS Cluster
2+
3+
This demonstrates a very basic NATS Cluster using only routes across 3 regions.
4+
5+
This should be used in cases where a multi region setup is needed and one requires a stream to be globally deployed and have globally consistent message deduplication.
6+
7+
## Constraints
8+
9+
* This should be deployed in networks with generally below 100ms latency. For example in GCP using Tier-1 network connectivity in US east, west and central.
10+
* Streams should generally be R3 when stretched out of a single region to mitigate the big exposure they would have to latency if R5
11+
* Multi-region streams should not be the default, ideally these are only used for those cases where global deduplication is needed
12+
* For general streams where eventual consistency is acceptible streams should be bound to a region and replicated into others if desired
13+
14+
## Configuration guidelines
15+
16+
### Server Tags
17+
18+
Each server is tagged with a regional tag, for example, `region:east` or `region:west`. The servers are configured with anti-affinity on this tag using the following configuration:
19+
20+
```
21+
jetstream {
22+
unique_tag: "region:"
23+
}
24+
```
25+
26+
This ensures that streams that are not specifically bound to a single region will be split across the 3 regions.
27+
28+
### Configuring single region streams
29+
30+
```
31+
$ nats stream add --tag region:east --replicas 3 EAST
32+
...
33+
Cluster Information:
34+
35+
Name: c1
36+
Leader: n1-east
37+
Replica: n2-east, current, seen 1ms ago
38+
Replica: n3-east, current, seen 0s ago
39+
```
40+
41+
Note the servers are `n1-east`, `n2-east`, `n3-east`.
42+
43+
### Configuring a global stream
44+
45+
```
46+
$ nats stream add --replicas 3 GLOBAL
47+
...
48+
Cluster Information:
49+
50+
Name: c1
51+
Leader: n1-central
52+
Replica: n2-east, current, seen 0s ago
53+
Replica: n3-west, current, seen 0s ago
54+
55+
```
56+
57+
Note here we give no placement directive so the server will spread it across regions due to the `unique_tag` setting. Servers are `n1-central`, `n2-east` and `n3-west` - 1 per region.
58+
59+
### Configuring replicated streams
60+
61+
To facilitate an eventually consistent but single config set up we show how to create 3 streams:
62+
63+
* `ORDERS_EAST` listening on subjects `js.in.orders_east`
64+
* `ORDERS_WEST` listening on subjects `js.in.orders_west`
65+
* `ORDERS_CENTRAL` listening on subjects `js.in.orders.central`
66+
67+
We then use server mappings in each region to map `js.in.orders` to the in-region subject:
68+
69+
```
70+
accounts {
71+
one: {
72+
jetstream: enabled
73+
mappings: {
74+
js.in.orders: js.in.orders_central
75+
}
76+
}
77+
}
78+
```
79+
80+
We can now publish to one subject and depend on which region we connect to the local stream will handle - then replicate - the data:
81+
82+
```
83+
$ nats --context contexts/central-user.json req js.in.orders 1
84+
{"stream":"ORDERS_CENTRAL", "seq":4}
85+
86+
$ nats --context contexts/east-user.json req js.in.orders 1
87+
{"stream":"ORDERS_EAST", "seq":5}
88+
```
89+
90+
After a short while all streams hold the same data.
91+
92+
```
93+
╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
94+
│ Stream Report │
95+
├────────────────┬─────────┬──────────────────────┬───────────┬──────────┬───────┬──────┬─────────┬─────────────────────────────────────┤
96+
│ Stream │ Storage │ Placement │ Consumers │ Messages │ Bytes │ Lost │ Deleted │ Replicas │
97+
├────────────────┼─────────┼──────────────────────┼───────────┼──────────┼───────┼──────┼─────────┼─────────────────────────────────────┤
98+
│ GLOBAL │ File │ │ 0 │ 0 │ 0 B │ 0 │ 0 │ n1-central, n2-west*, n3-east │
99+
│ ORDERS_CENTRAL │ File │ tags: region:central │ 0 │ 5 │ 399 B │ 0 │ 0 │ n1-central, n2-central*, n3-central │
100+
│ ORDERS_EAST │ File │ tags: region:east │ 0 │ 5 │ 405 B │ 0 │ 0 │ n1-east*, n2-east, n3-east │
101+
│ ORDERS_WEST │ File │ tags: region:west │ 0 │ 5 │ 456 B │ 0 │ 0 │ n1-west*, n2-west, n3-west │
102+
╰────────────────┴─────────┴──────────────────────┴───────────┴──────────┴───────┴──────┴─────────┴─────────────────────────────────────╯
103+
```
104+
105+
This allows portable publishers to be built, consumers will need to know the region they are in and bind to the correct stream.
106+
107+

‎examples/topologies/multi-region/cli/configs/cluster-central.conf‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
port: 4222
2+
monitor_port: 8222
3+
server_name: $NAME
4+
client_advertise: $ADVERTISE
5+
6+
server_tags: [$GATEWAY, $REGION]
7+
8+
cluster {
9+
port: 6222
10+
11+
routes = [
12+
nats-route://n1-east:6222
13+
nats-route://n2-east:6222
14+
nats-route://n3-east:6222
15+
nats-route://n1-west:6222
16+
nats-route://n2-west:6222
17+
nats-route://n3-west:6222
18+
nats-route://n1-central:6222
19+
nats-route://n2-central:6222
20+
nats-route://n3-central:6222
21+
]
22+
}
23+
24+
leafnodes {
25+
port: 7422
26+
}
27+
28+
gateway {
29+
name: $GATEWAY
30+
port: 7222
31+
}
32+
33+
jetstream {
34+
store_dir: /data
35+
unique_tag: "region:"
36+
}
37+
38+
accounts {
39+
one: {
40+
jetstream: enabled
41+
users = [
42+
{user: one, password: secret}
43+
]
44+
mappings: {
45+
js.in.orders: js.in.orders_central
46+
}
47+
}
48+
49+
system: {
50+
users = [
51+
{user: system, password: secret}
52+
]
53+
}
54+
}
55+
56+
system_account: system

‎examples/topologies/multi-region/cli/configs/cluster-east.conf‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
port: 4222
2+
monitor_port: 8222
3+
server_name: $NAME
4+
client_advertise: $ADVERTISE
5+
6+
server_tags: [$GATEWAY, $REGION]
7+
8+
cluster {
9+
port: 6222
10+
11+
routes = [
12+
nats-route://n1-east:6222
13+
nats-route://n2-east:6222
14+
nats-route://n3-east:6222
15+
nats-route://n1-west:6222
16+
nats-route://n2-west:6222
17+
nats-route://n3-west:6222
18+
nats-route://n1-central:6222
19+
nats-route://n2-central:6222
20+
nats-route://n3-central:6222
21+
]
22+
}
23+
24+
leafnodes {
25+
port: 7422
26+
}
27+
28+
gateway {
29+
name: $GATEWAY
30+
port: 7222
31+
}
32+
33+
jetstream {
34+
store_dir: /data
35+
unique_tag: "region:"
36+
}
37+
38+
accounts {
39+
one: {
40+
jetstream: enabled
41+
users = [
42+
{user: one, password: secret}
43+
]
44+
45+
mappings: {
46+
js.in.orders: js.in.orders_east
47+
}
48+
}
49+
50+
system: {
51+
users = [
52+
{user: system, password: secret}
53+
]
54+
}
55+
}
56+
57+
system_account: system

0 commit comments

Comments
 (0)
⚡