1
2
3
4
5
6
7
作者：李晓辉

联系方式：

1. 微信：Lxh_Chat

2. 邮箱：939958092@qq.com

Ceph RADOS 网关支持在全局命名空间进行多站点部署，RADOS 网关因此可以在多个红帽 Ceph 存储集群之间自动复制对象数据。一个比较常见的支持用例是在地理上分隔的集群之间进行主动/主动复制，以便于灾难恢复。

多站点组件

  • Zone（区域）：Zone是一个实际的集群，由至少一个RADOS网关实例提供服务，拥有独立的Ceph存储集群。
  • Zonegroup（区域组）：Zonegroup是包含多个Zone的逻辑组，其中包含一个主Zone（Master Zone）和多个从Zone（Secondary Zone），用于数据的同步和故障切换。
  • Realm（领域）：Realm代表一个全局唯一的命名空间，由多个Zonegroup组成，管理着不同Zonegroup的配置和状态。

部署第一个站点

创建realm

创建一个east-china的域作为默认站点

1
[root@serverc ~]# systemctl stop firewalld.service
1
2
3
4
5
6
7
[root@serverc ~]# radosgw-admin realm create --rgw-realm=east-china --default
{
    "id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
    "name": "east-china",
    "current_period": "56816fef-0ee7-45e3-9c41-99b45b20eb2d",
    "epoch": 1
}

创建zonegroup

创建一个名为shanghai且用serverc主机上的rgw作为默认服务的zonegroup

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
[root@serverc ~]# radosgw-admin zonegroup create --rgw-zonegroup=shanghai --endpoints=http://serverc.lab.example.com:80 --master --default
{
    "id": "5ba1db70-05fb-40bf-8a96-4e85e2848e1f",
    "name": "shanghai",
    "api_name": "shanghai",
    "is_master": "true",
    "endpoints": [
        "http://serverc.lab.example.com:80"
    ],
    "hostnames": [],
    "hostnames_s3website": [],
    "master_zone": "",
    "zones": [],
    "placement_targets": [],
    "default_placement": "",
    "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
    "sync_policy": {
        "groups": []
    }
}

创建Master Zone

在shanghai组中创建一个名为shanghai-baoshan且指向serverc的默认服务作为主zone

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
[root@serverc ~]# radosgw-admin zone create --rgw-zonegroup=shanghai --rgw-zone=shanghai-baoshan --endpoints=http://serverc.lab.example.com:80 --master --default --access-key=replication --secret=secret
{
    "id": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
    "name": "shanghai-baoshan",
    "domain_root": "shanghai-baoshan.rgw.meta:root",
    "control_pool": "shanghai-baoshan.rgw.control",
    "gc_pool": "shanghai-baoshan.rgw.log:gc",
    "lc_pool": "shanghai-baoshan.rgw.log:lc",
    "log_pool": "shanghai-baoshan.rgw.log",
    "intent_log_pool": "shanghai-baoshan.rgw.log:intent",
    "usage_log_pool": "shanghai-baoshan.rgw.log:usage",
    "roles_pool": "shanghai-baoshan.rgw.meta:roles",
    "reshard_pool": "shanghai-baoshan.rgw.log:reshard",
    "user_keys_pool": "shanghai-baoshan.rgw.meta:users.keys",
    "user_email_pool": "shanghai-baoshan.rgw.meta:users.email",
    "user_swift_pool": "shanghai-baoshan.rgw.meta:users.swift",
    "user_uid_pool": "shanghai-baoshan.rgw.meta:users.uid",
    "otp_pool": "shanghai-baoshan.rgw.otp",
    "system_key": {
        "access_key": "replication",
        "secret_key": "secret"
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "shanghai-baoshan.rgw.buckets.index",
                "storage_classes": {
                    "STANDARD": {
                        "data_pool": "shanghai-baoshan.rgw.buckets.data"
                    }
                },
                "data_extra_pool": "shanghai-baoshan.rgw.buckets.non-ec",
                "index_type": 0
            }
        }
    ],
    "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
    "notif_pool": "shanghai-baoshan.rgw.log:notif"
}

创建系统用户

创建一个名为lxh的系统用户用于访问区域池，用户的密钥必须与和该区域配置的密钥相匹配

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
[root@serverc ~]# radosgw-admin user create --uid="lxh" --system --display-name="Replication user" --secret=secret --access-key=replication
{
    "user_id": "lxh",
    "display_name": "Replication user",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lxh",
            "access_key": "replication",
            "secret_key": "secret"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "system": "true",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

更新配置

将域配置更改提交到该period。注意与当前配置相关联的period id。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
[root@serverc ~]# radosgw-admin period update --commit
{
    "id": "a6256afb-8532-4891-8fab-5df80836041d",
    "epoch": 1,
    "predecessor_uuid": "56816fef-0ee7-45e3-9c41-99b45b20eb2d",
    "sync_status": [],
    "period_map": {
        "id": "a6256afb-8532-4891-8fab-5df80836041d",
        "zonegroups": [
            {
                "id": "5ba1db70-05fb-40bf-8a96-4e85e2848e1f",
                "name": "shanghai",
                "api_name": "shanghai",
                "is_master": "true",
                "endpoints": [
                    "http://serverc.lab.example.com:80"
                ],
                "hostnames": [],
                "hostnames_s3website": [],
                "master_zone": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                "zones": [
                    {
                        "id": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                        "name": "shanghai-baoshan",
                        "endpoints": [
                            "http://serverc.lab.example.com:80"
                        ],
                        "log_meta": "false",
                        "log_data": "false",
                        "bucket_index_max_shards": 11,
                        "read_only": "false",
                        "tier_type": "",
                        "sync_from_all": "true",
                        "sync_from": [],
                        "redirect_zone": ""
                    }
                ],
                "placement_targets": [
                    {
                        "name": "default-placement",
                        "tags": [],
                        "storage_classes": [
                            "STANDARD"
                        ]
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
                "sync_policy": {
                    "groups": []
                }
            }
        ],
        "short_zone_ids": [
            {
                "key": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                "val": 4285768614
            }
        ]
    },
    "master_zonegroup": "5ba1db70-05fb-40bf-8a96-4e85e2848e1f",
    "master_zone": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        }
    },
    "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
    "realm_name": "east-china",
    "realm_epoch": 2
}

部署RGW网关

在上面配置的是80端口，这里要相匹配

在主机上部署了一个进程，此进程位于shanghai的realm以及shanghai-baoshan的zone中

1
2
[root@serverc ~]# ceph orch apply rgw baoshan-zone --realm=east-china --zone=shanghai-baoshan --placement="1 serverc.lab.example.com"
Scheduled rgw.baoshan-zone update...

在数据库中标记本站点zone名称

1
2
3
[root@serverc ~]# ceph config set client.rgw rgw_zone shanghai-baoshan
[root@serverc ~]# ceph config get client.rgw rgw_zone
shanghai-baoshan

部署第二个站点

第二个站点是第二套集群

拉取源站点信息

从 serverc 节点拉取域和period配置。使用 repl.user 的凭据进行身份验证。验证当前的period id 与 serverc 节点的period id 是否period。

1
[root@serverf ~]# systemctl stop firewalld.service
1
2
3
4
5
6
7
8
9
[root@serverf ~]# radosgw-admin realm pull --url=http://serverc.lab.example.com:80 --secret=secret --access-key=replication
2023-09-08T18:58:29.833-0400 7f844d8b7380  1 error read_lastest_epoch .rgw.root:periods.a6256afb-8532-4891-8fab-5df80836041d.latest_epoch
2023-09-08T18:58:29.868-0400 7f844d8b7380  1 Set the period's master zonegroup 5ba1db70-05fb-40bf-8a96-4e85e2848e1f as the default
{
    "id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
    "name": "east-china",
    "current_period": "a6256afb-8532-4891-8fab-5df80836041d",
    "epoch": 2
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
[root@serverf ~]# radosgw-admin period pull --url=http://serverc.lab.example.com:80 --access-key=replication --secret-key=secret
{
    "id": "a6256afb-8532-4891-8fab-5df80836041d",
    "epoch": 1,
    "predecessor_uuid": "56816fef-0ee7-45e3-9c41-99b45b20eb2d",
    "sync_status": [],
    "period_map": {
        "id": "a6256afb-8532-4891-8fab-5df80836041d",
        "zonegroups": [
            {
                "id": "5ba1db70-05fb-40bf-8a96-4e85e2848e1f",
                "name": "shanghai",
                "api_name": "shanghai",
                "is_master": "true",
                "endpoints": [
                    "http://serverc.lab.example.com:80"
                ],
                "hostnames": [],
                "hostnames_s3website": [],
                "master_zone": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                "zones": [
                    {
                        "id": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                        "name": "shanghai-baoshan",
                        "endpoints": [
                            "http://serverc.lab.example.com:80"
                        ],
                        "log_meta": "false",
                        "log_data": "false",
                        "bucket_index_max_shards": 11,
                        "read_only": "false",
                        "tier_type": "",
                        "sync_from_all": "true",
                        "sync_from": [],
                        "redirect_zone": ""
                    }
                ],
                "placement_targets": [
                    {
                        "name": "default-placement",
                        "tags": [],
                        "storage_classes": [
                            "STANDARD"
                        ]
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
                "sync_policy": {
                    "groups": []
                }
            }
        ],
        "short_zone_ids": [
            {
                "key": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                "val": 4285768614
            }
        ]
    },
    "master_zonegroup": "5ba1db70-05fb-40bf-8a96-4e85e2848e1f",
    "master_zone": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        }
    },
    "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
    "realm_name": "east-china",
    "realm_epoch": 2
}

验证当前period id

1
2
3
4
[root@serverf ~]# radosgw-admin period get-current
{
    "current_period": "a6256afb-8532-4891-8fab-5df80836041d"
}

设置默认的realm和zonegroup

1
2
[root@serverf ~]# radosgw-admin realm default --rgw-realm=east-china
[root@serverf ~]# radosgw-admin zonegroup default --rgw-zonegroup=shanghai

创建默认zone

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
[root@serverf ~]# radosgw-admin zone create --rgw-zonegroup=shanghai --rgw-zone=shanghai-xuhui --endpoints=http://serverf.lab.example.com:80 --default --access-key=replication --secret=secret
2023-09-08T19:02:40.505-0400 7f53e2eec380  0 failed reading obj info from .rgw.root:zone_info.96796bde-405e-4cc2-acd2-1b4cc0bd650b: (2) No such file or directory
2023-09-08T19:02:40.505-0400 7f53e2eec380  0 WARNING: could not read zone params for zone id=96796bde-405e-4cc2-acd2-1b4cc0bd650b name=shanghai-baoshan
{
    "id": "e2e4994a-37c3-443b-af75-cab909d31360",
    "name": "shanghai-xuhui",
    "domain_root": "shanghai-xuhui.rgw.meta:root",
    "control_pool": "shanghai-xuhui.rgw.control",
    "gc_pool": "shanghai-xuhui.rgw.log:gc",
    "lc_pool": "shanghai-xuhui.rgw.log:lc",
    "log_pool": "shanghai-xuhui.rgw.log",
    "intent_log_pool": "shanghai-xuhui.rgw.log:intent",
    "usage_log_pool": "shanghai-xuhui.rgw.log:usage",
    "roles_pool": "shanghai-xuhui.rgw.meta:roles",
    "reshard_pool": "shanghai-xuhui.rgw.log:reshard",
    "user_keys_pool": "shanghai-xuhui.rgw.meta:users.keys",
    "user_email_pool": "shanghai-xuhui.rgw.meta:users.email",
    "user_swift_pool": "shanghai-xuhui.rgw.meta:users.swift",
    "user_uid_pool": "shanghai-xuhui.rgw.meta:users.uid",
    "otp_pool": "shanghai-xuhui.rgw.otp",
    "system_key": {
        "access_key": "replication",
        "secret_key": "secret"
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "shanghai-xuhui.rgw.buckets.index",
                "storage_classes": {
                    "STANDARD": {
                        "data_pool": "shanghai-xuhui.rgw.buckets.data"
                    }
                },
                "data_extra_pool": "shanghai-xuhui.rgw.buckets.non-ec",
                "index_type": 0
            }
        }
    ],
    "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
    "notif_pool": "shanghai-xuhui.rgw.log:notif"
}

更新配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
[root@serverf ~]# radosgw-admin period update --commit --rgw-zone=shanghai-xuhui
2023-09-08T19:03:20.307-0400 7fe5b545c380  1 Cannot find zone id=e2e4994a-37c3-443b-af75-cab909d31360 (name=shanghai-xuhui), switching to local zonegroup configuration
Sending period to new master zone 96796bde-405e-4cc2-acd2-1b4cc0bd650b
{
    "id": "a6256afb-8532-4891-8fab-5df80836041d",
    "epoch": 2,
    "predecessor_uuid": "56816fef-0ee7-45e3-9c41-99b45b20eb2d",
    "sync_status": [],
    "period_map": {
        "id": "a6256afb-8532-4891-8fab-5df80836041d",
        "zonegroups": [
            {
                "id": "5ba1db70-05fb-40bf-8a96-4e85e2848e1f",
                "name": "shanghai",
                "api_name": "shanghai",
                "is_master": "true",
                "endpoints": [
                    "http://serverc.lab.example.com:80"
                ],
                "hostnames": [],
                "hostnames_s3website": [],
                "master_zone": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                "zones": [
                    {
                        "id": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                        "name": "shanghai-baoshan",
                        "endpoints": [
                            "http://serverc.lab.example.com:80"
                        ],
                        "log_meta": "false",
                        "log_data": "true",
                        "bucket_index_max_shards": 11,
                        "read_only": "false",
                        "tier_type": "",
                        "sync_from_all": "true",
                        "sync_from": [],
                        "redirect_zone": ""
                    },
                    {
                        "id": "e2e4994a-37c3-443b-af75-cab909d31360",
                        "name": "shanghai-xuhui",
                        "endpoints": [
                            "http://serverf.lab.example.com:80"
                        ],
                        "log_meta": "false",
                        "log_data": "true",
                        "bucket_index_max_shards": 11,
                        "read_only": "false",
                        "tier_type": "",
                        "sync_from_all": "true",
                        "sync_from": [],
                        "redirect_zone": ""
                    }
                ],
                "placement_targets": [
                    {
                        "name": "default-placement",
                        "tags": [],
                        "storage_classes": [
                            "STANDARD"
                        ]
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
                "sync_policy": {
                    "groups": []
                }
            }
        ],
        "short_zone_ids": [
            {
                "key": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
                "val": 4285768614
            },
            {
                "key": "e2e4994a-37c3-443b-af75-cab909d31360",
                "val": 2203885415
            }
        ]
    },
    "master_zonegroup": "5ba1db70-05fb-40bf-8a96-4e85e2848e1f",
    "master_zone": "96796bde-405e-4cc2-acd2-1b4cc0bd650b",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "check_on_raw": false,
            "max_size": -1,
            "max_size_kb": 0,
            "max_objects": -1
        }
    },
    "realm_id": "e01eccbf-a72c-4571-aab1-dc7dc8502d23",
    "realm_name": "east-china",
    "realm_epoch": 2
}

在数据库中标记本站点zone名称

1
2
3
[root@serverf ~]# ceph config set client.rgw rgw_zone shanghai-xuhui
[root@serverf ~]# ceph config get client.rgw rgw_zone
shanghai-xuhui

部署RGW网关

1
2
[root@serverf ~]# ceph orch apply rgw shanghai-xuhui --realm=east-china --zone=shanghai-xuhui --placement="1 serverf.lab.example.com"
Scheduled rgw.shanghai-xuhui update...

再次验证period

1
2
3
4
[root@serverf ~]# radosgw-admin period get-current
{
    "current_period": "a6256afb-8532-4891-8fab-5df80836041d"
}

查询同步状态

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@serverf ~]# radosgw-admin sync status
          realm e01eccbf-a72c-4571-aab1-dc7dc8502d23 (east-china)
      zonegroup 5ba1db70-05fb-40bf-8a96-4e85e2848e1f (shanghai)
           zone e2e4994a-37c3-443b-af75-cab909d31360 (shanghai-xuhui)
  metadata sync syncing
                full sync: 0/64 shards
                incremental sync: 64/64 shards
                metadata is caught up with master
      data sync source: 96796bde-405e-4cc2-acd2-1b4cc0bd650b (shanghai-baoshan)
                        syncing
                        full sync: 0/128 shards
                        incremental sync: 128/128 shards
                        data is caught up with source

测试同步

在第一个站点上创建一个账号

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
[root@serverc ~]# radosgw-admin user create --uid="xiaohui" --display-name="Xiaohui Li" --access=full
{
    "user_id": "xiaohui",
    "display_name": "Xiaohui Li",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "xiaohui",
            "access_key": "K0HV3YFM109WTLNP2OJS",
            "secret_key": "Rq8UZwaafPsaCaoJlu1Qi2jDx0Gmz8HD5UGBqDeJ"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

迅速去第二个站点查询同步状态以及账号列表

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@serverf ~]# radosgw-admin sync status
          realm e01eccbf-a72c-4571-aab1-dc7dc8502d23 (east-china)
      zonegroup 5ba1db70-05fb-40bf-8a96-4e85e2848e1f (shanghai)
           zone e2e4994a-37c3-443b-af75-cab909d31360 (shanghai-xuhui)
  metadata sync syncing
                full sync: 0/64 shards
                incremental sync: 64/64 shards
                metadata is behind on 1 shards
                behind shards: [24]
                oldest incremental change not applied: 2023-09-08T19:08:42.531382-0400 [24]
      data sync source: 96796bde-405e-4cc2-acd2-1b4cc0bd650b (shanghai-baoshan)
                        syncing
                        full sync: 0/128 shards
                        incremental sync: 128/128 shards
                        data is caught up with source

我们发现第二个集群上已经出现了数据

1
2
3
4
5
[root@serverf ~]# radosgw-admin user list
[
    "lxh",
    "xiaohui"
]