1
2
3
4
5
6
7
作者：李晓辉

联系方式：

1. 微信：Lxh_Chat

2. 邮箱：939958092@qq.com

RADOS网关中的Amazon S3 API

Amazon S3 界面会定义命名空间，其中的对象作为存储桶进行存储。要使用 S3 API 访问和管理对象和存储桶，应用需通过 RADOS 网关用户进行身份验证。每个用户都有一个识别用户的 access key 和一个对用户进行身份验证的 secret key。

使用 Amazon S3 API 时，需考虑对象和元数据的大小限制：

  • 对象大小介于 0B 和 5 TB 之间。

  • 单次上传操作的最大大小为 5GB。

  • 使用分段上传功能可上传 100MB 以上的对象。

  • 单个 HTTP 请求中的元数据大小最大为 16,000 字节。

创建 Amazon S3 API 用户

创建 RADOS 网关用户时，需要用到 --uid 和 --display-name 两个选项，并且要指定唯一的帐户名和人性化显示名。使用 --access-key 和 --secret 选项可为 RADOS 用户指定自定义 AWS 帐户和机密密钥。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
[root@serverc ~]# radosgw-admin user create --uid=lxh \
--display-name="Xiaohui Li" --email="939958092@qq.com" \
--access-key=lixiaohui --secret=lixiaohui
{
    "user_id": "lxh",
    "display_name": "Xiaohui Li",
    "email": "939958092@qq.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lxh",
            "access_key": "lixiaohui",
            "secret_key": "lixiaohui"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

如果未指定访问密钥和机密密钥，可通过 radosgw-admin 命令自动生成，并显示在输出中。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
[root@serverc ~]# radosgw-admin user create --uid=lxhuser1 \
--display-name="Xiaohui Li" --email="1234@qq.com"
{
    "user_id": "lxhuser1",
    "display_name": "Xiaohui Li",
    "email": "1234@qq.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lxhuser1",
            "access_key": "CMOGETUJ262DWDMEDQIL",
            "secret_key": "xCpI9wVJmX4ALhobBMhOEC6PofnXUXHUe0pBMVb6"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

由 radosgw-admin 命令自动生成的访问密钥和机密密钥可能包含 JSON 转义字符 ()。客户端可能无法正确处理此字符。建议重新生成或手动指定密钥以避免此问题。

管理Ceph对象网关用户

要仅重新生成现有用户的机密密钥，需使用 radosgw-admin key create 命令及 --gen-secret 选项。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
[root@serverc ~]# radosgw-admin key create --uid=lxhuser1 --gen-access-key --gen-secret
{
    "user_id": "lxhuser1",
    "display_name": "Xiaohui Li",
    "email": "1234@qq.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lxhuser1",
            "access_key": "39N78NRIPBGEN6X5MM5K",
            "secret_key": "sA9ii3SF1JKLEshiSrtMonsqiismmZtGScepNqar"
        },
        {
            "user": "lxhuser1",
            "access_key": "JAFF0MM293F6KCL9VA64",
            "secret_key": "TKbxlHVAKvlCSI0FnHaXvAMEiAfrhlUbh9YmyotO"
        }

要从用户删除访问密钥和相关的机密密钥，请使用 radosgw-admin key rm 命令及 --access-key 选项。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@serverc ~]# radosgw-admin key rm --uid=lxhuser1 --access-key=39N78NRIPBGEN6X5MM5K
{
    "user_id": "lxhuser1",
    "display_name": "Xiaohui Li",
    "email": "1234@qq.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lxhuser1",
            "access_key": "JAFF0MM293F6KCL9VA64",
            "secret_key": "TKbxlHVAKvlCSI0FnHaXvAMEiAfrhlUbh9YmyotO"
        }
    ],

使用 radosgw-admin user suspend 和 radosgw-admin user enable 命令可临时禁用和启用 RADOS 网关用户

要注意下方是suspend后面的值

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@serverc ~]# radosgw-admin user suspend --uid=lxhuser1
{
    "user_id": "lxhuser1",
    "display_name": "Xiaohui Li",
    "email": "1234@qq.com",
    "suspended": 1,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lxhuser1",
            "access_key": "JAFF0MM293F6KCL9VA64",
            "secret_key": "TKbxlHVAKvlCSI0FnHaXvAMEiAfrhlUbh9YmyotO"
        }
    ],

再启用用户

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@serverc ~]# radosgw-admin user enable --uid=lxhuser1
{
    "user_id": "lxhuser1",
    "display_name": "Xiaohui Li",
    "email": "1234@qq.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lxhuser1",
            "access_key": "JAFF0MM293F6KCL9VA64",
            "secret_key": "TKbxlHVAKvlCSI0FnHaXvAMEiAfrhlUbh9YmyotO"
        }

可修改用户信息，如电子邮件、显示名、密钥和访问控制级别。访问控制级别有：read、write、 readwrite 和 full。full 访问级别包括 readwrite 级别和访问控制管理功能。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
[root@serverc ~]# radosgw-admin user modify --uid=lxhuser1 --access=read
{
    "user_id": "lxhuser1",
    "display_name": "Xiaohui Li",
    "email": "939958092@qq.com",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lxhuser1",
            "access_key": "CMOGETUJ262DWDMEDQIL",
            "secret_key": "xCpI9wVJmX4ALhobBMhOEC6PofnXUXHUe0pBMVb6"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "default_storage_class": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

要删除用户并删除他们的对象和存储桶，请使用–purge-data选项

1
[root@serverc ~]# radosgw-admin user rm --uid=lxhuser1 --purge-data

通过设置配额来限制用户或桶使用的存储空间。请先设置配额参数，再启用配额。如果需要禁用配额，请将quota参数设置为负值

桶配额适用于指定UID所拥有的所有桶，与访问或向这些桶进行上传的用户无关

本例中为“lxh”用户设置的最大配额为1024个对象。启用用户配额

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
[root@serverc ~]# radosgw-admin quota set --quota-scope=user --uid=lxh --max-objects=1024
[root@serverc ~]# radosgw-admin quota enable --quota-scope=user --uid=lxh

[root@serverc ~]# radosgw-admin user info --uid=lxh
...
    "user_quota": {
        "enabled": true,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": 1024
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

同样，通过将- -quota-scope选项设置为桶，可以对桶使用配额。本例中设置的lxh用户在桶级别上，最大的大小不超过1024字节

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@serverc ~]# radosgw-admin quota set --quota-scope=bucket --uid=lxh --max-objects=1024
[root@serverc ~]# radosgw-admin quota enable --quota-scope=bucket --uid=lxh
[root@serverc ~]# radosgw-admin user info --uid=lxh
...
    "bucket_quota": {
        "enabled": true,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": 1024
    },
    "user_quota": {
        "enabled": true,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": 1024
    },
    "temp_url_keys": [],
    "type": "rgw",
    "mfa_ids": []
}

全局配额影响集群中的所有桶

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
[root@serverc ~]# radosgw-admin global quota set --quota-scope bucket --max-objects 2048
Global quota changes saved. Use 'period update' to apply them to the staging period, and 'period commit' to commit the new period.
{
    "bucket quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": 2048
    }
}
[root@serverc ~]# radosgw-admin global quota enable --quota-scope bucket
Global quota changes saved. Use 'period update' to apply them to the staging period, and 'period commit' to commit the new period.
{
    "bucket quota": {
        "enabled": true,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": 2048
    }
}

要在区域和period配置中实施，需使用 radosgw-admin period update --commit 命令来提交更改。也可以重启 RGW 实例来实施配额。

1
2
[root@serverc ~]# radosgw-admin period update --commit
[root@serverc ~]# ceph orch restart rgw.lixiaohui

使用 radosgw-admin user stats 和 radosgw-admin user info 命令可查看用户信息和统计信息。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@serverc ~]# radosgw-admin user info --uid=lxh
[root@serverc ~]# radosgw-admin user stats --uid=lxh --sync-stats
{
    "stats": {
        "size": 0,
        "size_actual": 0,
        "size_utilized": 0,
        "size_kb": 0,
        "size_kb_actual": 0,
        "size_kb_utilized": 0,
        "num_objects": 0
    },
    "last_stats_sync": "2024-08-08T07:54:58.484995Z",
    "last_stats_update": "0.000000"
}

使用radosgw-admin usage show命令显示某用户在指定日期的使用统计信息

1
2
3
4
5
[root@serverc ~]# radosgw-admin usage show --uid=lxh --uid=uid --start-date="2020-09-18 01:00:00" --end-date="2022-09-19 17:00:00"
{
    "entries": [],
    "summary": []
}

使用radosgw-admin usage show命令查看所有用户的统计信息。使用这些总体统计信息可以帮助理解对象存储模式，并为扩展RADOS网关服务规划新实例的部署

1
2
3
4
[root@serverc ~]# radosgw-admin usage show --show-log-entries=false
{
    "summary": []
}

使用RADOS网关访问S3对象

Amazon S3 API 支持多种存储桶 URL 格式，包括 http://server.example.com/bucket/ 或 http://bucket.server.example.com/。有些客户端（如 s3cmd 命令）仅支持第二种 URL 格式。默认情况下，RADOS 网关不启用这种格式。要启用第二种 URL 格式，需为您的 DNS 后缀设置 rgw_dns_name 参数。

1
2
3
[root@serverc ~]# ceph config set client.rgw rgw_dns_name lab.example.com
[root@serverc ~]# ceph config get client.rgw rgw_dns_name
lab.example.com

其中dns_suffix是用于创建bucket名称的完全限定域名

除了配置 rgw_dns_name 外，您还必须为您的 DNS 服务器配置一个指向 RADOS 网关 IP 地址的该域的通配符 DNS 记录

使用Amazon S3 API客户端

awscli 软件包中的命令支持使用 S3 API 进行存储桶和对象管理

1
[root@serverc ~]# yum install awscli -y

复用或创建新用户

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@serverc ~]# radosgw-admin user create --uid=lixiaohui --display-name 'Xiaohui'
{
    "user_id": "lixiaohui",
    "display_name": "Xiaohui",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "subusers": [],
    "keys": [
        {
            "user": "lixiaohui",
            "access_key": "6IDVK5HW0QDAA602Z8T4",
            "secret_key": "24gfHDBO0jdJXjZ5K1OKsdQtmhY738Xg9cLIvh3E"
        }

配置一下命令的发起者身份

1
2
3
4
5
[root@serverc ~]# aws configure --profile=ceph
AWS Access Key ID [None]: 6IDVK5HW0QDAA602Z8T4
AWS Secret Access Key [None]: 24gfHDBO0jdJXjZ5K1OKsdQtmhY738Xg9cLIvh3E
Default region name [None]:
Default output format [None]:

创建一个名为lxh-bucket的存储桶

1
2
3
4
5
6
7
[root@serverc ~]# aws s3 mb s3://lxh-bucket --endpoint-url http://serverc.lab.example.com
make_bucket: lxh-bucket

[root@serverc ~]# radosgw-admin bucket list
[
    "lxh-bucket"
]

使用 aws cp 命令上传对象到存储桶

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
[root@serverc ~]# aws --acl=public-read-write s3 cp /etc/fstab s3://lxh-bucket/fstab --endpoint-url http://serverc.lab.example.com
upload: ../etc/fstab to s3://lxh-bucket/fstab

[root@serverc ~]# ceph osd pool ls
device_health_metrics
.rgw.root
default.rgw.log
default.rgw.control
default.rgw.meta
shanghai-baoshan.rgw.log
shanghai-baoshan.rgw.control
shanghai-baoshan.rgw.meta
shanghai-baoshan.rgw.buckets.index
shanghai-baoshan.rgw.buckets.data

[root@serverc ~]# rados -p shanghai-baoshan.rgw.buckets.data ls
64df6f18-db0c-459a-9ef5-1f01598870d5.54746.1_fstab

测试文件下载

1
2
3
4
5
6
7
8
9
10
11
[root@serverc ~]# wget http://serverc.lab.example.com/lxh-bucket/fstab
--2024-08-08 04:07:00--  http://serverc.lab.example.com/lxh-bucket/fstab
Resolving serverc.lab.example.com (serverc.lab.example.com)... 172.25.250.12
Connecting to serverc.lab.example.com (serverc.lab.example.com)|172.25.250.12|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 142 [binary/octet-stream]
Saving to: ‘fstab’

fstab                                         100%[=================================================================================================>]     142  --.-KB/s    in 0s

2024-08-08 04:07:00 (27.1 MB/s) - ‘fstab’ saved [142/142]

删除文件

1
2
[root@serverc ~]# aws s3 rm --endpoint-url http://serverc.lab.example.com s3://lxh-bucket/fstab
delete: s3://lxh-bucket/fstab