RGW 缓存组件解析
2023-01-01 本文已影响0人
Xiao_Yang
一、RGW 缓存架构
topo.png
二、背景知识: AWS签名流程
认证的签名流程
1. 创建规范请求。
2. 使用规范请求和其他元数据来创建供签署的字符串。
3. 从 AWS 秘密访问密钥派生签名密钥。然后将该签名密钥与在上一步中创建的字符串结合使用来创建签名。
4. 将生成的签名添加到 HTTP 请求的标头中或者添加为查询字符串参数。
例:HTTPS 请求
GET https://iam.amazonaws.com/?Action=ListUsers&Version=2010-05-08 HTTP/1.1
Host: iam.amazonaws.com
Content-Type: application/x-www-form-urlencoded; charset=utf-8
X-Amz-Date: 20150830T123600Z
例:待签字符串
Signature Version 4 签名工作原理 参考：SignV4
计算签名
kSecret = wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY
# 派生签名密钥的生成伪代码
HMAC(HMAC(HMAC(HMAC("AWS4" + kSecret,"20150830"),"us-east-1"),"iam"),"aws4_request")
#输出 c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9
# 计算签名伪代码
signature = HexEncode(HMAC(derived signing key, string to sign))
#输出：5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400e06b5924a6f2b5d7
将签名添加到HTTP请求
方式一: 将签名信息添加到 Authorization 标头
# 伪代码说明 Authorization 标头的构造
Authorization: algorithm Credential=access key ID/credential scope, SignedHeaders=SignedHeaders, Signature=signature
# 示例说明一个完整的 Authorization 标头
Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/iam/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400e06b5924a6f2b5d7
方式二: 将签名信息添加到查询字符串
# 示例说明一个完整的查询字答串
https://iam.amazonaws.com?Action=ListUsers&Version=2010-05-08&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIDEXAMPLE%2F20150830%2Fus-east-1%2Fiam%2Faws4_request&X-Amz-Date=20150830T123600Z&X-Amz-Expires=60&X-Amz-SignedHeaders=content-type%3Bhost&X-Amz-Signature=37ac2f4fde00b0ac9bd9eadeb459b1bbee224158d66e7ae5fcadb70b2d181d02
三、背景知识: ngx.aws.auth 插件认证模块逻辑分析
代码逻辑分析，认证签名生成流程（遵从S3 API认证规范）
# 代码仓库
# https://github.com/kaltura/nginx-aws-auth-module/blob/master/src/ngx_http_aws_auth_module.c
ngx_http_aws_auth_block() ->
ngx_http_aws_auth_variable() ->
ngx_http_aws_auth_generate_signing_key() ->
ngx_http_aws_auth_sign() ->
四、RGW Data Caching Configs 流程及各配置文件解析
Nginx.conf
user nginx;
#Process per core
worker_processes auto;
pid /var/run/nginx.pid;
events {
#Number of connections per worker
worker_connections 1024;
}
# 初始化
# 基于已创建的“缓存用户”配置的AK/SK，以AWS API标准协议方式向
# 后端发启认证请求并获取认证Token -> $awsauth
# 最终值认证Token赋予变量为 $awsauthfour 以用于后续对象缓存逻辑使用
http {
types_hash_max_size 4096;
lua_package_path '/usr/local/openresty/lualib/?.lua;;';
aws_auth $aws_token {
# 为缓存用户配置 AK/SK，下面“cache”需被替换掉
# 此用户赋予了 amz-cache 能力属性
access_key cache;
secret_key cache;
service s3;
region us-east-1;
}
# 如果 aws_auth 插件模块未返回值则此map方法选择一个原 authorization 头值
map $aws_token $awsauth {
default $http_authorization; # 默认值
~. $aws_token; # 正则匹配任意值，将上面模块 aws_auth 返值
}
map $request_uri $awsauthtwo {
"/" $http_authorization; # 请求uri为“/”或“～/” 则为 authorization 头值
"~\?" $http_authorization;
default $awsauth; # 否则为 awsauthtwo = awsauth
}
map $request_method $awsauththree {
default $awsauthtwo; # GET 方法 awsauththree == awsauthtwo
"PUT" $http_authorization;
"HEAD" $http_authorization;
"POST" $http_authorization;
"DELETE" $http_authorization;
"COPY" $http_authorization;
}
map $http_if_match $awsauthfour {
~. $http_authorization; # 头 IF_MATCH 存在则 awsauthfour== IF_MATCH
default $awsauththree; # 不然 awsauthfour == awsauththree
}
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
error_log /var/log/nginx/error.log;
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nodelay on;
keepalive_timeout 65;
include /etc/nginx/conf.d/*.conf;
}
Nginx-default.conf
# 按环境与需求配置缓存目录及大小。max_size 为 Nginx 最大缓存内容大小。
proxy_cache_path /data/cache levels=2:2:2 keys_zone=mycache:999m max_size=20G inactive=1d use_temp_path=off;
upstream rgws {
# 按需修改后端 rgw 列表的IP地址或域名
server rgw1:8000 max_fails=2 fail_timeout=5s;
server rgw2:8000 max_fails=2 fail_timeout=5s;
server rgw3:8000 max_fails=2 fail_timeout=5s;
}
server {
listen 80;
server_name cacher;
location /authentication {
internal;
client_max_body_size 0;
# 将S3标准认证代理转发到后端
proxy_pass http://rgws$request_uri;
proxy_pass_request_body off;
proxy_set_header Host $host;
# 设置头 x-rgw-auth 允许 RGW 能仅做认证无需获取对象数据操作
proxy_set_header x-rgw-auth "yes";
proxy_set_header Authorization $http_authorization;
proxy_http_version 1.1;
proxy_method $request_method;
# 后面仅逻辑判断之用，非正常认证请求直接返回
# Do not convert HEAD requests into GET requests
proxy_cache_convert_head off;
error_page 404 = @outage;
proxy_intercept_errors on;
if ($request_uri = "/") {
return 200;
}
# URI included with question mark is not being cached
if ($request_uri ~* (\?)) {
return 200;
}
if ($request_method = "PUT") {
return 200;
}
if ($request_method = "POST") {
return 200;
}
if ($request_method = "HEAD") {
return 200;
}
if ($request_method = "COPY") {
return 200;
}
if ($request_method = "DELETE") {
return 200;
}
if ($http_if_match) {
return 200;
}
if ($http_authorization !~* "aws4_request") {
return 200;
}
}
location @outage{
return 403;
}
location / {
auth_request /authentication;
proxy_pass http://rgws;
set $authvar '';
# if $do_not_cache is not empty the request would not be cached, this is relevant for list op for example
set $do_not_cache '';
# the IP or name of the RGWs
rewrite_by_lua_file /etc/nginx/nginx-lua-file.lua;
#proxy_set_header Authorization $http_authorization;
# my cache configured at the top of the file
proxy_cache mycache;
proxy_cache_lock_timeout 0s;
proxy_cache_lock_age 1000s;
proxy_http_version 1.1;
set $date $aws_auth_date;
# Getting 403 if this header not set
proxy_set_header Host $host;
# Cache all 200 OK's for 1 day
proxy_cache_valid 200 206 1d;
# Use stale cache file in all errors from upstream if we can
proxy_cache_use_stale updating;
proxy_cache_background_update on;
# Try to check if etag have changed, if yes, do not re-fetch from rgw the object
proxy_cache_revalidate on;
# Lock the cache so that only one request can populate it at a time
proxy_cache_lock on;
# prevent conversion of head requests to get requests
proxy_cache_convert_head off;
# Listing all buckets should not be cached
if ($request_uri = "/") {
set $do_not_cache "no";
set $date $http_x_amz_date;
}
# URI including question mark are not supported to prevent bucket listing cache
if ($request_uri ~* (\?)) {
set $do_not_cache "no";
set $date $http_x_amz_date;
}
# 仅 aws4 请求支持缓存
if ($http_authorization !~* "aws4_request") {
set $date $http_x_amz_date;
}
if ($request_method = "PUT") {
set $date $http_x_amz_date;
}
if ($request_method = "POST") {
set $date $http_x_amz_date;
}
if ($request_method = "HEAD") {
set $do_not_cache "no";
set $date $http_x_amz_date;
}
if ($request_method = "COPY") {
set $do_not_cache "no";
set $date $http_x_amz_date;
}
if ($http_if_match) {
#set $do_not_cache "no";
set $date $http_x_amz_date;
set $myrange $http_range;
}
if ($request_method = "DELETE") {
set $do_not_cache "no";
set $date $http_x_amz_date;
}
# 设置后端RGW所需的头信息
proxy_set_header if_match $http_if_match;
proxy_set_header Range $myrange;
proxy_set_header x-amz-date $date;
proxy_set_header X-Amz-Cache $authvar;
proxy_no_cache $do_not_cache;
# 此处设置了S3认证头信息
# *** 核心与关键点 ***
proxy_set_header Authorization $awsauthfour;
# 缓存 Key 值的构造信息
proxy_cache_key "$request_uri$request_method$request_body$myrange";
client_max_body_size 0;
}
}
Nginx-lua-file.Lua 代码
-- 此代码逻辑的核心目的是创建X-Amz-Cache的值，此值是基于所有签名的头文件名称组成；
-- 取所有签名的头文件名称，为了创建X-Amz-Cache，这是覆盖范围头文件所必须的，以便能够重新引导一个对象。
local check = ngx.req.get_headers()["AUTHORIZATION"]
local uri = ngx.var.request_uri
local ngx_re = require "ngx.re"
local hdrs = ngx.req.get_headers()
-- 请求实例
-- Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/iam/aws4_request, SignedHeaders=content-type;X-Amz-Cache;host;x-amz-date, Signature=5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400e06b5924a6f2b5d7
local res, err = ngx_re.split(check,"SignedHeaders=")
local res2, err2 = ngx_re.split(res[2],",")
local res3, err3 = ngx_re.split(res2[1],";")
-- 1） Check Splited
-- res:["Authorization: AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/iam/aws4_request, SignedHeaders=","content-type;X-Amz-Cache;host;x-amz-date, Signature=5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400e06b5924a6f2b5d7"]
-- res2: ["content-type;X-Amz-Cache;host;x-amz-date","Signature=5d672d79c15b13162d9279b0855cfba6789a8edb4c82c400e06b5924a6f2b5d7""]
-- res3: [content-type,host,x-amz-date]
local t = {}
local concathdrs = string.char(0x00)
for i = 1, #res3, 1 do
-- 判断 x-amz-date 不为空
if hdrs[res3[i]] ~= nil then
--0xB1 符分隔头名称与值
t[i] = res3[i] .. string.char(0xB1) .. hdrs[res3[i]]
--0xB2 符分隔两个头
concathdrs = concathdrs .. string.char(0xB2) .. t[i]
end
end
-- 2）拼接请求头字串
-- concathdrs的值为头与值组成的字符串，字符串以指定的相应的分隔符分隔开
-- authorization认证头不为空处理
if check ~= nil then
local xamzcache = concathdrs:sub(2)
xamzcache = xamzcache .. string.char(0xB2) .. "Authorization" .. string.char(0xB1) .. check
if xamzcache:find("aws4_request") ~= nil and uri ~= "/" and uri:find("?") == nil and hdrs["if-match"] == nil then
ngx.var.authvar = xamzcache
end
end
-- 判断存在"aws4_request"，请求URI不为“/”，不包含“？“，不包含”if-match“并没有
-- 返回变量“authvar” ngx.var.authvar 为 xamzcache 的接拼值
五、RGW Rados Gateway 验证API （兼容 S3_API 和 Swift_API ）
RGW兼容 S3_API 语法之验证实例
HTTP/1.1
PUT /buckets/bucket/object.mpeg
Host: cname.domain.com
Date: Mon, 2 Jan 2012 00:01:01 +0000
Content-Encoding: mpeg
Content-Length: 9999999
Authorization: AWS {access-key}:{hash-of-header-and-secret}
使用管理工具radosgw-admin对Ceph用户管理
# 创建用户
$ radosgw-admin user create --display-name="johnny rotten" --uid=johnny
{ "user_id": "johnny",
"rados_uid": 0,
"display_name": "johnny rotten",
"email": "",
"suspended": 0,
"subusers": [],
"keys": [
{ "user": "johnny",
"access_key": "TCICW53D9BQ2VGC46I44",
"secret_key": "tfm9aHMI8X76L3UdgE+ZQaJag1vJQmE6HDb5Lbrz"}],
"swift_keys": []}
# 删除用户
$ radosgw-admin user rm --uid=johnny
# 为用户绑定 Bucket
$ radosgw-admin bucket link --bucket=foo --bucket_id=<bucket id> --uid=johnny
# 解绑 Bucket
$ radosgw-admin bucket unlink --bucket=foo --uid=johnny
# 删除 Bucket
$ radosgw-admin bucket rm --bucket=foo
