Kylin api 整理（部分官网未给出）
2017-07-31 本文已影响958人
Skye_kh
kylin 的官网没有列出保存 cube 信息，model 信息，project等 rest api，这里通过查看源码对实际项目使用中有用到的 api 进行列举
官方文档
http://kylin.apache.org/docs16/howto/howto_use_restapi.html#build-cube
保存项目
POST /kylin/api/projects
ProjectController.java
Request Body
- name -
requiredString项目名 - description -
optionalString项目描述
导入 hive table
POST /kylin/api/tables/{tables}/{project}
Request Parameters
- tables -
requiredstring需要导入的表名，用 , 分隔 - project -
requiredString导入的目标项目
Response Sample
{
"result.loaded": ["DEFAULT.SAMPLE_07"],
"result.unloaded": ["sapmle_08"]
}
保存 model
POST /kylin/api/models
ModelController.java
Request Body
- project -
requiredString项目名 - modelName -
requiredStringModel 名 - modelDescData -
requiredStringmodel 描述字符串，json 中必须用字符串，即 " 用 " 表示，详见下面例子
post json 请求示例
{
"project": "game_inner",
"modelName": "test_model2",
"modelDescData": "{\"name\": \"test_model2\", \"owner\": \"ADMIN\", \"description\": \"\", \"fact_table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"lookups\": [], \"dimensions\": [ { \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"columns\": [ \"STARTTIME\", \"USERID\", \"GAME\", \"TYPEID\", \"DURATION\", \"LOCATIONID\" ] } ], \"metrics\": [ \"STARTTIME\", \"USERID\", \"GAME\", \"TYPEID\", \"DURATION\", \"LOCATIONID\" ], \"filter_condition\": \"\", \"partition_desc\": { \"partition_date_column\": null, \"partition_time_column\": null, \"partition_date_start\": 0, \"partition_date_format\": \"yyyy-MM-dd\", \"partition_time_format\": \"HH:mm:ss\", \"partition_type\": \"APPEND\", \"partition_condition_builder\": \"org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder\" }, \"capacity\": \"MEDIUM\"}"
}
Response Sample
{
"uuid": "536582f6-ffa8-415e-b9c8-58f864994ac5",
"modelName": "test_model2",
"modelDescData": "{\"name\": \"test_model2\", \"owner\": \"ADMIN\", \"description\": \"\", \"fact_table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"lookups\": [], \"dimensions\": [ { \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"columns\": [ \"STARTTIME\", \"USERID\", \"GAME\", \"TYPEID\", \"DURATION\", \"LOCATIONID\" ] } ], \"metrics\": [ \"STARTTIME\", \"USERID\", \"GAME\", \"TYPEID\", \"DURATION\", \"LOCATIONID\" ], \"filter_condition\": \"\", \"partition_desc\": { \"partition_date_column\": null, \"partition_time_column\": null, \"partition_date_start\": 0, \"partition_date_format\": \"yyyy-MM-dd\", \"partition_time_format\": \"HH:mm:ss\", \"partition_type\": \"APPEND\", \"partition_condition_builder\": \"org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder\" }, \"capacity\": \"MEDIUM\"}",
"successful": true,
"message": null,
"project": "game_inner"
}
保存 cube 详细配置
POST /kylin/api/cubes
CubeController.java
Request Body
- project -
requiredString项目名 - cubeName -
requiredStringCube 名 - cubeDescData -
requiredStringcube 描述字符串，json 中必须用字符串，即 " 用 " 表示，详见下面例子
Curl Example
# “QURNSU46S1lMSU4=”是 “ADMIN:KYLIN”的base64编码
curl -X POST -H "Authorization: Basic QURNSU46S1lMSU4=" -H "Content-Type: application/json" -d '{ "project":"test_project","cubeName":"test_cube5","cubeDescData":”cube描述的字符串”}' http://localhost:7070/kylin/api/cubes
post json 示例
{
"project":"game_inner",
"cubeName":"test_cube4",
"cubeDescData":"{ \"name\": \"test_cube4\", \"model_name\": \"test_game_model\", \"description\": \"\", \"null_string\": null, \"dimensions\": [ { \"name\": \"STARTTIME\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"STARTTIME\", \"derived\": null }, { \"name\": \"USERID\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"USERID\", \"derived\": null }, { \"name\": \"GAME\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"GAME\", \"derived\": null }, { \"name\": \"TYPEID\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"TYPEID\", \"derived\": null }, { \"name\": \"DURATION\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"DURATION\", \"derived\": null }, { \"name\": \"LOCATIONID\", \"table\": \"KYLIN_FLAT_DB.DIM_LOCATION\", \"column\": null, \"derived\": [ \"LOCATIONID\" ] } ], \"measures\": [ { \"name\": \"_COUNT_\", \"function\": { \"expression\": \"COUNT\", \"parameter\": { \"type\": \"constant\", \"value\": \"1\", \"next_parameter\": null }, \"returntype\": \"bigint\" }, \"dependent_measure_ref\": null } ], \"dictionaries\": [], \"rowkey\": { \"rowkey_columns\": [ { \"column\": \"STARTTIME\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"USERID\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"GAME\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"TYPEID\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"DURATION\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"LOCATIONID\", \"encoding\": \"dict\", \"isShardBy\": false } ] }, \"hbase_mapping\": { \"column_family\": [ { \"name\": \"F1\", \"columns\": [ { \"qualifier\": \"M\", \"measure_refs\": [ \"_COUNT_\" ] } ] } ] }, \"aggregation_groups\": [ { \"includes\": [ \"STARTTIME\", \"USERID\", \"GAME\", \"TYPEID\", \"DURATION\", \"LOCATIONID\" ], \"select_rule\": { \"hierarchy_dims\": [], \"mandatory_dims\": [], \"joint_dims\": [] } } ], \"signature\": \"YH/rFI7MAllwLXuyD3tBlw==\", \"notify_list\": [], \"status_need_notify\": [ \"ERROR\", \"DISCARDED\", \"SUCCEED\" ], \"partition_date_start\": 0, \"partition_date_end\": 3153600000000, \"auto_merge_time_ranges\": [ 604800000, 2419200000 ], \"retention_range\": 0, \"engine_type\": 2, \"storage_type\": 2, \"override_kylin_properties\": {}}"
}
Response Sample
{
"uuid": "2225fb4c-aafa-470a-8708-ddfbc44d9e78",
"cubeName": "test_cube4",
"cubeDescData": "{ \"name\": \"test_cube4\", \"model_name\": \"test_game_model\", \"description\": \"\", \"null_string\": null, \"dimensions\": [ { \"name\": \"STARTTIME\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"STARTTIME\", \"derived\": null }, { \"name\": \"USERID\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"USERID\", \"derived\": null }, { \"name\": \"GAME\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"GAME\", \"derived\": null }, { \"name\": \"TYPEID\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"TYPEID\", \"derived\": null }, { \"name\": \"DURATION\", \"table\": \"KYLIN_FLAT_DB.FACT_GAME\", \"column\": \"DURATION\", \"derived\": null }, { \"name\": \"LOCATIONID\", \"table\": \"KYLIN_FLAT_DB.DIM_LOCATION\", \"column\": null, \"derived\": [ \"LOCATIONID\" ] } ], \"measures\": [ { \"name\": \"_COUNT_\", \"function\": { \"expression\": \"COUNT\", \"parameter\": { \"type\": \"constant\", \"value\": \"1\", \"next_parameter\": null }, \"returntype\": \"bigint\" }, \"dependent_measure_ref\": null } ], \"dictionaries\": [], \"rowkey\": { \"rowkey_columns\": [ { \"column\": \"STARTTIME\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"USERID\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"GAME\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"TYPEID\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"DURATION\", \"encoding\": \"dict\", \"isShardBy\": false }, { \"column\": \"LOCATIONID\", \"encoding\": \"dict\", \"isShardBy\": false } ] }, \"hbase_mapping\": { \"column_family\": [ { \"name\": \"F1\", \"columns\": [ { \"qualifier\": \"M\", \"measure_refs\": [ \"_COUNT_\" ] } ] } ] }, \"aggregation_groups\": [ { \"includes\": [ \"STARTTIME\", \"USERID\", \"GAME\", \"TYPEID\", \"DURATION\", \"LOCATIONID\" ], \"select_rule\": { \"hierarchy_dims\": [], \"mandatory_dims\": [], \"joint_dims\": [] } } ], \"signature\": \"YH/rFI7MAllwLXuyD3tBlw==\", \"notify_list\": [], \"status_need_notify\": [ \"ERROR\", \"DISCARDED\", \"SUCCEED\" ], \"partition_date_start\": 0, \"partition_date_end\": 3153600000000, \"auto_merge_time_ranges\": [ 604800000, 2419200000 ], \"retention_range\": 0, \"engine_type\": 2, \"storage_type\": 2, \"override_kylin_properties\": {}}",
"streamingData": null,
"kafkaData": null,
"successful": true,
"message": null,
"project": "game_inner",
"streamingCube": null
}
构建 Cube
PUT /kylin/api/cubes/{cubeName}/rebuild
Path Variable
-
cubeName -
requiredstringCube name.
Request Body -
startTime -
requiredlongStart timestamp of data to build, e.g. 1388563200000 for 2014-1-1 -
endTime -
requiredlongEnd timestamp of data to build- -
buildType -
requiredstringSupported build type: ‘BUILD’, ‘MERGE’, ‘REFRESH’
获取 Cube （查看信息）
GET /kylin/api/cubes/{cubeName}
Path Variable
- cubeName -
requiredstringCube name to find.
GET /kylin/api/cube_desc/{cubeName}
Get descriptor for specified cube instance.
Path Variable
- cubeName -
requiredstringCube name.
Response Sample
[
{
"uuid": "a24ca905-1fc6-4f67-985c-38fa5aeafd92",
"name": "test_kylin_cube_with_slr_desc",
"description": null,
"dimensions": [
{
"id": 0,
"name": "CAL_DT",
"table": "EDW.TEST_CAL_DT",
"column": null,
"derived": [
"WEEK_BEG_DT"
],
"hierarchy": false
},
{
"id": 1,
"name": "CATEGORY",
"table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
"column": null,
"derived": [
"USER_DEFINED_FIELD1",
"USER_DEFINED_FIELD3",
"UPD_DATE",
"UPD_USER"
],
"hierarchy": false
},
{
"id": 2,
"name": "CATEGORY_HIERARCHY",
"table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
"column": [
"META_CATEG_NAME",
"CATEG_LVL2_NAME",
"CATEG_LVL3_NAME"
],
"derived": null,
"hierarchy": true
},
{
"id": 3,
"name": "LSTG_FORMAT_NAME",
"table": "DEFAULT.TEST_KYLIN_FACT",
"column": [
"LSTG_FORMAT_NAME"
],
"derived": null,
"hierarchy": false
},
{
"id": 4,
"name": "SITE_ID",
"table": "EDW.TEST_SITES",
"column": null,
"derived": [
"SITE_NAME",
"CRE_USER"
],
"hierarchy": false
},
{
"id": 5,
"name": "SELLER_TYPE_CD",
"table": "EDW.TEST_SELLER_TYPE_DIM",
"column": null,
"derived": [
"SELLER_TYPE_DESC"
],
"hierarchy": false
},
{
"id": 6,
"name": "SELLER_ID",
"table": "DEFAULT.TEST_KYLIN_FACT",
"column": [
"SELLER_ID"
],
"derived": null,
"hierarchy": false
}
],
"measures": [
{
"id": 1,
"name": "GMV_SUM",
"function": {
"expression": "SUM",
"parameter": {
"type": "column",
"value": "PRICE",
"next_parameter": null
},
"returntype": "decimal(19,4)"
},
"dependent_measure_ref": null
},
{
"id": 2,
"name": "GMV_MIN",
"function": {
"expression": "MIN",
"parameter": {
"type": "column",
"value": "PRICE",
"next_parameter": null
},
"returntype": "decimal(19,4)"
},
"dependent_measure_ref": null
},
{
"id": 3,
"name": "GMV_MAX",
"function": {
"expression": "MAX",
"parameter": {
"type": "column",
"value": "PRICE",
"next_parameter": null
},
"returntype": "decimal(19,4)"
},
"dependent_measure_ref": null
},
{
"id": 4,
"name": "TRANS_CNT",
"function": {
"expression": "COUNT",
"parameter": {
"type": "constant",
"value": "1",
"next_parameter": null
},
"returntype": "bigint"
},
"dependent_measure_ref": null
},
{
"id": 5,
"name": "ITEM_COUNT_SUM",
"function": {
"expression": "SUM",
"parameter": {
"type": "column",
"value": "ITEM_COUNT",
"next_parameter": null
},
"returntype": "bigint"
},
"dependent_measure_ref": null
}
],
"rowkey": {
"rowkey_columns": [
{
"column": "SELLER_ID",
"length": 18,
"dictionary": null,
"mandatory": true
},
{
"column": "CAL_DT",
"length": 0,
"dictionary": "true",
"mandatory": false
},
{
"column": "LEAF_CATEG_ID",
"length": 0,
"dictionary": "true",
"mandatory": false
},
{
"column": "META_CATEG_NAME",
"length": 0,
"dictionary": "true",
"mandatory": false
},
{
"column": "CATEG_LVL2_NAME",
"length": 0,
"dictionary": "true",
"mandatory": false
},
{
"column": "CATEG_LVL3_NAME",
"length": 0,
"dictionary": "true",
"mandatory": false
},
{
"column": "LSTG_FORMAT_NAME",
"length": 12,
"dictionary": null,
"mandatory": false
},
{
"column": "LSTG_SITE_ID",
"length": 0,
"dictionary": "true",
"mandatory": false
},
{
"column": "SLR_SEGMENT_CD",
"length": 0,
"dictionary": "true",
"mandatory": false
}
],
"aggregation_groups": [
[
"LEAF_CATEG_ID",
"META_CATEG_NAME",
"CATEG_LVL2_NAME",
"CATEG_LVL3_NAME",
"CAL_DT"
]
]
},
"signature": "lsLAl2jL62ZApmOLZqWU3g==",
"last_modified": 1445850327000,
"model_name": "test_kylin_with_slr_model_desc",
"null_string": null,
"hbase_mapping": {
"column_family": [
{
"name": "F1",
"columns": [
{
"qualifier": "M",
"measure_refs": [
"GMV_SUM",
"GMV_MIN",
"GMV_MAX",
"TRANS_CNT",
"ITEM_COUNT_SUM"
]
}
]
}
]
},
"notify_list": null,
"auto_merge_time_ranges": null,
"retention_range": 0
}
]
获取构建任务状态
GET /kylin/api/jobs/{jobId}
Path variable
jobId - required string Job id. 此处构建 cube 时返回的 uuid 即为任务 id
Response Sample
{
"uuid":"c143e0e4-ac5f-434d-acf3-46b0d15e3dc6",
"last_modified":1407908916705,
"name":"test_kylin_cube_with_slr_empty - 19700101000000_20140731160000 - BUILD - PDT 2014-08-12 22:48:36",
"type":"BUILD",
"duration":0,
"related_cube":"test_kylin_cube_with_slr_empty",
"related_segment":"19700101000000_20140731160000",
"exec_start_time":0,
"exec_end_time":0,
"mr_waiting":0,
"steps":[
{
"interruptCmd":null,
"name":"Create Intermediate Flat Hive Table",
"sequence_id":0,
"exec_cmd":"hive -e \"DROP TABLE IF EXISTS kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6;\nCREATE EXTERNAL TABLE IF NOT EXISTS kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6\n(\nCAL_DT date\n,LEAF_CATEG_ID int\n,LSTG_SITE_ID int\n,META_CATEG_NAME string\n,CATEG_LVL2_NAME string\n,CATEG_LVL3_NAME string\n,LSTG_FORMAT_NAME string\n,SLR_SEGMENT_CD smallint\n,SELLER_ID bigint\n,PRICE decimal\n)\nROW FORMAT DELIMITED FIELDS TERMINATED BY '\\177'\nSTORED AS SEQUENCEFILE\nLOCATION '/tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6';\nSET mapreduce.job.split.metainfo.maxsize=-1;\nSET mapred.compress.map.output=true;\nSET mapred.map.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;\nSET mapred.output.compress=true;\nSET mapred.output.compression.codec=com.hadoop.compression.lzo.LzoCodec;\nSET mapred.output.compression.type=BLOCK;\nSET mapreduce.job.max.split.locations=2000;\nSET hive.exec.compress.output=true;\nSET hive.auto.convert.join.noconditionaltask = true;\nSET hive.auto.convert.join.noconditionaltask.size = 300000000;\nINSERT OVERWRITE TABLE kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6\nSELECT\nTEST_KYLIN_FACT.CAL_DT\n,TEST_KYLIN_FACT.LEAF_CATEG_ID\n,TEST_KYLIN_FACT.LSTG_SITE_ID\n,TEST_CATEGORY_GROUPINGS.META_CATEG_NAME\n,TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME\n,TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME\n,TEST_KYLIN_FACT.LSTG_FORMAT_NAME\n,TEST_KYLIN_FACT.SLR_SEGMENT_CD\n,TEST_KYLIN_FACT.SELLER_ID\n,TEST_KYLIN_FACT.PRICE\nFROM TEST_KYLIN_FACT\nINNER JOIN TEST_CAL_DT\nON TEST_KYLIN_FACT.CAL_DT = TEST_CAL_DT.CAL_DT\nINNER JOIN TEST_CATEGORY_GROUPINGS\nON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID\nINNER JOIN TEST_SITES\nON TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_SITES.SITE_ID\nINNER JOIN TEST_SELLER_TYPE_DIM\nON TEST_KYLIN_FACT.SLR_SEGMENT_CD = TEST_SELLER_TYPE_DIM.SELLER_TYPE_CD\nWHERE (test_kylin_fact.cal_dt < '2014-07-31 16:00:00')\n;\n\"",
"interrupt_cmd":null,
"exec_start_time":0,
"exec_end_time":0,
"exec_wait_time":0,
"step_status":"PENDING",
"cmd_type":"SHELL_CMD_HADOOP",
"info":null,
"run_async":false
},
{
"interruptCmd":null,
"name":"Extract Fact Table Distinct Columns",
"sequence_id":1,
"exec_cmd":" -conf C:/kylin/Kylin/server/src/main/resources/hadoop_job_conf_medium.xml -cubename test_kylin_cube_with_slr_empty -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/kylin_intermediate_test_kylin_cube_with_slr_desc_19700101000000_20140731160000_c143e0e4_ac5f_434d_acf3_46b0d15e3dc6 -output /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/fact_distinct_columns -jobname Kylin_Fact_Distinct_Columns_test_kylin_cube_with_slr_empty_Step_1",
"interrupt_cmd":null,
"exec_start_time":0,
"exec_end_time":0,
"exec_wait_time":0,
"step_status":"PENDING",
"cmd_type":"JAVA_CMD_HADOOP_FACTDISTINCT",
"info":null,
"run_async":true
},
{
"interruptCmd":null,
"name":"Load HFile to HBase Table",
"sequence_id":12,
"exec_cmd":" -input /tmp/kylin-c143e0e4-ac5f-434d-acf3-46b0d15e3dc6/test_kylin_cube_with_slr_empty/hfile/ -htablename KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_EMPTY-19700101000000_20140731160000_11BB4326-5975-4358-804C-70D53642E03A -cubename test_kylin_cube_with_slr_empty",
"interrupt_cmd":null,
"exec_start_time":0,
"exec_end_time":0,
"exec_wait_time":0,
"step_status":"PENDING",
"cmd_type":"JAVA_CMD_HADOOP_NO_MR_BULKLOAD",
"info":null,
"run_async":false
}
],
"job_status":"PENDING",
"progress":0.0
}
