ElasticSearch学习笔记

2019-12-15  本文已影响0人  small瓜瓜 
GET /customer/_search
{
  "size": 0,
  "aggs": {
    "group_by_state": {
      "terms": {
        "field": "state.keyword"
      },0
      "aggs": {
        "group_by_gender": {
          "terms": {
            "field": "gender.keyword"
          },
          "aggs": {
            "avg_pri": {
              "avg": {
                "field": "balance"
              }
            }
          }
        }
      }
    },
    "avg_balance":{
      "avg": {
        "field": "balance"
      }
    }
  }
}

GET /customer/_search?size=0
{
  "aggs": {
    "avg_grade": {
      "avg": {
        "field": "balance"
      }
    }
  }
}

GET /customer/_search?size=0
{
  "aggs": {
    "avg_grade": {
      "avg": {
        "script": {
          "source": "doc.balance.value"
        }
      }
    }
  }
}

POST /customer/_search?size=0
{
    "aggs" : {
        "avg_corrected_grade" : {
            "avg" : {
                "field" : "balance",
                "script" : {
                    "lang": "painless",
                    "source": "_value * params.correction",
                    "params" : {
                        "correction" : 1
                    }
                }
            }
        }
    }
}

自定义分词器

# 指定analyer进行分词校验
POST _analyze
{
  "analyzer": "ik_smart",
  "text": "这里是江西的师范大学"
}

# 指定字段进行校验，得出检索不出预期结果的原因
POST /test_index/_analyze
{
  "field": "name",
  "text": "Hello world"
}

# 指定char_filter和tokenizer
POST _analyze
{
  "tokenizer": "keyword",
  "char_filter": [
    "html_strip"
  ],
  "text": "<p>Hello&nbsp;World</p>"
}

# 在创建index时指定analyer
PUT test_index_1
{
  "settings": {
    "analysis": {
      "analyzer": {
        "my_custom_analyer": {
          "type": "custom",
          "char_filter": [
            "html_strip"
          ],
          "tokenizer": "standard",
          "filter": [
            "lowercase",
            "asciifolding"
          ]
        }
      }
    }
  }
}

# 自定义分词器
PUT test_index_2
{
  "settings": {
    "analysis": {
      "analyzer": {
        "my_custom_analyer": {
          "type": "custom",
          "char_filter": [
            "emoticons"
          ],
          "tokenizer": "punctuation",
          "filter": [
            "lowercase",
            "english_stop"
          ]
        }
      },
      "tokenizer": {
        "punctuation": {
          "type": "pattern",
          "pattern": "[.,!?]"
        }
      },
      "char_filter": {
        "emoticons": {
          "type": "mapping",
          "mappings": [
            ":> => happy",
            ":< => sad",
            "cat => yjn"
          ]
        }
      },
      "filter": {
        "english_stop": {
          "type": "stop",
          "stopwords": "_english_"
        }
      }
    }
  }
}

POST test_index_2/_analyze
{
  "analyzer":"my_custom_analyer",
  "text": "cat.cat.cat"
}

PUT test_analyze
{
  "settings": {
    "analysis": {
      "analyzer": {
        "pinyin_test": {
          "char_filter": [
            "html_strip",
            "mymap",
            "mypattern"
          ],
          "tokenizer": "test_tokenizer"
        }
      },
      "tokenizer": {
        "test_tokenizer": {
          "type": "keyword"
        }
      },
      "char_filter": {
        "mymap": {
          "type": "mapping",
          "mappings": [
            "* => 我",
            "# => 爱",
            "^ => 你"
          ]
        },
        "mypattern": {
          "type": "pattern_replace",
          "pattern": "\\s+",
          "replacement": ""
        }
      }
    }
  }
}

GET test_analyze/_analyze
{
  "text": "<p>fkj<br>s<ajf>*ld#fk^^d</p>",
  "analyzer": "pinyin_test"
}

自定义Mapping

PUT test_index_3
{
    "mappings" : {
      "properties" : {
        "age" : {
          "type" : "long"
        },
        "name" : {
          "type" : "keyword"
        },
        "sex" : {
          "type" : "boolean"
        }
      }
    }
  }
  
GET /test_index_3/_mapping

自定义mapping和dynamic对新增字段的意义

dynamic值 结果
false 可以正常插入新字段数据，但是不会创建新字段
true 可以正常插入新字段数据，会创建新字段
"strict" 插入新的字段会报错 
PUT my_index
{
  "mappings" : {
      "dynamic": true, 
      "properties" : {
        "age" : {
          "type" : "long"
        },
        "name" : {
          "type" : "keyword"
        },
        "sex" : {
          "type" : "boolean"
        }
      }
    }
}

GET my_index/_mapping

PUT my_index/_doc/1
{
  "name": "zjb",
  "title": "yjn"
}

GET my_index/_search

DELETE my_index

自定义mapping之index，字段能否被检索

PUT my_index
{
  "mappings": {
    "properties": {
      "cookie": {
        "type": "text",
        "index": false
      }
    }
  }
}

PUT my_index/_doc/1
{
  "cookie": "name=zjb"
}

GET my_index/_search
{
  "query": {
    "match": {
      "cookie": "TEXT"
    }
  }
}

自定义mapping之copy_to

DELETE my_index

PUT my_index
{
  "mappings": {
    "properties": {
      "first_name":{
        "type": "text",
        "copy_to": "full_name"
      },
      "last_name":{
        "type":"text",
        "copy_to": "full_name"
      },
      "full_name":{
        "type": "text"
      }
    }
  }
}

POST my_index/_doc/1
{
  "first_name":"zhan",
  "last_name":"jinbing"
}

GET my_index/_search
{
  "query": {
    "match":{
      "full_name": {
        "query":"zhan jinbing",
        "operator": "or"
      }
    }
  }
}

自定义Mapping--index_options

自定义Mapping--null_value

PUT my_index
{
  "mappings": {
    "properties": {
      "first_name":{
        "type": "text",
        "null_value": "这是默认值"
      }
    }
  }
}

自定义mapping--数据类型

多字段特性 multi-fields

PUT my_index
{
  "mappings": {
    "properties": {
      "name": {
        "type": "text",
        "fields": {
          "pinyin": {
            "type": "text",
            "analyzer": "pinyin"
          }
        }
      }
    }
  }
}

# 检索方式
GET test_index/_search
{
  "query": {
    "match": {
      "username.pinyin": "hanhan"
    }
  }
}

# 配置多个附属字段方便查询
PUT test_index
{
  "mappings": {
    "properties": {
      "name": {
        "type": "text",
        "fields": {
          "py_name": {
            "type": "text",
            "analyzer": "pinyin"
          },
          "ik_name": {
            "type": "text",
            "analyzer": "ik_smart"
          },
          "smartcn_name": {
            "type": "text",
            "analyzer": "smartcn"
          }
        },
        "analyzer": "keyword"
      }
    }
  }
}

这里要注意分词器analyzer是要具有将文字转拼音的功能的

配置多个查询字段
自定义mapping之通过numeric_detection开启字符串数值探测和dynamic_date_formats设置日期格式

Query

URI Search

GET /my_index/_search?q=alfred&df=user&sort=age:asc&from=4&size=10&timeout=1s

查询user字段包含alfred的文档，结果按照age升序排序，返回第5个-到第14个文档，如果超过1s没有结束，则超时结束

查看这个查询es执行的过程

布尔操作符

范围查询，支持数值和日期

Query DSL - 字段类查询

fuzziness 模糊一个character，（可以错一个，多一个，少一个）
GET test_index/_search
{
  "_source": "name",
  "profile": "true", 
  "query": {
    "match": {
      "name.py_name":{
        "query":  "占金兵1",
        "analyzer": "keyword",
        "fuzziness": 1
      }
    }
  }
}
minimum_should_match至少包含几个
GET test/_search
{
  "query": {
    "match": {
      "name": {
        "query": "zhan jin bing1",
        "minimum_should_match": 2
      }
    }
  }
}

相关性算分

document frequency 说明：查询的词中有一个词只在很少的文档中出现了，用户想得到的结果很可能在这些少数的文档中，所以加大权重，相关性得分高

相关性算分 - TF/IDF 模型

GET test/_search
{
  "explain": true,
  "query": {
    "match": {
      "name": "zhan jin"
    }
  }
}

match_phrase

通过测试可以发现match_phrase也会对查询的语句进行分词处理，在与倒排序单词匹配的时候有顺序要求

match_phrase
match_phrase slop

因为match_phrase查询有顺序要求，缺了一个都不行，slop就是一个定义match_phrase可以缺几个的字段。可以理解slop默认为0

slop 
GET test_index/_search
{
  "query": {
    "match_phrase": {
      "name": {
        "query": "金兵",
        "analyzer": "keyword",
        "slop": 1
      }
    }
  }
}

regexp 正则查询

正则查询的语句是不会分词的，如果查询的字段是分词的，那查询就会有问题，可以设置一个子字段不分词


最后查询的语句，只显示可以查到的语句
GET test_index/_search
{
  "profile": "true", 
  "query": {
    "regexp": {
      "name.py_name": "[忆占](江南|金兵)"
    }
  }
}

query_string

query_string就是URI方式的q，df就是default_field,也可以指定fields字段数组

query_string

Simple Query String Query

Simple Query String Query

注意simple_query_string容错性很好

Term和Terms

match会对查询的语句进行分词处理，但是有时候我们并不想查询的词被分词处理，这时候就可以使用term和terms了

image.png

Range Query - Date Math

range查询用于数值范围和日期范围的查询

假设now为 2018-01-02 12:00:00，那么如下的计算结果实际为：

计算公式 实际结果
now+1h 2018-01-02 13:00:00
now-1h 2018-01-02 11:00:00
now-1h/d 2018-01-02 00:00:00
2016-01-01||+1M/d 2016-02-01 00:00:00

上面的公式不能有空格

GET test/_search
{
  "profile": true,
  "query": {
    "range": {
      "age": {
        "gte": 25,
        "lte": 29
      }
    }
  }
}

GET test/_search
{
  "query": {
    "range": {
      "birth": {
        "gte": "01/1990/10||+4y"
      }
    }
  }
}

Query DSL - 复合查询

Bool Query
子句key 说明
filter 只过滤符合条件的文档，不计算相关性得分
must 文档必须符合must中的所有条件，会影响相关性得分
must_not 文档必须不符合must_not中的所有条件
should 文档可以符合should中的条件，会影响相关性得分
bool格式 
GET test/_search
{
  "query": {
   "bool": {
     "filter": {
       "term": {
         "name": "zhan"
       }
     }
   }
  }
}

GET test/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "name": "zjb"
          }
        },
        {
          "match": {
            "age": 22
          }
        }
      ]
    }
  }
}

GET test/_search
{
  "query": {
    "bool": {
      "must_not": [
        {
          "match": {
            "name": "zjb"
          }
        },
        {
          "match": {
            "age": 22
          }
        }
      ]
    }
  }
}


GET test/_search
{
  "query": {
    "bool": {
      "should": [
        {
          "match": {
            "name": "zjb"
          }
        },
        {
          "match": {
            "age": 22
          }
        }
      ]
    }
  }
}

注意：当只有should子句没有must子句时，should子句是必须至少要匹配一个条件的document才会返回，当有must子句时，只要满足了must的条件就会返回，should子句中的条件可以不必满足，满不满足只是对相关性得分有影响

Query Context VS Filter Context

上下文类型 执行类型 使用方式
Query 查找与查询语句最匹配的文档，对所有文档进行相关性算分并排序 * query
* bool中的must和should
Filter 查询与查询语句相匹配的文档 * bool 中的 filter 与 must_not
* constant_score 中的filter

查询匹配的数量count 

GET test/_count
{
  "query": {
    "match_all": {
      
    }
  }
}

指定返回的source 

GET test/_search
{
  "_source": false,
  "query": {
    "match_all": {}
  }
}
GET test/_search
{
  "_source": "name", 
  "query": {
    "match_all": {}
  }
}
GET test/_search
{
  "_source": ["name","age"], 
  "query": {
    "match_all": {}
  }
}
GET test/_search
{
  "_source": {
    "includes": "n*"
  }, 
  "query": {
    "match_all": {}
  }
}
GET test/_search
{
  "_source": {
    "includes": "n*",
    "excludes": "name"
  },
  "query": {
    "match_all": {}
  }
}
_source用法

分布式特性

安装cerebro进行可视化集群状态监控

启动一个节点

不同的node需要不同的数据目录和端口，如果在同一台机器上部署多个实例，启动节点时可以使用-Epath.data=数据目录 -Ehttp.port=端口

PUT /stconvert/
{
  "settings": {
    "analysis": {
      "analyzer": {
        "tsconvert": {
          "tokenizer": "tsconvert"
        }
      },
      "char_filter": {
        "tsconvert": {
          "type": "stconvert",
          "convert_type": "t2s"
        }
      },
      "tokenizer": {
        "tsconvert": {
          "type": "stconvert",
          "delimiter": "#",
          "keep_both": false,
          "convert_type": "t2s"
        }
      },
      "filter": {
        "tsconvert": {
          "type": "stconvert",
          "delimiter": "#",
          "keep_both": false,
          "convert_type": "t2s"
        }
      }
    }
  }
}


GET stconvert/_analyze
{
  "tokenizer": "keyword",
  "filter": [
    "lowercase"
  ],
  "char_filter": [
    "tsconvert"
  ],
  "text": "擴大寄生蜂離開的"
}

Fielddata vs DocValues

对比 FieldData DocValues
创建时机 搜索时即时创建 索引 时创建，与倒排索引创建时机一致
创建位置 JVM Heap 磁盘
优点 不会占用额外的磁盘空间 不会占用Heap内存
缺点 文档过多时，即时创建会花过多时间，占用过多的Heap内存 减慢索引的速度，占用额外的磁盘资源
image.png 
GET _analyze
{
  "tokenizer": "whitespace",
  "filter": [
    {
      "type": "stop",
      "ignore_case": true,
      "stopwords": [
        "这",
        "个",
        "一",
        "是"
      ]
    },
    {
      "type": "synonym",
      "lenient": true,
      "expand": true,
      "synonyms": [
        "占金兵 => 易江南"
      ]
    }
  ],
  "text": "这 是 一 个 占金兵"
}
上一篇 下一篇

猜你喜欢

热点阅读