springboot2+es7使用RestHighLevelClient的示例代码
目录
- 一、引入依赖jar
- 二、application.properties配置
- 三、使用
- 其它
由于spring和es的集成并不是特别友好,es的高低版本兼容问题、api更新频率高等问题,所以我选择是官网提供的原生Client(RestHighLevelClient),但又不想去关注es的配置类以及和spring的集成配置、jar包冲突等问题,所以使用spring-boot-starter-data-elasticsearch。
一、引入依赖jar
<dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-elasticsearch</artifactId> </dependency>
二、application.properties配置
spring.elasticsearch.rest.uris=http://127.0.0.1:9200,http://127.0.0.1:9201,http://127.0.0.1:9202 spring.elasticsearch.rest.connection-timeout=5s spring.elasticsearch.rest.read-timeout=30s logging.level.org.springframework.data.convert.CustomConversions=error
spring-boot-starter-data-elasticsearch中自动装配es的配置类:ElasticsearchRestClientAutoConfiguration、ElasticsearchRestClientProperties。
ElasticsearchRestClientAutoConfiguration:
@ConditionalOnClass({RestHighLevelClient.class}) @ConditionalOnMissingBean({RestClient.class}) @EnableConfigurationProperties({ElasticsearchRestClientProperties.class}) public class ElasticsearchRestClientAutoConfiguration { @Configuration( proxyBeanMethods = false ) @ConditionalOnMissingBean({RestHighLevelClient.class}) static class RestHighLevelClientConfiguration { RestHighLevelClientConfiguration() { } @Bean RestHighLevelClient elasticsearchRestHighLevelClient(RestClientBuilder restClientBuilder) { return new RestHighLevelClient(restClientBuilder); } } @Configuration( proxyBeanMethods = false ) @ConditionalOnMissingBean({RestClientBuilder.class}) static class RestClientBuilderConfiguration { RestClientBuilderConfiguration() { } @Bean RestClientBuilderCustomizer defaultRestClientBuilderCustomizer(ElasticsearchRestClientProperties properties) { return new ElasticsearchRestClientAutoConfiguration.DefaultRestClientBuilderCustomizer(properties); } @Bean RestClientBuilder elasticsearchRestClientBuilder(ElasticsearchRestClientProperties properties, ObjectProvider<RestClientBuilderCustomizer> builderCustomizers) { HttpHost[] hosts = (HttpHost[])properties.getUris().stream().map(this::createHttpHost).toArray((x$0) -> { return new HttpHost[x$0]; }); RestClientBuilder builder = RestClient.builder(hosts); builder.setHttpClientConfigCallback((httpClientBuilder) -> { builderCustomizers.orderedStream().forEach((customizer) -> { customizer.customize(httpClientBuilder); }); return httpClientBuilder; }); builder.setRequestConfigCallback((requestConfigBuilder) -> { builderCustomizers.orderedStream().forEach((customizer) -> { customizer.customize(requestConfigBuilder); }); return requestConfigBuilder; }); builderCustomizers.orderedStream().forEach((customizer) -> { customizer.customize(builder); }); return builder; } private HttpHost createHttpHost(String uri) { try { return this.createHttpHost(URI.create(uri)); } catch (IllegalArgumentException var3) { return HttpHost.create(uri); } } private HttpHost createHttpHost(URI uri) { if (!StringUtils.hasLength(uri.getUserInfo())) { return HttpHost.create(uri.toString()); } else { try { return HttpHost.create((new URI(uri.getScheme(), (String)null, uri.getHost(), uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment())).toString()); } catch (URISyntaxException var3) { throw new IllegalStateException(var3); } } } } }
ElasticsearchRestClientProperties:
@ConfigurationProperties( prefix = "spring.elasticsearch.rest" ) public class ElasticsearchRestClientProperties { private List<String> uris = new ArrayList(Collections.singletonList("http://localhost:9200")); private String username; private String password; private Duration connectionTimeout = Duration.ofSeconds(1L); private Duration readTimeout = Duration.ofSeconds(30L); public ElasticsearchRestClientProperties() { } public List<String> getUris() { return this.uris; } public void setUris(List<String> uris) { this.uris = uris; } public String getUsername() { return this.username; } public void setUsername(String username) { this.username = username; } public String getPassword() { return this.password; } public void setPassword(String password) { this.password = password; } public Duration getConnectionTimeout() { return this.connectionTimeout; } public void setConnectionTimeout(Duration connectionTimeout) { this.connectionTimeout = connectionTimeout; } public Duration getReadTimeout() { return this.readTimeout; } public void setReadTimeout(Duration readTimeout) { this.readTimeout = readTimeout; } }
三、使用
ES基本操作持久层
/** * es持久层 * * @author yangzihe * @date 2022/1/24 */ @Repository @Slf4j public class EsRepository { @Resource private RestHighLevelClient highLevelClient; /** * 判断索引是否存在 */ public boolean existIndex(String index) { try { return highLevelClient.indices().exists(new GetIndexRequest(index), RequestOptions.DEFAULT); } catch (IOException e) { log.error("es持久层异常!index={}", index, e); } return Boolean.FALSE; } /** * 创建索引 */ public boolean createIndex(String index, Map<String, Object> columnMap) { if (existIndex(index)) { return Boolean.FALSE; } CreateIndexRequest request = new CreateIndexRequest(index); if (columnMap != null && columnMap.size() > 0) { Map<String, Object> source = new HashMap<>(); source.put("properties", columnMap); request.mapping(source); } try { highLevelClient.indices().create(request, RequestOptions.DEFAULT); return Boolean.TRUE; } catch (IOException e) { log.error("es持久层异常!index={}, columnMap={}", index, columnMap, e); } return Boolean.FALSE; } /** * 删除索引 */ public boolean deleteIndex(String index) { try { if (existIndex(index)) { AcknowledgedResponse response = highLevelClient.indices().delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT); return response.isAcknowledged(); } } catch (Exception e) { log.error("es持久层异常!index={}", index, e); } return Boolean.FALSE; } /** * 数据新增 */ public boolean insert(String index, String jsonString) { IndexRequest indexRequest = new IndexRequest(index); indexRequest.id(new Snowflake().nextIdStr()); indexRequest.source(jsonString, XContentType.JSON); try { log.info("indexRequest={}", indexRequest); IndexResponse indexResponse = highLevelClient.index(indexRequest, RequestOptions.DEFAULT); log.info("indexResponse={}", indexResponse); return Boolean.TRUE; } catch (IOException e) { log.error("es持久层异常!index={}, jsonString={}", index, jsonString, e); } return Boolean.FALSE; } /** * 数据更新,可以直接修改索引结构 */ public boolean update(String index, Map<String, Object> dataMap) { UpdateRequest updateRequest = new UpdateRequest(index, dataMap.remove("id").toString()); updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); updateRequest.doc(dataMap); try { highLevelClient.update(updateRequest, RequestOptions.DEFAULT); } catch (IOException e) { log.error("es持久层异常!index={}, dataMap={}", index, dataMap, e); return Boolean.FALSE; } return Boolean.TRUE; } /** * 删除数据 */ public boolean delete(String index, String id) { DeleteRequest deleteRequest = new DeleteRequest(index, id); try { highLevelClient.delete(deleteRequest, RequestOptions.DEFAULT); } catch (IOException e) { log.error("es持久层异常!index={}, id={}", index, id, e); return Boolean.FALSE; } return Boolean.TRUE; } }
ES查询持久层
/** * es查询持久层 * * @author yangzihe * @date 2022/1/25 */ @Repository @Slf4j public class EsSearchRepository { @Resource private RestHighLevelClient highLevelClient; /** * 分页查询 * * @param queryPO 分页查询对象 * * @return 分页查询结果 */ public EsQueryRespPO<Map<String, Object>> searchPage(EsQueryReqPO queryPO) { // 默认分页参数设置 if (queryPO.getPageNum() == null) { queryPO.setPageNum(1); } if (queryPO.getPageSize() == null) { queryPO.setPageSize(10); } // 设置索引 SearchRequest searchRequest = new SearchRequest(queryPO.getIndex()); // 封装查询源对象 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); searchRequest.source(sourceBuilder); // 查询条件 sourceBuilder.query(queryPO.getQuery()); // 排序字段 if (StringUtils.isNotBlank(queryPO.getSortField()) && queryPO.getSort() != null) { FieldSortBuilder order = new FieldSortBuilder(queryPO.getSortField()).order(queryPO.getSort()); sourceBuilder.sort(order); } // 开始行数,默认0 sourceBuilder.from((queryPO.getPageNum() - 1) * queryPO.getPageSize()); // 页大小,默认10 sourceBuilder.size(queryPO.getPageSize()); // 查询结果 SearchResponse searchResponse = null; try { // log.info("es查询请求对象:searchRequest={}", searchRequest); log.info("es查询请求对象source:sourceBuilder={}", searchRequest.source()); // 执行搜索 searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); log.info("es查询响应结果:searchResponse={}", searchResponse); } catch (IOException e) { log.error("es查询,IO异常!searchRequest={}", searchRequest, e); // 异常处理 return EsQueryRespPO.error("es查询,IO异常!"); } if (RestStatus.OK.equals(searchResponse.status())) { // 解析对象 SearchHit[] hits = searchResponse.getHits().getHits(); // 获取source List<Map<String, Object>> sourceList = Arrays.stream(hits).map(SearchHit::getSourceAsMap).collect(Collectors.toList()); long totalHits = searchResponse.getHits().getTotalHits().value; return EsQueryRespPO.success(sourceList, queryPO.getPageNum(), queryPO.getPageSize(), totalHits); } else { log.error("es查询返回的状态码异常!searchResponse.status={}, searchRequest={}", searchResponse.status(), searchRequest); return EsQueryRespPO.error("es查询返回的状态码异常!"); } } /** * 聚合的分页查询 * * @param queryPO 查询请求对象 * * @return 聚合分页查询结果 */ public EsQueryRespPO<AggregationBucketPO> searchAggregation(EsQueryReqPO queryPO) { // 设置索引 SearchRequest searchRequest = new SearchRequest(queryPO.getIndex()); // 封装查询源对象 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); searchRequest.source(sourceBuilder); // 查询条件 sourceBuilder.query(queryPO.getQuery()); // 排序字段 if (StringUtils.isNotBlank(queryPO.getSortField()) && queryPO.getSort() != null) { FieldSortBuilder order = new FieldSortBuilder(queryPO.getSortField()).order(queryPO.getSort()); sourceBuilder.sort(order); } // 页大小0,只返回聚合结果 sourceBuilder.size(0); // 设置聚合查询,可以设置多个聚合查询条件,只要聚合查询命名不同就行 // 聚合分组条件, group by sourceBuilder.aggregation(queryPO.getTermsAggregation()); // 聚合统计条件, count分组后的数据,计算分组后的总大小 sourceBuilder.aggregation(queryPO.getCardinalityAggregation()); // 查询结果 SearchResponse searchResponse = null; try { // log.info("es查询请求对象:searchRequest={}", searchRequest); log.info("es查询请求对象source:sourceBuilder={}", searchRequest.source()); // 执行搜索 searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); log.info("es查询响应结果:searchResponse={}", searchResponse); } catch (IOException e) { log.error("es查询,IO异常!searchRequest={}", searchRequest, e); return EsQueryRespPO.error("es查询,IO异常!"); } if (RestStatus.OK.equals(searchResponse.status())) { // 解析对象 Aggregations aggregations = searchResponse.getAggregations(); long docTotal = searchResponse.getHits().getTotalHits().value; // 遍历terms聚合结果 Terms terms = aggregations.get(queryPO.getTermsAggregation().getName()); List<AggregationBucketPO> bucketList = terms.getBuckets().stream().map(bucket -> { String key = bucket.getKeyAsString(); long docCount = bucket.getDocCount(); return new AggregationBucketPO(key, docCount, docTotal); }).collect(Collectors.toList()); // 总数量 Cardinality cardinality = aggregations.get(queryPO.getCardinalityAggregation().getName()); long totalHits = cardinality.getValue(); return EsQueryRespPO.success(bucketList, queryPO.getPageNum(), queryPO.getPageSize(), totalHits); } else { log.error("es查询返回的状态码异常!searchResponse.status={}, searchRequest={}", searchResponse.status(), searchRequest); return EsQueryRespPO.error("es查询返回的状态码异常!"); } } }
其中,EsQueryReqPO、EsQueryRespPO、AggregationBucketPO等类如下:
/** * es查询请求对象 */ @Data public class EsQueryReqPO { /** * 索引名 */ String[] index; /** * 查询条件 */ QueryBuilder query; /** * 排序字段 */ String sortField; /** * 排序方式 SortOrder.ASC、SortOrder.DESC */ SortOrder sort; /** * 页数 */ private Integer pageNum; /** * 页大小 */ private Integer pageSize; /** * 聚合分组条件, group by */ private TermsAggregationBuilder termsAggregation; /** * 聚合统计条件, count分组后的数据 */ private CardinalityAggregationBuilder cardinalityAggregation; } /** * es分页响应对象 * * @author yangzihe * @date 2022/1/25 */ @Data @NoArgsConstructor @AllArgsConstructor public class EsQueryRespPO<T> { /** * 是否成功 */ private Boolean success; /** * 信息 */ private String message; /** * 页数 */ private Integer pageNum; /** * 页大小 */ private Integer pageSize; /** * 总大小 */ private Long totalSize; /** * 数据 */ private List<T> sourceList; public static <T> EsQueryRespPO<T> success(List<T> sourceList, Integer pageNum, Integer pageSize, Long totalSize) { EsQueryRespPO<T> esQueryRespPO = new EsQueryRespPO<>(); esQueryRespPO.setSuccess(true); esQueryRespPO.setSourceList(sourceList); esQueryRespPO.setPageNum(pageNum); esQueryRespPO.setPageSize(pageSize); esQueryRespPO.setTotalSize(totalSize); return esQueryRespPO; } public static EsQueryRespPO error() { EsQueryRespPO esQueryRespPO = new EsQueryRespPO(); esQueryRespPO.setSuccess(false); esQueryRespPO.setMessage("es查询异常"); return esQueryRespPO; } public static EsQueryRespPO error(String message) { EsQueryRespPO esQueryRespPO = new EsQueryRespPO(); esQueryRespPO.setSuccess(false); esQueryRespPO.setMessage(message); return esQueryRespPO; } }
/** * 聚合桶对象 * * @author yangzihe * @date 2022/1/26 */ @Data @NoArgsConstructor @AllArgsConstructor public class AggregationBucketPO { /** * 聚合Bucket的key名 */ private String key; /** * 聚合Bucket的文档数量 */ private Long docCount; /** * 文档总数量 */ private Long docTotal; }
ES多级(二级)聚合分桶查询
import com.yy.armor.manager.common.exception.EsException; import com.yy.armor.manager.persist.es.po.AggregationBucketPO; import com.yy.armor.manager.persist.es.po.EsMultiAggregationReqPO; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.shade.org.apache.commons.compress.utils.Lists; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.springframework.stereotype.Repository; @Repository @Slf4j public class EsSearchRepository { @Resource private RestHighLevelClient highLevelClient; /** * 多级聚合查询(二级聚合) * * @param reqPO 查询请求对象 * * @return 聚合查询结果 */ public List<AggregationBucketPO> searchMultiAggregation(EsMultiAggregationReqPO reqPO) { // 设置索引 SearchRequest searchRequest = new SearchRequest(reqPO.getIndex()); // 封装查询源对象 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); searchRequest.source(sourceBuilder); // 查询条件 sourceBuilder.query(reqPO.getQuery()); // 排序字段 if (StringUtils.isNotBlank(reqPO.getSortField()) && reqPO.getSort() != null) { FieldSortBuilder order = new FieldSortBuilder(reqPO.getSortField()).order(reqPO.getSort()); sourceBuilder.sort(order); } // 页大小0,只返回聚合结果 sourceBuilder.size(0); // 聚合分桶。创建terms桶聚合,聚合名字=terms_by_XXX, 字段=XXX TermsAggregationBuilder termsAggregationBuilder = AggregationBuilders.terms("terms_by_" + reqPO.getField()).field(reqPO.getField()); if (reqPO.getFieldSize() != null) { termsAggregationBuilder.size(reqPO.getFieldSize()); } // 二级聚合分桶 TermsAggregationBuilder subTermsAggregationBuilder = AggregationBuilders.terms("terms_by_" + reqPO.getSubField()).field(reqPO.getSubField()); if (reqPO.getSubFieldSize() != null) { subTermsAggregationBuilder.size(reqPO.getSubFieldSize()); } termsAggregationBuilder.subAggregation(subTermsAggregationBuilder); // 聚合分组条件 sourceBuilder.aggregation(termsAggregationBuilder); // 查询结果 SearchResponse searchResponse = null; try { log.info("es查询请求对象source:sourceBuilder={}", searchRequest.source()); // 执行搜索 searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT); log.info("es查询响应结果:searchResponse={}", searchResponse); } catch (IOException e) { log.error("es查询,IO异常!searchRequest={}", searchRequest, e); throw new EsException("es查询,IO异常!"); } if (RestStatus.OK.equals(searchResponse.status())) { // 遍历terms聚合结果 Terms terms = searchResponse.getAggregations().get(termsAggregationBuilder.getName()); List<AggregationBucketPO> bucketList = terms.getBuckets().stream().map(bucket -> { // 一级聚合分桶的数据 String key = bucket.getKeyAsString(); long docCount = bucket.getDocCount(); // 二级聚合分桶的数据 Terms subTerms = bucket.getAggregations().get(subTermsAggregationBuilder.getName()); List<AggregationBucketPO> subBucketList = convertTerms(subTerms); return new AggregationBucketPO(key, docCount, subBucketList); }).collect(Collectors.toList()); return bucketList; } else { log.error("es查询返回的状态码异常!searchResponse.status={}, searchRequest={}", searchResponse.status(), searchRequest); throw new EsException("es查询返回的状态码异常!"); } } private List<AggregationBucketPO> convertTerms(Terms terms) { if (CollectionUtils.isEmpty(terms.getBuckets())) { return Lists.newArrayList(); } return terms.getBuckets().stream().map(bucket -> { String key = bucket.getKeyAsString(); long docCount = bucket.getDocCount(); return new AggregationBucketPO(key, docCount); }).collect(Collectors.toList()); } }
其中,EsMultiAggregationReqPO、AggregationBucketPO类如下:
@Data public class EsMultiAggregationReqPO { /** * 索引名 */ String[] index; /** * 查询条件 */ QueryBuilder query; /** * 聚合分桶字段 */ private String field; /** * 二级聚合分桶字段 */ private String subField; /** * 聚合分桶大小,非必传 */ private Integer fieldSize; /** * 二级聚合分桶大小,非必传 */ private Integer subFieldSize; /** * 排序字段,非必传 */ String sortField; /** * 排序方式 SortOrder.ASC、SortOrder.DESC,非必传 */ SortOrder sort; }
@Data @NoArgsConstructor @AllArgsConstructor public class AggregationBucketPO { /** * 聚合Bucket的key名 */ private String key; /** * 聚合Bucket的文档数量 */ private Long docCount; /** * 子桶集合 */ private List<AggregationBucketPO> subBucketList; public AggregationBucketPO(String key, Long docCount) { this.key = key; this.docCount = docCount; } }
二级聚合分桶测试代码
@PostConstruct private void init() { // 查询对象的封装 EsMultiAggregationReqPO reqPO = new EsMultiAggregationReqPO(); reqPO.setIndex(new String[]{"test_log"}); List<Long> ids = Lists.newArrayList(); ids.add(140L); ids.add(141L); ids.add(142L); QueryBuilder queryBuilder4 = QueryBuilders.termsQuery("eventId", ids); BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(queryBuilder4); reqPO.setQuery(queryBuilder); reqPO.setField("eventId"); reqPO.setFieldSize(9999); reqPO.setSubField("riskFlag"); // 执行查询 List<AggregationBucketPO> esQueryRespPO = searchMultiAggregation(reqPO); System.out.println("esQueryRespPO=" + esQueryRespPO); }
其它
如果没有用spring-boot-starter-data-elasticsearch来自动注入es组件,那么需要自己做es client的注入,es配置类如下:
/** * @author yangzihe * @date 2022/1/25 */ @Configuration public class EsClientConfig { @Value("${spring.elasticsearch.rest.uris}") private List<String> uris; @Bean public RestHighLevelClient restHighLevelClient() { List<HttpHost> httpHostList = uris.stream().map(HttpHost::create).collect(Collectors.toList()); HttpHost[] httpHost = new HttpHost[uris.size()]; httpHostList.toArray(httpHost); RestClientBuilder clientBuilder = RestClient.builder(httpHost); return new RestHighLevelClient(clientBuilder); } }
Snowflake是hutool包里的,导包:
<!-- https://mvnrepository.com/artifact/cn.hutool/hutool-all --> <dependency> <groupId>cn.hutool</groupId> <artifactId>hutool-all</artifactId> <version>5.7.14</version> </dependency>
聚合查询的测试用例:
@RunWith(SpringJUnit4ClassRunner.class) @SpringBootTest(classes = StartApplication.class) public class EsTest { @Resource private EsSearchRepository esSearchRepository; @Test public void testSearchAggregation() { // 查询对象的封装 EsQueryReqPO queryPO = new EsQueryReqPO(); queryPO.setIndex(new String[]{"yzh1", "yzh2"}); queryPO.setPageNum(1); queryPO.setPageSize(10); // 时间戳范围 QueryBuilder queryBuilder1 = QueryBuilders.rangeQuery("timestamp") .from(System.currentTimeMillis() - 1000) .to(System.currentTimeMillis()); // 登录标识 QueryBuilder queryBuilder2 = QueryBuilders.termQuery("name", "yang"); BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(queryBuilder1).must(queryBuilder2); queryPO.setQuery(queryBuilder); // 根据userName分组。创建terms桶聚合,聚合名字=terms_by_userName, 字段=payload.userName.keyword TermsAggregationBuilder termsAggregationBuilder = AggregationBuilders .terms("terms_by_userName").field("payload.userName.keyword"); termsAggregationBuilder.size(queryPO.getPageSize() * queryPO.getPageNum()); termsAggregationBuilder.subAggregation(new BucketSortPipelineAggregationBuilder("bucket_field", null) .from((queryPO.getPageNum() - 1) * queryPO.getPageSize()).size(queryPO.getPageSize())); queryPO.setTermsAggregation(termsAggregationBuilder); // 根据userName聚合count统计. cardinality名=count_userName, 字段=payload.userName.keyword CardinalityAggregationBuilder cardinalityAggregationBuilder = AggregationBuilders .cardinality("count_userName").field("payload.userName.keyword"); queryPO.setCardinalityAggregation(cardinalityAggregationBuilder); // 执行查询 EsQueryRespPO<AggregationBucketPO> esQueryRespPO = esSearchRepository.searchAggregation(queryPO); } }
到此这篇关于springboot2+es7使用RestHighLevelClient的示例代码的文章就介绍到这了,更多相关springboot2 es7使用RestHighLevelClient内容请搜索我们以前的文章或继续浏览下面的相关文章希望大家以后多多支持我们!