001/*- 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2023 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.search.lastn; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.i18n.Msg; 024import ca.uhn.fhir.jpa.dao.TolerantJsonParser; 025import ca.uhn.fhir.jpa.model.config.PartitionSettings; 026import ca.uhn.fhir.jpa.model.util.CodeSystemHash; 027import ca.uhn.fhir.jpa.search.lastn.json.CodeJson; 028import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson; 029import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; 030import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper; 031import ca.uhn.fhir.model.api.IQueryParameterType; 032import ca.uhn.fhir.parser.IParser; 033import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; 034import ca.uhn.fhir.rest.param.DateParam; 035import ca.uhn.fhir.rest.param.ParamPrefixEnum; 036import ca.uhn.fhir.rest.param.ReferenceParam; 037import ca.uhn.fhir.rest.param.TokenParam; 038import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 039import com.fasterxml.jackson.core.JsonProcessingException; 040import com.fasterxml.jackson.databind.ObjectMapper; 041import com.google.common.annotations.VisibleForTesting; 042import org.apache.commons.lang3.Validate; 043import org.elasticsearch.action.DocWriteResponse; 044import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; 045import org.elasticsearch.action.index.IndexRequest; 046import org.elasticsearch.action.index.IndexResponse; 047import org.elasticsearch.action.search.SearchRequest; 048import org.elasticsearch.action.search.SearchResponse; 049import org.elasticsearch.client.RequestOptions; 050import org.elasticsearch.client.RestHighLevelClient; 051import org.elasticsearch.client.indices.CreateIndexRequest; 052import org.elasticsearch.client.indices.CreateIndexResponse; 053import org.elasticsearch.client.indices.GetIndexRequest; 054import org.elasticsearch.index.query.BoolQueryBuilder; 055import org.elasticsearch.index.query.MatchQueryBuilder; 056import org.elasticsearch.index.query.QueryBuilders; 057import org.elasticsearch.index.query.RangeQueryBuilder; 058import org.elasticsearch.index.reindex.DeleteByQueryRequest; 059import org.elasticsearch.search.SearchHit; 060import org.elasticsearch.search.SearchHits; 061import org.elasticsearch.search.aggregations.AggregationBuilder; 062import org.elasticsearch.search.aggregations.AggregationBuilders; 063import org.elasticsearch.search.aggregations.Aggregations; 064import org.elasticsearch.search.aggregations.BucketOrder; 065import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; 066import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; 067import org.elasticsearch.search.aggregations.bucket.composite.ParsedComposite; 068import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; 069import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; 070import org.elasticsearch.search.aggregations.bucket.terms.Terms; 071import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; 072import org.elasticsearch.search.aggregations.metrics.ParsedTopHits; 073import org.elasticsearch.search.builder.SearchSourceBuilder; 074import org.elasticsearch.search.sort.SortOrder; 075import org.elasticsearch.xcontent.XContentType; 076import org.hl7.fhir.instance.model.api.IBaseResource; 077import org.slf4j.Logger; 078import org.slf4j.LoggerFactory; 079import org.springframework.beans.factory.annotation.Autowired; 080 081import java.io.BufferedReader; 082import java.io.IOException; 083import java.io.InputStreamReader; 084import java.util.ArrayList; 085import java.util.Arrays; 086import java.util.Collection; 087import java.util.List; 088import java.util.function.Function; 089import java.util.stream.Collectors; 090import javax.annotation.Nullable; 091 092import static org.apache.commons.lang3.StringUtils.isBlank; 093 094public class ElasticsearchSvcImpl implements IElasticsearchSvc { 095 096 private static final Logger ourLog = LoggerFactory.getLogger(ElasticsearchSvcImpl.class); 097 098 // Index Constants 099 public static final String OBSERVATION_INDEX = "observation_index"; 100 public static final String OBSERVATION_CODE_INDEX = "code_index"; 101 public static final String OBSERVATION_DOCUMENT_TYPE = 102 "ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity"; 103 public static final String CODE_DOCUMENT_TYPE = 104 "ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity"; 105 public static final String OBSERVATION_INDEX_SCHEMA_FILE = "ObservationIndexSchema.json"; 106 public static final String OBSERVATION_CODE_INDEX_SCHEMA_FILE = "ObservationCodeIndexSchema.json"; 107 108 // Aggregation Constants 109 private static final String GROUP_BY_SUBJECT = "group_by_subject"; 110 private static final String GROUP_BY_SYSTEM = "group_by_system"; 111 private static final String GROUP_BY_CODE = "group_by_code"; 112 private static final String MOST_RECENT_EFFECTIVE = "most_recent_effective"; 113 114 // Observation index document element names 115 private static final String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier"; 116 private static final String OBSERVATION_SUBJECT_FIELD_NAME = "subject"; 117 private static final String OBSERVATION_CODEVALUE_FIELD_NAME = "codeconceptcodingcode"; 118 private static final String OBSERVATION_CODESYSTEM_FIELD_NAME = "codeconceptcodingsystem"; 119 private static final String OBSERVATION_CODEHASH_FIELD_NAME = "codeconceptcodingcode_system_hash"; 120 private static final String OBSERVATION_CODEDISPLAY_FIELD_NAME = "codeconceptcodingdisplay"; 121 private static final String OBSERVATION_CODE_TEXT_FIELD_NAME = "codeconcepttext"; 122 private static final String OBSERVATION_EFFECTIVEDTM_FIELD_NAME = "effectivedtm"; 123 private static final String OBSERVATION_CATEGORYHASH_FIELD_NAME = "categoryconceptcodingcode_system_hash"; 124 private static final String OBSERVATION_CATEGORYVALUE_FIELD_NAME = "categoryconceptcodingcode"; 125 private static final String OBSERVATION_CATEGORYSYSTEM_FIELD_NAME = "categoryconceptcodingsystem"; 126 private static final String OBSERVATION_CATEGORYDISPLAY_FIELD_NAME = "categoryconceptcodingdisplay"; 127 private static final String OBSERVATION_CATEGORYTEXT_FIELD_NAME = "categoryconcepttext"; 128 129 // Code index document element names 130 private static final String CODE_HASH = "codingcode_system_hash"; 131 private static final String CODE_TEXT = "text"; 132 133 private static final String OBSERVATION_RESOURCE_NAME = "Observation"; 134 135 private final RestHighLevelClient myRestHighLevelClient; 136 137 private final ObjectMapper objectMapper = new ObjectMapper(); 138 139 @Autowired 140 private PartitionSettings myPartitionSettings; 141 142 @Autowired 143 private FhirContext myContext; 144 145 // This constructor used to inject a dummy partitionsettings in test. 146 public ElasticsearchSvcImpl( 147 PartitionSettings thePartitionSetings, 148 String theProtocol, 149 String theHostname, 150 @Nullable String theUsername, 151 @Nullable String thePassword) { 152 this(theProtocol, theHostname, theUsername, thePassword); 153 this.myPartitionSettings = thePartitionSetings; 154 } 155 156 public ElasticsearchSvcImpl( 157 String theProtocol, String theHostname, @Nullable String theUsername, @Nullable String thePassword) { 158 myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient( 159 theProtocol, theHostname, theUsername, thePassword); 160 161 try { 162 createObservationIndexIfMissing(); 163 createObservationCodeIndexIfMissing(); 164 } catch (IOException theE) { 165 throw new RuntimeException(Msg.code(1175) + "Failed to create document index", theE); 166 } 167 } 168 169 private String getIndexSchema(String theSchemaFileName) throws IOException { 170 InputStreamReader input = 171 new InputStreamReader(ElasticsearchSvcImpl.class.getResourceAsStream(theSchemaFileName)); 172 BufferedReader reader = new BufferedReader(input); 173 StringBuilder sb = new StringBuilder(); 174 String str; 175 while ((str = reader.readLine()) != null) { 176 sb.append(str); 177 } 178 179 return sb.toString(); 180 } 181 182 private void createObservationIndexIfMissing() throws IOException { 183 if (indexExists(OBSERVATION_INDEX)) { 184 return; 185 } 186 String observationMapping = getIndexSchema(OBSERVATION_INDEX_SCHEMA_FILE); 187 if (!createIndex(OBSERVATION_INDEX, observationMapping)) { 188 throw new RuntimeException(Msg.code(1176) + "Failed to create observation index"); 189 } 190 } 191 192 private void createObservationCodeIndexIfMissing() throws IOException { 193 if (indexExists(OBSERVATION_CODE_INDEX)) { 194 return; 195 } 196 String observationCodeMapping = getIndexSchema(OBSERVATION_CODE_INDEX_SCHEMA_FILE); 197 if (!createIndex(OBSERVATION_CODE_INDEX, observationCodeMapping)) { 198 throw new RuntimeException(Msg.code(1177) + "Failed to create observation code index"); 199 } 200 } 201 202 private boolean createIndex(String theIndexName, String theMapping) throws IOException { 203 CreateIndexRequest request = new CreateIndexRequest(theIndexName); 204 request.source(theMapping, XContentType.JSON); 205 CreateIndexResponse createIndexResponse = 206 myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT); 207 return createIndexResponse.isAcknowledged(); 208 } 209 210 private boolean indexExists(String theIndexName) throws IOException { 211 GetIndexRequest request = new GetIndexRequest(theIndexName); 212 return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT); 213 } 214 215 @Override 216 public List<String> executeLastN( 217 SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, Integer theMaxResultsToFetch) { 218 Validate.isTrue( 219 !myPartitionSettings.isPartitioningEnabled(), 220 "$lastn is not currently supported on partitioned servers"); 221 222 String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME}; 223 return buildAndExecuteSearch( 224 theSearchParameterMap, 225 theFhirContext, 226 topHitsInclude, 227 ObservationJson::getIdentifier, 228 theMaxResultsToFetch); 229 } 230 231 private <T> List<T> buildAndExecuteSearch( 232 SearchParameterMap theSearchParameterMap, 233 FhirContext theFhirContext, 234 String[] topHitsInclude, 235 Function<ObservationJson, T> setValue, 236 Integer theMaxResultsToFetch) { 237 String patientParamName = LastNParameterHelper.getPatientParamName(theFhirContext); 238 String subjectParamName = LastNParameterHelper.getSubjectParamName(theFhirContext); 239 List<T> searchResults = new ArrayList<>(); 240 if (theSearchParameterMap.containsKey(patientParamName) 241 || theSearchParameterMap.containsKey(subjectParamName)) { 242 for (String subject : 243 getSubjectReferenceCriteria(patientParamName, subjectParamName, theSearchParameterMap)) { 244 if (theMaxResultsToFetch != null && searchResults.size() >= theMaxResultsToFetch) { 245 break; 246 } 247 SearchRequest myLastNRequest = buildObservationsSearchRequest( 248 subject, 249 theSearchParameterMap, 250 theFhirContext, 251 createObservationSubjectAggregationBuilder( 252 getMaxParameter(theSearchParameterMap), topHitsInclude)); 253 ourLog.debug("ElasticSearch query: {}", myLastNRequest.source().toString()); 254 try { 255 SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); 256 searchResults.addAll(buildObservationList( 257 lastnResponse, setValue, theSearchParameterMap, theFhirContext, theMaxResultsToFetch)); 258 } catch (IOException theE) { 259 throw new InvalidRequestException(Msg.code(1178) + "Unable to execute LastN request", theE); 260 } 261 } 262 } else { 263 SearchRequest myLastNRequest = buildObservationsSearchRequest( 264 theSearchParameterMap, 265 theFhirContext, 266 createObservationCodeAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude)); 267 ourLog.debug("ElasticSearch query: {}", myLastNRequest.source().toString()); 268 try { 269 SearchResponse lastnResponse = executeSearchRequest(myLastNRequest); 270 searchResults.addAll(buildObservationList( 271 lastnResponse, setValue, theSearchParameterMap, theFhirContext, theMaxResultsToFetch)); 272 } catch (IOException theE) { 273 throw new InvalidRequestException(Msg.code(1179) + "Unable to execute LastN request", theE); 274 } 275 } 276 return searchResults; 277 } 278 279 private int getMaxParameter(SearchParameterMap theSearchParameterMap) { 280 if (theSearchParameterMap.getLastNMax() == null) { 281 return 1; 282 } else { 283 return theSearchParameterMap.getLastNMax(); 284 } 285 } 286 287 private List<String> getSubjectReferenceCriteria( 288 String thePatientParamName, String theSubjectParamName, SearchParameterMap theSearchParameterMap) { 289 List<String> subjectReferenceCriteria = new ArrayList<>(); 290 291 List<List<IQueryParameterType>> patientParams = new ArrayList<>(); 292 if (theSearchParameterMap.get(thePatientParamName) != null) { 293 patientParams.addAll(theSearchParameterMap.get(thePatientParamName)); 294 } 295 if (theSearchParameterMap.get(theSubjectParamName) != null) { 296 patientParams.addAll(theSearchParameterMap.get(theSubjectParamName)); 297 } 298 for (List<? extends IQueryParameterType> nextSubjectList : patientParams) { 299 subjectReferenceCriteria.addAll(getReferenceValues(nextSubjectList)); 300 } 301 return subjectReferenceCriteria; 302 } 303 304 private List<String> getReferenceValues(List<? extends IQueryParameterType> referenceParams) { 305 List<String> referenceList = new ArrayList<>(); 306 307 for (IQueryParameterType nextOr : referenceParams) { 308 309 if (nextOr instanceof ReferenceParam) { 310 ReferenceParam ref = (ReferenceParam) nextOr; 311 if (isBlank(ref.getChain())) { 312 referenceList.add(ref.getValue()); 313 } 314 } else { 315 throw new IllegalArgumentException( 316 Msg.code(1180) + "Invalid token type (expecting ReferenceParam): " + nextOr.getClass()); 317 } 318 } 319 return referenceList; 320 } 321 322 private CompositeAggregationBuilder createObservationSubjectAggregationBuilder( 323 Integer theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { 324 CompositeValuesSourceBuilder<?> subjectValuesBuilder = 325 new TermsValuesSourceBuilder(OBSERVATION_SUBJECT_FIELD_NAME).field(OBSERVATION_SUBJECT_FIELD_NAME); 326 List<CompositeValuesSourceBuilder<?>> compositeAggSubjectSources = new ArrayList<>(); 327 compositeAggSubjectSources.add(subjectValuesBuilder); 328 CompositeAggregationBuilder compositeAggregationSubjectBuilder = 329 new CompositeAggregationBuilder(GROUP_BY_SUBJECT, compositeAggSubjectSources); 330 compositeAggregationSubjectBuilder.subAggregation( 331 createObservationCodeAggregationBuilder(theMaxNumberObservationsPerCode, theTopHitsInclude)); 332 compositeAggregationSubjectBuilder.size(10000); 333 334 return compositeAggregationSubjectBuilder; 335 } 336 337 private TermsAggregationBuilder createObservationCodeAggregationBuilder( 338 int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) { 339 TermsAggregationBuilder observationCodeCodeAggregationBuilder = 340 new TermsAggregationBuilder(GROUP_BY_CODE).field(OBSERVATION_CODEVALUE_FIELD_NAME); 341 observationCodeCodeAggregationBuilder.order(BucketOrder.key(true)); 342 // Top Hits Aggregation 343 observationCodeCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits(MOST_RECENT_EFFECTIVE) 344 .sort(OBSERVATION_EFFECTIVEDTM_FIELD_NAME, SortOrder.DESC) 345 .fetchSource(theTopHitsInclude, null) 346 .size(theMaxNumberObservationsPerCode)); 347 observationCodeCodeAggregationBuilder.size(10000); 348 TermsAggregationBuilder observationCodeSystemAggregationBuilder = 349 new TermsAggregationBuilder(GROUP_BY_SYSTEM).field(OBSERVATION_CODESYSTEM_FIELD_NAME); 350 observationCodeSystemAggregationBuilder.order(BucketOrder.key(true)); 351 observationCodeSystemAggregationBuilder.subAggregation(observationCodeCodeAggregationBuilder); 352 return observationCodeSystemAggregationBuilder; 353 } 354 355 private SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException { 356 return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); 357 } 358 359 private <T> List<T> buildObservationList( 360 SearchResponse theSearchResponse, 361 Function<ObservationJson, T> setValue, 362 SearchParameterMap theSearchParameterMap, 363 FhirContext theFhirContext, 364 Integer theMaxResultsToFetch) 365 throws IOException { 366 List<T> theObservationList = new ArrayList<>(); 367 if (theSearchParameterMap.containsKey(LastNParameterHelper.getPatientParamName(theFhirContext)) 368 || theSearchParameterMap.containsKey(LastNParameterHelper.getSubjectParamName(theFhirContext))) { 369 for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) { 370 if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { 371 break; 372 } 373 for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) { 374 if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { 375 break; 376 } 377 for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { 378 if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { 379 break; 380 } 381 String indexedObservation = lastNMatch.getSourceAsString(); 382 ObservationJson observationJson = 383 objectMapper.readValue(indexedObservation, ObservationJson.class); 384 theObservationList.add(setValue.apply(observationJson)); 385 } 386 } 387 } 388 } else { 389 for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(theSearchResponse)) { 390 if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { 391 break; 392 } 393 for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) { 394 if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) { 395 break; 396 } 397 String indexedObservation = lastNMatch.getSourceAsString(); 398 ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class); 399 theObservationList.add(setValue.apply(observationJson)); 400 } 401 } 402 } 403 404 return theObservationList; 405 } 406 407 private List<ParsedComposite.ParsedBucket> getSubjectBuckets(SearchResponse theSearchResponse) { 408 Aggregations responseAggregations = theSearchResponse.getAggregations(); 409 ParsedComposite aggregatedSubjects = responseAggregations.get(GROUP_BY_SUBJECT); 410 return aggregatedSubjects.getBuckets(); 411 } 412 413 private List<? extends Terms.Bucket> getObservationCodeBuckets(SearchResponse theSearchResponse) { 414 Aggregations responseAggregations = theSearchResponse.getAggregations(); 415 return getObservationCodeBuckets(responseAggregations); 416 } 417 418 private List<? extends Terms.Bucket> getObservationCodeBuckets(ParsedComposite.ParsedBucket theSubjectBucket) { 419 Aggregations observationCodeSystemAggregations = theSubjectBucket.getAggregations(); 420 return getObservationCodeBuckets(observationCodeSystemAggregations); 421 } 422 423 private List<? extends Terms.Bucket> getObservationCodeBuckets(Aggregations theObservationCodeSystemAggregations) { 424 List<Terms.Bucket> retVal = new ArrayList<>(); 425 ParsedTerms aggregatedObservationCodeSystems = theObservationCodeSystemAggregations.get(GROUP_BY_SYSTEM); 426 for (Terms.Bucket observationCodeSystem : aggregatedObservationCodeSystems.getBuckets()) { 427 Aggregations observationCodeCodeAggregations = observationCodeSystem.getAggregations(); 428 ParsedTerms aggregatedObservationCodeCodes = observationCodeCodeAggregations.get(GROUP_BY_CODE); 429 retVal.addAll(aggregatedObservationCodeCodes.getBuckets()); 430 } 431 return retVal; 432 } 433 434 private SearchHit[] getLastNMatches(Terms.Bucket theObservationCodeBucket) { 435 Aggregations topHitObservationCodes = theObservationCodeBucket.getAggregations(); 436 ParsedTopHits parsedTopHits = topHitObservationCodes.get(MOST_RECENT_EFFECTIVE); 437 return parsedTopHits.getHits().getHits(); 438 } 439 440 private SearchRequest buildObservationsSearchRequest( 441 SearchParameterMap theSearchParameterMap, 442 FhirContext theFhirContext, 443 AggregationBuilder theAggregationBuilder) { 444 SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX); 445 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 446 // Query 447 if (!searchParamsHaveLastNCriteria(theSearchParameterMap, theFhirContext)) { 448 searchSourceBuilder.query(QueryBuilders.matchAllQuery()); 449 } else { 450 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); 451 addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); 452 addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); 453 addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); 454 searchSourceBuilder.query(boolQueryBuilder); 455 } 456 searchSourceBuilder.size(0); 457 458 // Aggregation by order codes 459 searchSourceBuilder.aggregation(theAggregationBuilder); 460 searchRequest.source(searchSourceBuilder); 461 462 return searchRequest; 463 } 464 465 private SearchRequest buildObservationsSearchRequest( 466 String theSubjectParam, 467 SearchParameterMap theSearchParameterMap, 468 FhirContext theFhirContext, 469 AggregationBuilder theAggregationBuilder) { 470 SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX); 471 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 472 // Query 473 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); 474 boolQueryBuilder.must(QueryBuilders.termQuery(OBSERVATION_SUBJECT_FIELD_NAME, theSubjectParam)); 475 addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); 476 addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); 477 addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext); 478 searchSourceBuilder.query(boolQueryBuilder); 479 searchSourceBuilder.size(0); 480 481 // Aggregation by order codes 482 searchSourceBuilder.aggregation(theAggregationBuilder); 483 searchRequest.source(searchSourceBuilder); 484 485 return searchRequest; 486 } 487 488 private Boolean searchParamsHaveLastNCriteria( 489 SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { 490 return theSearchParameterMap != null 491 && (theSearchParameterMap.containsKey(LastNParameterHelper.getPatientParamName(theFhirContext)) 492 || theSearchParameterMap.containsKey(LastNParameterHelper.getSubjectParamName(theFhirContext)) 493 || theSearchParameterMap.containsKey(LastNParameterHelper.getCategoryParamName(theFhirContext)) 494 || theSearchParameterMap.containsKey(LastNParameterHelper.getCodeParamName(theFhirContext))); 495 } 496 497 private void addCategoriesCriteria( 498 BoolQueryBuilder theBoolQueryBuilder, 499 SearchParameterMap theSearchParameterMap, 500 FhirContext theFhirContext) { 501 String categoryParamName = LastNParameterHelper.getCategoryParamName(theFhirContext); 502 if (theSearchParameterMap.containsKey(categoryParamName)) { 503 ArrayList<String> codeSystemHashList = new ArrayList<>(); 504 ArrayList<String> codeOnlyList = new ArrayList<>(); 505 ArrayList<String> systemOnlyList = new ArrayList<>(); 506 ArrayList<String> textOnlyList = new ArrayList<>(); 507 List<List<IQueryParameterType>> andOrParams = theSearchParameterMap.get(categoryParamName); 508 for (List<? extends IQueryParameterType> nextAnd : andOrParams) { 509 codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); 510 codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); 511 systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); 512 textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); 513 } 514 if (codeSystemHashList.size() > 0) { 515 theBoolQueryBuilder.must( 516 QueryBuilders.termsQuery(OBSERVATION_CATEGORYHASH_FIELD_NAME, codeSystemHashList)); 517 } 518 if (codeOnlyList.size() > 0) { 519 theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYVALUE_FIELD_NAME, codeOnlyList)); 520 } 521 if (systemOnlyList.size() > 0) { 522 theBoolQueryBuilder.must( 523 QueryBuilders.termsQuery(OBSERVATION_CATEGORYSYSTEM_FIELD_NAME, systemOnlyList)); 524 } 525 if (textOnlyList.size() > 0) { 526 BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); 527 for (String textOnlyParam : textOnlyList) { 528 myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery( 529 OBSERVATION_CATEGORYDISPLAY_FIELD_NAME, textOnlyParam)); 530 myTextBoolQueryBuilder.should( 531 QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CATEGORYTEXT_FIELD_NAME, textOnlyParam)); 532 } 533 theBoolQueryBuilder.must(myTextBoolQueryBuilder); 534 } 535 } 536 } 537 538 private List<String> getCodingCodeSystemValues(List<? extends IQueryParameterType> codeParams) { 539 ArrayList<String> codeSystemHashList = new ArrayList<>(); 540 for (IQueryParameterType nextOr : codeParams) { 541 if (nextOr instanceof TokenParam) { 542 TokenParam ref = (TokenParam) nextOr; 543 if (ref.getSystem() != null && ref.getValue() != null) { 544 codeSystemHashList.add( 545 String.valueOf(CodeSystemHash.hashCodeSystem(ref.getSystem(), ref.getValue()))); 546 } 547 } else { 548 throw new IllegalArgumentException( 549 Msg.code(1181) + "Invalid token type (expecting TokenParam): " + nextOr.getClass()); 550 } 551 } 552 return codeSystemHashList; 553 } 554 555 private List<String> getCodingCodeOnlyValues(List<? extends IQueryParameterType> codeParams) { 556 ArrayList<String> codeOnlyList = new ArrayList<>(); 557 for (IQueryParameterType nextOr : codeParams) { 558 559 if (nextOr instanceof TokenParam) { 560 TokenParam ref = (TokenParam) nextOr; 561 if (ref.getValue() != null && ref.getSystem() == null && !ref.isText()) { 562 codeOnlyList.add(ref.getValue()); 563 } 564 } else { 565 throw new IllegalArgumentException( 566 Msg.code(1182) + "Invalid token type (expecting TokenParam): " + nextOr.getClass()); 567 } 568 } 569 return codeOnlyList; 570 } 571 572 private List<String> getCodingSystemOnlyValues(List<? extends IQueryParameterType> codeParams) { 573 ArrayList<String> systemOnlyList = new ArrayList<>(); 574 for (IQueryParameterType nextOr : codeParams) { 575 576 if (nextOr instanceof TokenParam) { 577 TokenParam ref = (TokenParam) nextOr; 578 if (ref.getValue() == null && ref.getSystem() != null) { 579 systemOnlyList.add(ref.getSystem()); 580 } 581 } else { 582 throw new IllegalArgumentException( 583 Msg.code(1183) + "Invalid token type (expecting TokenParam): " + nextOr.getClass()); 584 } 585 } 586 return systemOnlyList; 587 } 588 589 private List<String> getCodingTextOnlyValues(List<? extends IQueryParameterType> codeParams) { 590 ArrayList<String> textOnlyList = new ArrayList<>(); 591 for (IQueryParameterType nextOr : codeParams) { 592 593 if (nextOr instanceof TokenParam) { 594 TokenParam ref = (TokenParam) nextOr; 595 if (ref.isText() && ref.getValue() != null) { 596 textOnlyList.add(ref.getValue()); 597 } 598 } else { 599 throw new IllegalArgumentException( 600 Msg.code(1184) + "Invalid token type (expecting TokenParam): " + nextOr.getClass()); 601 } 602 } 603 return textOnlyList; 604 } 605 606 private void addObservationCodeCriteria( 607 BoolQueryBuilder theBoolQueryBuilder, 608 SearchParameterMap theSearchParameterMap, 609 FhirContext theFhirContext) { 610 String codeParamName = LastNParameterHelper.getCodeParamName(theFhirContext); 611 if (theSearchParameterMap.containsKey(codeParamName)) { 612 ArrayList<String> codeSystemHashList = new ArrayList<>(); 613 ArrayList<String> codeOnlyList = new ArrayList<>(); 614 ArrayList<String> systemOnlyList = new ArrayList<>(); 615 ArrayList<String> textOnlyList = new ArrayList<>(); 616 List<List<IQueryParameterType>> andOrParams = theSearchParameterMap.get(codeParamName); 617 for (List<? extends IQueryParameterType> nextAnd : andOrParams) { 618 codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd)); 619 codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd)); 620 systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd)); 621 textOnlyList.addAll(getCodingTextOnlyValues(nextAnd)); 622 } 623 if (codeSystemHashList.size() > 0) { 624 theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODEHASH_FIELD_NAME, codeSystemHashList)); 625 } 626 if (codeOnlyList.size() > 0) { 627 theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODEVALUE_FIELD_NAME, codeOnlyList)); 628 } 629 if (systemOnlyList.size() > 0) { 630 theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODESYSTEM_FIELD_NAME, systemOnlyList)); 631 } 632 if (textOnlyList.size() > 0) { 633 BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery(); 634 for (String textOnlyParam : textOnlyList) { 635 myTextBoolQueryBuilder.should( 636 QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CODEDISPLAY_FIELD_NAME, textOnlyParam)); 637 myTextBoolQueryBuilder.should( 638 QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CODE_TEXT_FIELD_NAME, textOnlyParam)); 639 } 640 theBoolQueryBuilder.must(myTextBoolQueryBuilder); 641 } 642 } 643 } 644 645 private void addDateCriteria( 646 BoolQueryBuilder theBoolQueryBuilder, 647 SearchParameterMap theSearchParameterMap, 648 FhirContext theFhirContext) { 649 String dateParamName = LastNParameterHelper.getEffectiveParamName(theFhirContext); 650 if (theSearchParameterMap.containsKey(dateParamName)) { 651 List<List<IQueryParameterType>> andOrParams = theSearchParameterMap.get(dateParamName); 652 for (List<? extends IQueryParameterType> nextAnd : andOrParams) { 653 BoolQueryBuilder myDateBoolQueryBuilder = new BoolQueryBuilder(); 654 for (IQueryParameterType nextOr : nextAnd) { 655 if (nextOr instanceof DateParam) { 656 DateParam myDate = (DateParam) nextOr; 657 createDateCriteria(myDate, myDateBoolQueryBuilder); 658 } 659 } 660 theBoolQueryBuilder.must(myDateBoolQueryBuilder); 661 } 662 } 663 } 664 665 private void createDateCriteria(DateParam theDate, BoolQueryBuilder theBoolQueryBuilder) { 666 Long dateInstant = theDate.getValue().getTime(); 667 RangeQueryBuilder myRangeQueryBuilder = new RangeQueryBuilder(OBSERVATION_EFFECTIVEDTM_FIELD_NAME); 668 669 ParamPrefixEnum prefix = theDate.getPrefix(); 670 if (prefix == ParamPrefixEnum.GREATERTHAN || prefix == ParamPrefixEnum.STARTS_AFTER) { 671 theBoolQueryBuilder.should(myRangeQueryBuilder.gt(dateInstant)); 672 } else if (prefix == ParamPrefixEnum.LESSTHAN || prefix == ParamPrefixEnum.ENDS_BEFORE) { 673 theBoolQueryBuilder.should(myRangeQueryBuilder.lt(dateInstant)); 674 } else if (prefix == ParamPrefixEnum.LESSTHAN_OR_EQUALS) { 675 theBoolQueryBuilder.should(myRangeQueryBuilder.lte(dateInstant)); 676 } else if (prefix == ParamPrefixEnum.GREATERTHAN_OR_EQUALS) { 677 theBoolQueryBuilder.should(myRangeQueryBuilder.gte(dateInstant)); 678 } else { 679 theBoolQueryBuilder.should(new MatchQueryBuilder(OBSERVATION_EFFECTIVEDTM_FIELD_NAME, dateInstant)); 680 } 681 } 682 683 @VisibleForTesting 684 public List<ObservationJson> executeLastNWithAllFieldsForTest( 685 SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) { 686 return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, null, t -> t, 100); 687 } 688 689 @VisibleForTesting 690 List<CodeJson> queryAllIndexedObservationCodesForTest() throws IOException { 691 SearchRequest codeSearchRequest = new SearchRequest(OBSERVATION_CODE_INDEX); 692 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 693 // Query 694 searchSourceBuilder.query(QueryBuilders.matchAllQuery()); 695 searchSourceBuilder.size(1000); 696 codeSearchRequest.source(searchSourceBuilder); 697 SearchResponse codeSearchResponse = executeSearchRequest(codeSearchRequest); 698 return buildCodeResult(codeSearchResponse); 699 } 700 701 private List<CodeJson> buildCodeResult(SearchResponse theSearchResponse) throws JsonProcessingException { 702 SearchHits codeHits = theSearchResponse.getHits(); 703 List<CodeJson> codes = new ArrayList<>(); 704 for (SearchHit codeHit : codeHits) { 705 CodeJson code = objectMapper.readValue(codeHit.getSourceAsString(), CodeJson.class); 706 codes.add(code); 707 } 708 return codes; 709 } 710 711 @Override 712 public ObservationJson getObservationDocument(String theDocumentID) { 713 if (theDocumentID == null) { 714 throw new InvalidRequestException( 715 Msg.code(1185) + "Require non-null document ID for observation document query"); 716 } 717 SearchRequest theSearchRequest = buildSingleObservationSearchRequest(theDocumentID); 718 ObservationJson observationDocumentJson = null; 719 try { 720 SearchResponse observationDocumentResponse = executeSearchRequest(theSearchRequest); 721 SearchHit[] observationDocumentHits = 722 observationDocumentResponse.getHits().getHits(); 723 if (observationDocumentHits.length > 0) { 724 // There should be no more than one hit for the identifier 725 String observationDocument = observationDocumentHits[0].getSourceAsString(); 726 observationDocumentJson = objectMapper.readValue(observationDocument, ObservationJson.class); 727 } 728 729 } catch (IOException theE) { 730 throw new InvalidRequestException( 731 Msg.code(1186) + "Unable to execute observation document query for ID " + theDocumentID, theE); 732 } 733 734 return observationDocumentJson; 735 } 736 737 private SearchRequest buildSingleObservationSearchRequest(String theObservationIdentifier) { 738 SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX); 739 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 740 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); 741 boolQueryBuilder.must(QueryBuilders.termQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, theObservationIdentifier)); 742 searchSourceBuilder.query(boolQueryBuilder); 743 searchSourceBuilder.size(1); 744 745 searchRequest.source(searchSourceBuilder); 746 747 return searchRequest; 748 } 749 750 @Override 751 public CodeJson getObservationCodeDocument(String theCodeSystemHash, String theText) { 752 if (theCodeSystemHash == null && theText == null) { 753 throw new InvalidRequestException(Msg.code(1187) 754 + "Require a non-null code system hash value or display value for observation code document query"); 755 } 756 SearchRequest theSearchRequest = buildSingleObservationCodeSearchRequest(theCodeSystemHash, theText); 757 CodeJson observationCodeDocumentJson = null; 758 try { 759 SearchResponse observationCodeDocumentResponse = executeSearchRequest(theSearchRequest); 760 SearchHit[] observationCodeDocumentHits = 761 observationCodeDocumentResponse.getHits().getHits(); 762 if (observationCodeDocumentHits.length > 0) { 763 // There should be no more than one hit for the code lookup. 764 String observationCodeDocument = observationCodeDocumentHits[0].getSourceAsString(); 765 observationCodeDocumentJson = objectMapper.readValue(observationCodeDocument, CodeJson.class); 766 } 767 768 } catch (IOException theE) { 769 throw new InvalidRequestException( 770 Msg.code(1188) + "Unable to execute observation code document query hash code or display", theE); 771 } 772 773 return observationCodeDocumentJson; 774 } 775 776 private SearchRequest buildSingleObservationCodeSearchRequest(String theCodeSystemHash, String theText) { 777 SearchRequest searchRequest = new SearchRequest(OBSERVATION_CODE_INDEX); 778 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 779 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); 780 if (theCodeSystemHash != null) { 781 boolQueryBuilder.must(QueryBuilders.termQuery(CODE_HASH, theCodeSystemHash)); 782 } else { 783 boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(CODE_TEXT, theText)); 784 } 785 786 searchSourceBuilder.query(boolQueryBuilder); 787 searchSourceBuilder.size(1); 788 789 searchRequest.source(searchSourceBuilder); 790 791 return searchRequest; 792 } 793 794 @Override 795 public Boolean createOrUpdateObservationIndex(String theDocumentId, ObservationJson theObservationDocument) { 796 try { 797 String documentToIndex = objectMapper.writeValueAsString(theObservationDocument); 798 return performIndex( 799 OBSERVATION_INDEX, theDocumentId, documentToIndex, ElasticsearchSvcImpl.OBSERVATION_DOCUMENT_TYPE); 800 } catch (IOException theE) { 801 throw new InvalidRequestException( 802 Msg.code(1189) + "Unable to persist Observation document " + theDocumentId); 803 } 804 } 805 806 @Override 807 public Boolean createOrUpdateObservationCodeIndex( 808 String theCodeableConceptID, CodeJson theObservationCodeDocument) { 809 try { 810 String documentToIndex = objectMapper.writeValueAsString(theObservationCodeDocument); 811 return performIndex( 812 OBSERVATION_CODE_INDEX, 813 theCodeableConceptID, 814 documentToIndex, 815 ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE); 816 } catch (IOException theE) { 817 throw new InvalidRequestException( 818 Msg.code(1190) + "Unable to persist Observation Code document " + theCodeableConceptID); 819 } 820 } 821 822 private boolean performIndex( 823 String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) 824 throws IOException { 825 IndexResponse indexResponse = myRestHighLevelClient.index( 826 createIndexRequest(theIndexName, theDocumentId, theIndexDocument, theDocumentType), 827 RequestOptions.DEFAULT); 828 829 return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) 830 || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED); 831 } 832 833 @Override 834 public void close() throws IOException { 835 myRestHighLevelClient.close(); 836 } 837 838 @Override 839 public List<IBaseResource> getObservationResources(Collection<? extends IResourcePersistentId> thePids) { 840 SearchRequest searchRequest = buildObservationResourceSearchRequest(thePids); 841 try { 842 SearchResponse observationDocumentResponse = executeSearchRequest(searchRequest); 843 SearchHit[] observationDocumentHits = 844 observationDocumentResponse.getHits().getHits(); 845 IParser parser = TolerantJsonParser.createWithLenientErrorHandling(myContext, null); 846 Class<? extends IBaseResource> resourceType = 847 myContext.getResourceDefinition(OBSERVATION_RESOURCE_NAME).getImplementingClass(); 848 /** 849 * @see ca.uhn.fhir.jpa.dao.BaseHapiFhirDao#toResource(Class, IBaseResourceEntity, Collection, boolean) for 850 * details about parsing raw json to BaseResource 851 */ 852 return Arrays.stream(observationDocumentHits) 853 .map(this::parseObservationJson) 854 .map(observationJson -> parser.parseResource(resourceType, observationJson.getResource())) 855 .collect(Collectors.toList()); 856 } catch (IOException theE) { 857 throw new InvalidRequestException( 858 Msg.code(2003) + "Unable to execute observation document query for provided IDs " + thePids, theE); 859 } 860 } 861 862 private ObservationJson parseObservationJson(SearchHit theSearchHit) { 863 try { 864 return objectMapper.readValue(theSearchHit.getSourceAsString(), ObservationJson.class); 865 } catch (JsonProcessingException exp) { 866 throw new InvalidRequestException(Msg.code(2004) + "Unable to parse the observation resource json", exp); 867 } 868 } 869 870 private SearchRequest buildObservationResourceSearchRequest(Collection<? extends IResourcePersistentId> thePids) { 871 SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX); 872 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 873 // Query 874 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); 875 List<String> pidParams = thePids.stream().map(Object::toString).collect(Collectors.toList()); 876 boolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, pidParams)); 877 searchSourceBuilder.query(boolQueryBuilder); 878 searchSourceBuilder.size(thePids.size()); 879 searchRequest.source(searchSourceBuilder); 880 return searchRequest; 881 } 882 883 private IndexRequest createIndexRequest( 884 String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) { 885 IndexRequest request = new IndexRequest(theIndexName); 886 request.id(theDocumentId); 887 request.source(theObservationDocument, XContentType.JSON); 888 return request; 889 } 890 891 @Override 892 public void deleteObservationDocument(String theDocumentId) { 893 DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(OBSERVATION_INDEX); 894 deleteByQueryRequest.setQuery(QueryBuilders.termQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, theDocumentId)); 895 try { 896 myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); 897 } catch (IOException theE) { 898 throw new InvalidRequestException(Msg.code(1191) + "Unable to delete Observation " + theDocumentId); 899 } 900 } 901 902 @VisibleForTesting 903 public void deleteAllDocumentsForTest(String theIndexName) throws IOException { 904 DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(theIndexName); 905 deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); 906 myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); 907 } 908 909 @VisibleForTesting 910 public void refreshIndex(String theIndexName) throws IOException { 911 myRestHighLevelClient.indices().refresh(new RefreshRequest(theIndexName), RequestOptions.DEFAULT); 912 } 913}