![](/hapi-fhir/images/logos/raccoon-forwards.png)
001/* 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.dao; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.i18n.Msg; 024import ca.uhn.fhir.interceptor.api.HookParams; 025import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; 026import ca.uhn.fhir.interceptor.api.Pointcut; 027import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 028import ca.uhn.fhir.jpa.api.svc.IIdHelperService; 029import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchClauseBuilder; 030import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchIndexExtractor; 031import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchResourceProjection; 032import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder; 033import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper; 034import ca.uhn.fhir.jpa.dao.search.LastNOperation; 035import ca.uhn.fhir.jpa.model.dao.JpaPid; 036import ca.uhn.fhir.jpa.model.entity.ResourceTable; 037import ca.uhn.fhir.jpa.model.search.ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams; 038import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData; 039import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; 040import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions; 041import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteSearch; 042import ca.uhn.fhir.jpa.search.builder.ISearchQueryExecutor; 043import ca.uhn.fhir.jpa.search.builder.SearchQueryExecutors; 044import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; 045import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; 046import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; 047import ca.uhn.fhir.model.api.IQueryParameterType; 048import ca.uhn.fhir.parser.IParser; 049import ca.uhn.fhir.rest.api.Constants; 050import ca.uhn.fhir.rest.api.server.RequestDetails; 051import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; 052import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; 053import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; 054import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; 055import ca.uhn.fhir.rest.server.util.ResourceSearchParams; 056import com.google.common.collect.Ordering; 057import jakarta.annotation.Nonnull; 058import jakarta.persistence.EntityManager; 059import jakarta.persistence.PersistenceContext; 060import jakarta.persistence.PersistenceContextType; 061import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension; 062import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep; 063import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory; 064import org.hibernate.search.engine.search.projection.dsl.CompositeProjectionOptionsStep; 065import org.hibernate.search.engine.search.projection.dsl.SearchProjectionFactory; 066import org.hibernate.search.engine.search.query.dsl.SearchQueryOptionsStep; 067import org.hibernate.search.mapper.orm.Search; 068import org.hibernate.search.mapper.orm.common.EntityReference; 069import org.hibernate.search.mapper.orm.search.loading.dsl.SearchLoadingOptionsStep; 070import org.hibernate.search.mapper.orm.session.SearchSession; 071import org.hibernate.search.mapper.orm.work.SearchIndexingPlan; 072import org.hibernate.search.util.common.SearchException; 073import org.hl7.fhir.instance.model.api.IBaseResource; 074import org.springframework.beans.factory.annotation.Autowired; 075import org.springframework.transaction.PlatformTransactionManager; 076import org.springframework.transaction.annotation.Transactional; 077import org.springframework.transaction.support.TransactionTemplate; 078 079import java.util.ArrayList; 080import java.util.Collection; 081import java.util.Collections; 082import java.util.List; 083import java.util.Spliterators; 084import java.util.stream.Collectors; 085import java.util.stream.StreamSupport; 086 087import static ca.uhn.fhir.rest.server.BasePagingProvider.DEFAULT_MAX_PAGE_SIZE; 088import static org.apache.commons.lang3.StringUtils.isNotBlank; 089 090public class FulltextSearchSvcImpl implements IFulltextSearchSvc { 091 private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FulltextSearchSvcImpl.class); 092 private static final int DEFAULT_MAX_NON_PAGED_SIZE = 500; 093 private final ExtendedHSearchSearchBuilder myAdvancedIndexQueryBuilder = new ExtendedHSearchSearchBuilder(); 094 095 @Autowired 096 ISearchParamExtractor mySearchParamExtractor; 097 098 @Autowired 099 IIdHelperService myIdHelperService; 100 101 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 102 private EntityManager myEntityManager; 103 104 @Autowired 105 private PlatformTransactionManager myTxManager; 106 107 @Autowired 108 private FhirContext myFhirContext; 109 110 @Autowired 111 private ISearchParamRegistry mySearchParamRegistry; 112 113 @Autowired 114 private JpaStorageSettings myStorageSettings; 115 116 @Autowired 117 private IHSearchSortHelper myExtendedFulltextSortHelper; 118 119 @Autowired(required = false) 120 private IHSearchEventListener myHSearchEventListener; 121 122 @Autowired 123 private IInterceptorBroadcaster myInterceptorBroadcaster; 124 125 private Boolean ourDisabled; 126 127 /** 128 * Constructor 129 */ 130 public FulltextSearchSvcImpl() { 131 super(); 132 } 133 134 @Override 135 public ExtendedHSearchIndexData extractLuceneIndexData( 136 IBaseResource theResource, ResourceIndexedSearchParams theNewParams) { 137 String resourceType = myFhirContext.getResourceType(theResource); 138 ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams(resourceType); 139 ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor( 140 myStorageSettings, myFhirContext, activeSearchParams, mySearchParamExtractor); 141 return extractor.extract(theResource, theNewParams); 142 } 143 144 @Override 145 public boolean canUseHibernateSearch(String theResourceType, SearchParameterMap myParams) { 146 boolean requiresHibernateSearchAccess = myParams.containsKey(Constants.PARAM_CONTENT) 147 || myParams.containsKey(Constants.PARAM_TEXT) 148 || myParams.isLastN(); 149 // we have to use it - _text and _content searches only use hibernate 150 if (requiresHibernateSearchAccess) { 151 return true; 152 } 153 154 // if the registry has not been initialized 155 // we cannot use HibernateSearch because it 156 // will, internally, trigger a new search 157 // when it refreshes the search parameters 158 // (which will cause an infinite loop) 159 if (!mySearchParamRegistry.isInitialized()) { 160 return false; 161 } 162 163 return myStorageSettings.isAdvancedHSearchIndexing() 164 && myAdvancedIndexQueryBuilder.canUseHibernateSearch(theResourceType, myParams, mySearchParamRegistry); 165 } 166 167 @Override 168 public void reindex(ResourceTable theEntity) { 169 validateHibernateSearchIsEnabled(); 170 171 SearchIndexingPlan plan = getSearchSession().indexingPlan(); 172 plan.addOrUpdate(theEntity); 173 } 174 175 @Override 176 public ISearchQueryExecutor searchNotScrolled( 177 String theResourceName, 178 SearchParameterMap theParams, 179 Integer theMaxResultsToFetch, 180 RequestDetails theRequestDetails) { 181 validateHibernateSearchIsEnabled(); 182 183 return doSearch(theResourceName, theParams, null, theMaxResultsToFetch, theRequestDetails); 184 } 185 186 // keep this in sync with supportsSomeOf(); 187 @SuppressWarnings("rawtypes") 188 private ISearchQueryExecutor doSearch( 189 String theResourceType, 190 SearchParameterMap theParams, 191 IResourcePersistentId theReferencingPid, 192 Integer theMaxResultsToFetch, 193 RequestDetails theRequestDetails) { 194 195 int offset = theParams.getOffset() == null ? 0 : theParams.getOffset(); 196 int count = getMaxFetchSize(theParams, theMaxResultsToFetch); 197 198 // perform an offset search instead of a scroll one, which doesn't allow for offset 199 SearchQueryOptionsStep<?, Long, SearchLoadingOptionsStep, ?, ?> searchQueryOptionsStep = 200 getSearchQueryOptionsStep(theResourceType, theParams, theReferencingPid); 201 logQuery(searchQueryOptionsStep, theRequestDetails); 202 List<Long> longs = searchQueryOptionsStep.fetchHits(offset, count); 203 204 // indicate param was already processed, otherwise queries DB to process it 205 theParams.setOffset(null); 206 return SearchQueryExecutors.from(longs); 207 } 208 209 private int getMaxFetchSize(SearchParameterMap theParams, Integer theMax) { 210 if (theMax != null) { 211 return theMax; 212 } 213 214 // todo mb we should really pass this in. 215 if (theParams.getCount() != null) { 216 return theParams.getCount(); 217 } 218 219 return DEFAULT_MAX_NON_PAGED_SIZE; 220 } 221 222 @SuppressWarnings("rawtypes") 223 private SearchQueryOptionsStep<?, Long, SearchLoadingOptionsStep, ?, ?> getSearchQueryOptionsStep( 224 String theResourceType, SearchParameterMap theParams, IResourcePersistentId theReferencingPid) { 225 226 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 227 var query = getSearchSession() 228 .search(ResourceTable.class) 229 // The document id is the PK which is pid. We use this instead of _myId to avoid fetching the doc body. 230 .select( 231 // adapt the String docRef.id() to the Long that it really is. 232 f -> f.composite(docRef -> Long.valueOf(docRef.id()), f.documentReference())) 233 .where(f -> buildWhereClause(f, theResourceType, theParams, theReferencingPid)); 234 235 if (theParams.getSort() != null) { 236 query.sort(f -> myExtendedFulltextSortHelper.getSortClauses(f, theParams.getSort(), theResourceType)); 237 238 // indicate parameter was processed 239 theParams.setSort(null); 240 } 241 242 return query; 243 } 244 245 @SuppressWarnings("rawtypes") 246 private PredicateFinalStep buildWhereClause( 247 SearchPredicateFactory f, 248 String theResourceType, 249 SearchParameterMap theParams, 250 IResourcePersistentId theReferencingPid) { 251 return f.bool(b -> { 252 ExtendedHSearchClauseBuilder builder = 253 new ExtendedHSearchClauseBuilder(myFhirContext, myStorageSettings, b, f); 254 255 /* 256 * Handle _content parameter (resource body content) 257 * 258 * Posterity: 259 * We do not want the HAPI-FHIR dao's to process the 260 * _content parameter, so we remove it from the map here 261 */ 262 List<List<IQueryParameterType>> contentAndTerms = theParams.remove(Constants.PARAM_CONTENT); 263 builder.addStringTextSearch(Constants.PARAM_CONTENT, contentAndTerms); 264 265 /* 266 * Handle _text parameter (resource narrative content) 267 * 268 * Posterity: 269 * We do not want the HAPI-FHIR dao's to process the 270 * _text parameter, so we remove it from the map here 271 */ 272 List<List<IQueryParameterType>> textAndTerms = theParams.remove(Constants.PARAM_TEXT); 273 builder.addStringTextSearch(Constants.PARAM_TEXT, textAndTerms); 274 275 if (theReferencingPid != null) { 276 b.must(f.match().field("myResourceLinksField").matching(theReferencingPid.toString())); 277 } 278 279 if (isNotBlank(theResourceType)) { 280 builder.addResourceTypeClause(theResourceType); 281 } 282 283 /* 284 * Handle other supported parameters 285 */ 286 if (myStorageSettings.isAdvancedHSearchIndexing() && theParams.getEverythingMode() == null) { 287 ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams params = 288 new ExtendedHSearchBuilderConsumeAdvancedQueryClausesParams(); 289 params.setSearchParamRegistry(mySearchParamRegistry) 290 .setResourceType(theResourceType) 291 .setSearchParameterMap(theParams); 292 myAdvancedIndexQueryBuilder.addAndConsumeAdvancedQueryClauses(builder, params); 293 } 294 // DROP EARLY HERE IF BOOL IS EMPTY? 295 }); 296 } 297 298 @Nonnull 299 private SearchSession getSearchSession() { 300 return Search.session(myEntityManager); 301 } 302 303 @SuppressWarnings("rawtypes") 304 private List<IResourcePersistentId> convertLongsToResourcePersistentIds(List<Long> theLongPids) { 305 return theLongPids.stream().map(JpaPid::fromId).collect(Collectors.toList()); 306 } 307 308 @Override 309 @SuppressWarnings({"rawtypes", "unchecked"}) 310 public List<IResourcePersistentId> everything( 311 String theResourceName, 312 SearchParameterMap theParams, 313 IResourcePersistentId theReferencingPid, 314 RequestDetails theRequestDetails) { 315 validateHibernateSearchIsEnabled(); 316 317 // todo mb what about max results here? 318 List<IResourcePersistentId> retVal = 319 toList(doSearch(null, theParams, theReferencingPid, 10_000, theRequestDetails), 10_000); 320 if (theReferencingPid != null) { 321 retVal.add(theReferencingPid); 322 } 323 return retVal; 324 } 325 326 private void validateHibernateSearchIsEnabled() { 327 if (isDisabled()) { 328 throw new UnsupportedOperationException(Msg.code(2137) + "Hibernate search is not enabled!"); 329 } 330 } 331 332 @Override 333 public boolean isDisabled() { 334 Boolean retVal = ourDisabled; 335 336 if (retVal == null) { 337 retVal = new TransactionTemplate(myTxManager).execute(t -> { 338 try { 339 SearchSession searchSession = getSearchSession(); 340 searchSession.search(ResourceTable.class); 341 return Boolean.FALSE; 342 } catch (Exception e) { 343 ourLog.trace("FullText test failed", e); 344 ourLog.debug( 345 "Hibernate Search (Lucene) appears to be disabled on this server, fulltext will be disabled"); 346 return Boolean.TRUE; 347 } 348 }); 349 ourDisabled = retVal; 350 } 351 352 assert retVal != null; 353 return retVal; 354 } 355 356 @Transactional() 357 @Override 358 @SuppressWarnings("unchecked") 359 public List<IResourcePersistentId> search( 360 String theResourceName, SearchParameterMap theParams, RequestDetails theRequestDetails) { 361 validateHibernateSearchIsEnabled(); 362 return toList( 363 doSearch(theResourceName, theParams, null, DEFAULT_MAX_NON_PAGED_SIZE, theRequestDetails), 364 DEFAULT_MAX_NON_PAGED_SIZE); 365 } 366 367 /** 368 * Adapt our async interface to the legacy concrete List 369 */ 370 @SuppressWarnings("rawtypes") 371 private List<IResourcePersistentId> toList(ISearchQueryExecutor theSearchResultStream, long theMaxSize) { 372 return StreamSupport.stream(Spliterators.spliteratorUnknownSize(theSearchResultStream, 0), false) 373 .map(JpaPid::fromId) 374 .limit(theMaxSize) 375 .collect(Collectors.toList()); 376 } 377 378 @Transactional() 379 @Override 380 public IBaseResource tokenAutocompleteValueSetSearch(ValueSetAutocompleteOptions theOptions) { 381 validateHibernateSearchIsEnabled(); 382 ensureElastic(); 383 384 ValueSetAutocompleteSearch autocomplete = 385 new ValueSetAutocompleteSearch(myFhirContext, myStorageSettings, getSearchSession()); 386 387 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 388 return autocomplete.search(theOptions); 389 } 390 391 /** 392 * Throws an error if configured with Lucene. 393 * <p> 394 * Some features only work with Elasticsearch. 395 * Lastn and the autocomplete search use nested aggregations which are Elasticsearch-only 396 */ 397 private void ensureElastic() { 398 try { 399 getSearchSession().scope(ResourceTable.class).aggregation().extension(ElasticsearchExtension.get()); 400 } catch (SearchException e) { 401 // unsupported. we are probably running Lucene. 402 throw new IllegalStateException( 403 Msg.code(2070) + "This operation requires Elasticsearch. Lucene is not supported."); 404 } 405 } 406 407 @Override 408 @SuppressWarnings("rawtypes") 409 public List<IResourcePersistentId> lastN(SearchParameterMap theParams, Integer theMaximumResults) { 410 ensureElastic(); 411 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 412 List<Long> pidList = new LastNOperation( 413 getSearchSession(), myFhirContext, myStorageSettings, mySearchParamRegistry) 414 .executeLastN(theParams, theMaximumResults); 415 return convertLongsToResourcePersistentIds(pidList); 416 } 417 418 @Override 419 public List<IBaseResource> getResources(Collection<Long> thePids) { 420 if (thePids.isEmpty()) { 421 return Collections.emptyList(); 422 } 423 424 SearchSession session = getSearchSession(); 425 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 426 List<ExtendedHSearchResourceProjection> rawResourceDataList = session.search(ResourceTable.class) 427 .select(this::buildResourceSelectClause) 428 .where( 429 f -> f.id().matchingAny(thePids) // matches '_id' from resource index 430 ) 431 .fetchAllHits(); 432 433 // order resource projections as per thePids 434 ArrayList<Long> pidList = new ArrayList<>(thePids); 435 List<ExtendedHSearchResourceProjection> orderedAsPidsResourceDataList = rawResourceDataList.stream() 436 .sorted(Ordering.explicit(pidList).onResultOf(ExtendedHSearchResourceProjection::getPid)) 437 .collect(Collectors.toList()); 438 439 return resourceProjectionsToResources(orderedAsPidsResourceDataList); 440 } 441 442 @Nonnull 443 private List<IBaseResource> resourceProjectionsToResources( 444 List<ExtendedHSearchResourceProjection> theResourceDataList) { 445 IParser parser = myFhirContext.newJsonParser(); 446 return theResourceDataList.stream().map(p -> p.toResource(parser)).collect(Collectors.toList()); 447 } 448 449 private CompositeProjectionOptionsStep<?, ExtendedHSearchResourceProjection> buildResourceSelectClause( 450 SearchProjectionFactory<EntityReference, ResourceTable> f) { 451 return f.composite( 452 ExtendedHSearchResourceProjection::new, 453 f.field("myId", Long.class), 454 f.field("myForcedId", String.class), 455 f.field("myRawResource", String.class)); 456 } 457 458 @Override 459 public long count(String theResourceName, SearchParameterMap theParams) { 460 SearchQueryOptionsStep<?, Long, SearchLoadingOptionsStep, ?, ?> queryOptionsStep = 461 getSearchQueryOptionsStep(theResourceName, theParams, null); 462 463 return queryOptionsStep.fetchTotalHitCount(); 464 } 465 466 @Override 467 @Transactional(readOnly = true) 468 public List<IBaseResource> searchForResources( 469 String theResourceType, SearchParameterMap theParams, RequestDetails theRequestDetails) { 470 int offset = 0; 471 int limit = theParams.getCount() == null ? DEFAULT_MAX_PAGE_SIZE : theParams.getCount(); 472 473 if (theParams.getOffset() != null && theParams.getOffset() != 0) { 474 offset = theParams.getOffset(); 475 // indicate param was already processed, otherwise queries DB to process it 476 theParams.setOffset(null); 477 } 478 479 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 480 481 var query = getSearchSession() 482 .search(ResourceTable.class) 483 .select(this::buildResourceSelectClause) 484 .where(f -> buildWhereClause(f, theResourceType, theParams, null)); 485 486 if (theParams.getSort() != null) { 487 query.sort(f -> myExtendedFulltextSortHelper.getSortClauses(f, theParams.getSort(), theResourceType)); 488 } 489 490 logQuery(query, theRequestDetails); 491 List<ExtendedHSearchResourceProjection> extendedLuceneResourceProjections = query.fetchHits(offset, limit); 492 493 return resourceProjectionsToResources(extendedLuceneResourceProjections); 494 } 495 496 /** 497 * Fire the JPA_PERFTRACE_INFO hook if it is enabled 498 * @param theQuery the query to log 499 * @param theRequestDetails the request details 500 */ 501 @SuppressWarnings("rawtypes") 502 private void logQuery(SearchQueryOptionsStep theQuery, RequestDetails theRequestDetails) { 503 if (CompositeInterceptorBroadcaster.hasHooks( 504 Pointcut.JPA_PERFTRACE_INFO, myInterceptorBroadcaster, theRequestDetails)) { 505 StorageProcessingMessage storageProcessingMessage = new StorageProcessingMessage(); 506 String queryString = theQuery.toQuery().queryString(); 507 storageProcessingMessage.setMessage(queryString); 508 HookParams params = new HookParams() 509 .add(RequestDetails.class, theRequestDetails) 510 .addIfMatchesType(ServletRequestDetails.class, theRequestDetails) 511 .add(StorageProcessingMessage.class, storageProcessingMessage); 512 CompositeInterceptorBroadcaster.doCallHooks( 513 myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_INFO, params); 514 } 515 } 516 517 @Override 518 public boolean supportsAllOf(SearchParameterMap theParams) { 519 return myAdvancedIndexQueryBuilder.isSupportsAllOf(theParams); 520 } 521 522 @Override 523 public boolean supportsAllSortTerms(String theResourceType, SearchParameterMap theParams) { 524 return myExtendedFulltextSortHelper.supportsAllSortTerms(theResourceType, theParams); 525 } 526 527 private void dispatchEvent(IHSearchEventListener.HSearchEventType theEventType) { 528 if (myHSearchEventListener != null) { 529 myHSearchEventListener.hsearchEvent(theEventType); 530 } 531 } 532 533 @Override 534 public void deleteIndexedDocumentsByTypeAndId(Class theClazz, List<Object> theGivenIds) { 535 SearchSession session = Search.session(myEntityManager); 536 SearchIndexingPlan indexingPlan = session.indexingPlan(); 537 for (Object givenId : theGivenIds) { 538 indexingPlan.purge(theClazz, givenId, null); 539 } 540 indexingPlan.process(); 541 indexingPlan.execute(); 542 } 543}