001/* 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.dao; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.i18n.Msg; 024import ca.uhn.fhir.interceptor.api.HookParams; 025import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; 026import ca.uhn.fhir.interceptor.api.Pointcut; 027import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 028import ca.uhn.fhir.jpa.api.svc.IIdHelperService; 029import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchClauseBuilder; 030import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchIndexExtractor; 031import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchResourceProjection; 032import ca.uhn.fhir.jpa.dao.search.ExtendedHSearchSearchBuilder; 033import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper; 034import ca.uhn.fhir.jpa.dao.search.LastNOperation; 035import ca.uhn.fhir.jpa.model.dao.JpaPid; 036import ca.uhn.fhir.jpa.model.entity.ResourceTable; 037import ca.uhn.fhir.jpa.model.search.ExtendedHSearchIndexData; 038import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; 039import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteOptions; 040import ca.uhn.fhir.jpa.search.autocomplete.ValueSetAutocompleteSearch; 041import ca.uhn.fhir.jpa.search.builder.ISearchQueryExecutor; 042import ca.uhn.fhir.jpa.search.builder.SearchQueryExecutors; 043import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; 044import ca.uhn.fhir.jpa.searchparam.extractor.ISearchParamExtractor; 045import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; 046import ca.uhn.fhir.model.api.IQueryParameterType; 047import ca.uhn.fhir.parser.IParser; 048import ca.uhn.fhir.rest.api.Constants; 049import ca.uhn.fhir.rest.api.server.RequestDetails; 050import ca.uhn.fhir.rest.api.server.storage.IResourcePersistentId; 051import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; 052import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; 053import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; 054import ca.uhn.fhir.rest.server.util.ResourceSearchParams; 055import com.google.common.collect.Ordering; 056import jakarta.annotation.Nonnull; 057import jakarta.persistence.EntityManager; 058import jakarta.persistence.PersistenceContext; 059import jakarta.persistence.PersistenceContextType; 060import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension; 061import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep; 062import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory; 063import org.hibernate.search.engine.search.projection.dsl.CompositeProjectionOptionsStep; 064import org.hibernate.search.engine.search.projection.dsl.SearchProjectionFactory; 065import org.hibernate.search.engine.search.query.dsl.SearchQueryOptionsStep; 066import org.hibernate.search.mapper.orm.Search; 067import org.hibernate.search.mapper.orm.common.EntityReference; 068import org.hibernate.search.mapper.orm.search.loading.dsl.SearchLoadingOptionsStep; 069import org.hibernate.search.mapper.orm.session.SearchSession; 070import org.hibernate.search.mapper.orm.work.SearchIndexingPlan; 071import org.hibernate.search.util.common.SearchException; 072import org.hl7.fhir.instance.model.api.IBaseResource; 073import org.springframework.beans.factory.annotation.Autowired; 074import org.springframework.transaction.PlatformTransactionManager; 075import org.springframework.transaction.annotation.Transactional; 076import org.springframework.transaction.support.TransactionTemplate; 077 078import java.util.ArrayList; 079import java.util.Collection; 080import java.util.Collections; 081import java.util.List; 082import java.util.Spliterators; 083import java.util.stream.Collectors; 084import java.util.stream.StreamSupport; 085 086import static ca.uhn.fhir.rest.server.BasePagingProvider.DEFAULT_MAX_PAGE_SIZE; 087import static org.apache.commons.lang3.StringUtils.isNotBlank; 088 089public class FulltextSearchSvcImpl implements IFulltextSearchSvc { 090 private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(FulltextSearchSvcImpl.class); 091 private static final int DEFAULT_MAX_NON_PAGED_SIZE = 500; 092 private final ExtendedHSearchSearchBuilder myAdvancedIndexQueryBuilder = new ExtendedHSearchSearchBuilder(); 093 094 @Autowired 095 ISearchParamExtractor mySearchParamExtractor; 096 097 @Autowired 098 IIdHelperService myIdHelperService; 099 100 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 101 private EntityManager myEntityManager; 102 103 @Autowired 104 private PlatformTransactionManager myTxManager; 105 106 @Autowired 107 private FhirContext myFhirContext; 108 109 @Autowired 110 private ISearchParamRegistry mySearchParamRegistry; 111 112 @Autowired 113 private JpaStorageSettings myStorageSettings; 114 115 @Autowired 116 private IHSearchSortHelper myExtendedFulltextSortHelper; 117 118 @Autowired(required = false) 119 private IHSearchEventListener myHSearchEventListener; 120 121 @Autowired 122 private IInterceptorBroadcaster myInterceptorBroadcaster; 123 124 private Boolean ourDisabled; 125 126 /** 127 * Constructor 128 */ 129 public FulltextSearchSvcImpl() { 130 super(); 131 } 132 133 @Override 134 public ExtendedHSearchIndexData extractLuceneIndexData( 135 IBaseResource theResource, ResourceIndexedSearchParams theNewParams) { 136 String resourceType = myFhirContext.getResourceType(theResource); 137 ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams(resourceType); 138 ExtendedHSearchIndexExtractor extractor = new ExtendedHSearchIndexExtractor( 139 myStorageSettings, myFhirContext, activeSearchParams, mySearchParamExtractor); 140 return extractor.extract(theResource, theNewParams); 141 } 142 143 @Override 144 public boolean supportsSomeOf(SearchParameterMap myParams) { 145 146 // keep this in sync with the guts of doSearch 147 boolean requiresHibernateSearchAccess = myParams.containsKey(Constants.PARAM_CONTENT) 148 || myParams.containsKey(Constants.PARAM_TEXT) 149 || myParams.isLastN(); 150 151 requiresHibernateSearchAccess |= 152 myStorageSettings.isAdvancedHSearchIndexing() && myAdvancedIndexQueryBuilder.isSupportsSomeOf(myParams); 153 154 return requiresHibernateSearchAccess; 155 } 156 157 @Override 158 public void reindex(ResourceTable theEntity) { 159 validateHibernateSearchIsEnabled(); 160 161 SearchIndexingPlan plan = getSearchSession().indexingPlan(); 162 plan.addOrUpdate(theEntity); 163 } 164 165 @Override 166 public ISearchQueryExecutor searchNotScrolled( 167 String theResourceName, 168 SearchParameterMap theParams, 169 Integer theMaxResultsToFetch, 170 RequestDetails theRequestDetails) { 171 validateHibernateSearchIsEnabled(); 172 173 return doSearch(theResourceName, theParams, null, theMaxResultsToFetch, theRequestDetails); 174 } 175 176 // keep this in sync with supportsSomeOf(); 177 private ISearchQueryExecutor doSearch( 178 String theResourceType, 179 SearchParameterMap theParams, 180 IResourcePersistentId theReferencingPid, 181 Integer theMaxResultsToFetch, 182 RequestDetails theRequestDetails) { 183 184 int offset = theParams.getOffset() == null ? 0 : theParams.getOffset(); 185 int count = getMaxFetchSize(theParams, theMaxResultsToFetch); 186 187 // perform an offset search instead of a scroll one, which doesn't allow for offset 188 SearchQueryOptionsStep<?, Long, SearchLoadingOptionsStep, ?, ?> searchQueryOptionsStep = 189 getSearchQueryOptionsStep(theResourceType, theParams, theReferencingPid); 190 logQuery(searchQueryOptionsStep, theRequestDetails); 191 List<Long> longs = searchQueryOptionsStep.fetchHits(offset, count); 192 193 // indicate param was already processed, otherwise queries DB to process it 194 theParams.setOffset(null); 195 return SearchQueryExecutors.from(longs); 196 } 197 198 private int getMaxFetchSize(SearchParameterMap theParams, Integer theMax) { 199 if (theMax != null) { 200 return theMax; 201 } 202 203 // todo mb we should really pass this in. 204 if (theParams.getCount() != null) { 205 return theParams.getCount(); 206 } 207 208 return DEFAULT_MAX_NON_PAGED_SIZE; 209 } 210 211 private SearchQueryOptionsStep<?, Long, SearchLoadingOptionsStep, ?, ?> getSearchQueryOptionsStep( 212 String theResourceType, SearchParameterMap theParams, IResourcePersistentId theReferencingPid) { 213 214 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 215 var query = getSearchSession() 216 .search(ResourceTable.class) 217 // The document id is the PK which is pid. We use this instead of _myId to avoid fetching the doc body. 218 .select( 219 // adapt the String docRef.id() to the Long that it really is. 220 f -> f.composite(docRef -> Long.valueOf(docRef.id()), f.documentReference())) 221 .where(f -> buildWhereClause(f, theResourceType, theParams, theReferencingPid)); 222 223 if (theParams.getSort() != null) { 224 query.sort(f -> myExtendedFulltextSortHelper.getSortClauses(f, theParams.getSort(), theResourceType)); 225 226 // indicate parameter was processed 227 theParams.setSort(null); 228 } 229 230 return query; 231 } 232 233 private PredicateFinalStep buildWhereClause( 234 SearchPredicateFactory f, 235 String theResourceType, 236 SearchParameterMap theParams, 237 IResourcePersistentId theReferencingPid) { 238 return f.bool(b -> { 239 ExtendedHSearchClauseBuilder builder = 240 new ExtendedHSearchClauseBuilder(myFhirContext, myStorageSettings, b, f); 241 242 /* 243 * Handle _content parameter (resource body content) 244 * 245 * Posterity: 246 * We do not want the HAPI-FHIR dao's to process the 247 * _content parameter, so we remove it from the map here 248 */ 249 List<List<IQueryParameterType>> contentAndTerms = theParams.remove(Constants.PARAM_CONTENT); 250 builder.addStringTextSearch(Constants.PARAM_CONTENT, contentAndTerms); 251 252 /* 253 * Handle _text parameter (resource narrative content) 254 * 255 * Posterity: 256 * We do not want the HAPI-FHIR dao's to process the 257 * _text parameter, so we remove it from the map here 258 */ 259 List<List<IQueryParameterType>> textAndTerms = theParams.remove(Constants.PARAM_TEXT); 260 builder.addStringTextSearch(Constants.PARAM_TEXT, textAndTerms); 261 262 if (theReferencingPid != null) { 263 b.must(f.match().field("myResourceLinksField").matching(theReferencingPid.toString())); 264 } 265 266 if (isNotBlank(theResourceType)) { 267 builder.addResourceTypeClause(theResourceType); 268 } 269 270 /* 271 * Handle other supported parameters 272 */ 273 if (myStorageSettings.isAdvancedHSearchIndexing() && theParams.getEverythingMode() == null) { 274 myAdvancedIndexQueryBuilder.addAndConsumeAdvancedQueryClauses( 275 builder, theResourceType, theParams, mySearchParamRegistry); 276 } 277 // DROP EARLY HERE IF BOOL IS EMPTY? 278 }); 279 } 280 281 @Nonnull 282 private SearchSession getSearchSession() { 283 return Search.session(myEntityManager); 284 } 285 286 private List<IResourcePersistentId> convertLongsToResourcePersistentIds(List<Long> theLongPids) { 287 return theLongPids.stream().map(JpaPid::fromId).collect(Collectors.toList()); 288 } 289 290 @Override 291 public List<IResourcePersistentId> everything( 292 String theResourceName, 293 SearchParameterMap theParams, 294 IResourcePersistentId theReferencingPid, 295 RequestDetails theRequestDetails) { 296 validateHibernateSearchIsEnabled(); 297 298 // todo mb what about max results here? 299 List<IResourcePersistentId> retVal = 300 toList(doSearch(null, theParams, theReferencingPid, 10_000, theRequestDetails), 10_000); 301 if (theReferencingPid != null) { 302 retVal.add(theReferencingPid); 303 } 304 return retVal; 305 } 306 307 private void validateHibernateSearchIsEnabled() { 308 if (isDisabled()) { 309 throw new UnsupportedOperationException(Msg.code(2137) + "Hibernate search is not enabled!"); 310 } 311 } 312 313 @Override 314 public boolean isDisabled() { 315 Boolean retVal = ourDisabled; 316 317 if (retVal == null) { 318 retVal = new TransactionTemplate(myTxManager).execute(t -> { 319 try { 320 SearchSession searchSession = getSearchSession(); 321 searchSession.search(ResourceTable.class); 322 return Boolean.FALSE; 323 } catch (Exception e) { 324 ourLog.trace("FullText test failed", e); 325 ourLog.debug( 326 "Hibernate Search (Lucene) appears to be disabled on this server, fulltext will be disabled"); 327 return Boolean.TRUE; 328 } 329 }); 330 ourDisabled = retVal; 331 } 332 333 assert retVal != null; 334 return retVal; 335 } 336 337 @Transactional() 338 @Override 339 public List<IResourcePersistentId> search( 340 String theResourceName, SearchParameterMap theParams, RequestDetails theRequestDetails) { 341 validateHibernateSearchIsEnabled(); 342 return toList( 343 doSearch(theResourceName, theParams, null, DEFAULT_MAX_NON_PAGED_SIZE, theRequestDetails), 344 DEFAULT_MAX_NON_PAGED_SIZE); 345 } 346 347 /** 348 * Adapt our async interface to the legacy concrete List 349 */ 350 private List<IResourcePersistentId> toList(ISearchQueryExecutor theSearchResultStream, long theMaxSize) { 351 return StreamSupport.stream(Spliterators.spliteratorUnknownSize(theSearchResultStream, 0), false) 352 .map(JpaPid::fromId) 353 .limit(theMaxSize) 354 .collect(Collectors.toList()); 355 } 356 357 @Transactional() 358 @Override 359 public IBaseResource tokenAutocompleteValueSetSearch(ValueSetAutocompleteOptions theOptions) { 360 validateHibernateSearchIsEnabled(); 361 ensureElastic(); 362 363 ValueSetAutocompleteSearch autocomplete = 364 new ValueSetAutocompleteSearch(myFhirContext, myStorageSettings, getSearchSession()); 365 366 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 367 return autocomplete.search(theOptions); 368 } 369 370 /** 371 * Throws an error if configured with Lucene. 372 * <p> 373 * Some features only work with Elasticsearch. 374 * Lastn and the autocomplete search use nested aggregations which are Elasticsearch-only 375 */ 376 private void ensureElastic() { 377 try { 378 getSearchSession().scope(ResourceTable.class).aggregation().extension(ElasticsearchExtension.get()); 379 } catch (SearchException e) { 380 // unsupported. we are probably running Lucene. 381 throw new IllegalStateException( 382 Msg.code(2070) + "This operation requires Elasticsearch. Lucene is not supported."); 383 } 384 } 385 386 @Override 387 public List<IResourcePersistentId> lastN(SearchParameterMap theParams, Integer theMaximumResults) { 388 ensureElastic(); 389 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 390 List<Long> pidList = new LastNOperation( 391 getSearchSession(), myFhirContext, myStorageSettings, mySearchParamRegistry) 392 .executeLastN(theParams, theMaximumResults); 393 return convertLongsToResourcePersistentIds(pidList); 394 } 395 396 @Override 397 public List<IBaseResource> getResources(Collection<Long> thePids) { 398 if (thePids.isEmpty()) { 399 return Collections.emptyList(); 400 } 401 402 SearchSession session = getSearchSession(); 403 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 404 List<ExtendedHSearchResourceProjection> rawResourceDataList = session.search(ResourceTable.class) 405 .select(this::buildResourceSelectClause) 406 .where( 407 f -> f.id().matchingAny(thePids) // matches '_id' from resource index 408 ) 409 .fetchAllHits(); 410 411 // order resource projections as per thePids 412 ArrayList<Long> pidList = new ArrayList<>(thePids); 413 List<ExtendedHSearchResourceProjection> orderedAsPidsResourceDataList = rawResourceDataList.stream() 414 .sorted(Ordering.explicit(pidList).onResultOf(ExtendedHSearchResourceProjection::getPid)) 415 .collect(Collectors.toList()); 416 417 return resourceProjectionsToResources(orderedAsPidsResourceDataList); 418 } 419 420 @Nonnull 421 private List<IBaseResource> resourceProjectionsToResources( 422 List<ExtendedHSearchResourceProjection> theResourceDataList) { 423 IParser parser = myFhirContext.newJsonParser(); 424 return theResourceDataList.stream().map(p -> p.toResource(parser)).collect(Collectors.toList()); 425 } 426 427 private CompositeProjectionOptionsStep<?, ExtendedHSearchResourceProjection> buildResourceSelectClause( 428 SearchProjectionFactory<EntityReference, ResourceTable> f) { 429 return f.composite( 430 ExtendedHSearchResourceProjection::new, 431 f.field("myId", Long.class), 432 f.field("myForcedId", String.class), 433 f.field("myRawResource", String.class)); 434 } 435 436 @Override 437 public long count(String theResourceName, SearchParameterMap theParams) { 438 SearchQueryOptionsStep<?, Long, SearchLoadingOptionsStep, ?, ?> queryOptionsStep = 439 getSearchQueryOptionsStep(theResourceName, theParams, null); 440 441 return queryOptionsStep.fetchTotalHitCount(); 442 } 443 444 @Override 445 @Transactional(readOnly = true) 446 public List<IBaseResource> searchForResources( 447 String theResourceType, SearchParameterMap theParams, RequestDetails theRequestDetails) { 448 int offset = 0; 449 int limit = theParams.getCount() == null ? DEFAULT_MAX_PAGE_SIZE : theParams.getCount(); 450 451 if (theParams.getOffset() != null && theParams.getOffset() != 0) { 452 offset = theParams.getOffset(); 453 // indicate param was already processed, otherwise queries DB to process it 454 theParams.setOffset(null); 455 } 456 457 dispatchEvent(IHSearchEventListener.HSearchEventType.SEARCH); 458 459 var query = getSearchSession() 460 .search(ResourceTable.class) 461 .select(this::buildResourceSelectClause) 462 .where(f -> buildWhereClause(f, theResourceType, theParams, null)); 463 464 if (theParams.getSort() != null) { 465 query.sort(f -> myExtendedFulltextSortHelper.getSortClauses(f, theParams.getSort(), theResourceType)); 466 } 467 468 logQuery(query, theRequestDetails); 469 List<ExtendedHSearchResourceProjection> extendedLuceneResourceProjections = query.fetchHits(offset, limit); 470 471 return resourceProjectionsToResources(extendedLuceneResourceProjections); 472 } 473 474 /** 475 * Fire the JPA_PERFTRACE_INFO hook if it is enabled 476 * @param theQuery the query to log 477 * @param theRequestDetails the request details 478 */ 479 @SuppressWarnings("rawtypes") 480 private void logQuery(SearchQueryOptionsStep theQuery, RequestDetails theRequestDetails) { 481 if (CompositeInterceptorBroadcaster.hasHooks( 482 Pointcut.JPA_PERFTRACE_INFO, myInterceptorBroadcaster, theRequestDetails)) { 483 StorageProcessingMessage storageProcessingMessage = new StorageProcessingMessage(); 484 String queryString = theQuery.toQuery().queryString(); 485 storageProcessingMessage.setMessage(queryString); 486 HookParams params = new HookParams() 487 .add(RequestDetails.class, theRequestDetails) 488 .addIfMatchesType(ServletRequestDetails.class, theRequestDetails) 489 .add(StorageProcessingMessage.class, storageProcessingMessage); 490 CompositeInterceptorBroadcaster.doCallHooks( 491 myInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_INFO, params); 492 } 493 } 494 495 @Override 496 public boolean supportsAllOf(SearchParameterMap theParams) { 497 return myAdvancedIndexQueryBuilder.isSupportsAllOf(theParams); 498 } 499 500 private void dispatchEvent(IHSearchEventListener.HSearchEventType theEventType) { 501 if (myHSearchEventListener != null) { 502 myHSearchEventListener.hsearchEvent(theEventType); 503 } 504 } 505 506 @Override 507 public void deleteIndexedDocumentsByTypeAndId(Class theClazz, List<Object> theGivenIds) { 508 SearchSession session = Search.session(myEntityManager); 509 SearchIndexingPlan indexingPlan = session.indexingPlan(); 510 for (Object givenId : theGivenIds) { 511 indexingPlan.purge(theClazz, givenId, null); 512 } 513 indexingPlan.process(); 514 indexingPlan.execute(); 515 } 516}