001/*- 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2025 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L%family 019 */ 020package ca.uhn.fhir.jpa.search; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.i18n.Msg; 024import ca.uhn.fhir.interceptor.api.HookParams; 025import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; 026import ca.uhn.fhir.interceptor.api.Pointcut; 027import ca.uhn.fhir.interceptor.model.RequestPartitionId; 028import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 029import ca.uhn.fhir.jpa.api.dao.DaoRegistry; 030import ca.uhn.fhir.jpa.api.dao.IDao; 031import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; 032import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc; 033import ca.uhn.fhir.jpa.config.SearchConfig; 034import ca.uhn.fhir.jpa.dao.BaseStorageDao; 035import ca.uhn.fhir.jpa.dao.ISearchBuilder; 036import ca.uhn.fhir.jpa.dao.NonPersistedSearch; 037import ca.uhn.fhir.jpa.dao.SearchBuilderFactory; 038import ca.uhn.fhir.jpa.dao.search.ResourceNotFoundInIndexException; 039import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; 040import ca.uhn.fhir.jpa.entity.Search; 041import ca.uhn.fhir.jpa.model.dao.JpaPid; 042import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; 043import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc; 044import ca.uhn.fhir.jpa.search.builder.StorageInterceptorHooksFacade; 045import ca.uhn.fhir.jpa.search.builder.tasks.SearchContinuationTask; 046import ca.uhn.fhir.jpa.search.builder.tasks.SearchTask; 047import ca.uhn.fhir.jpa.search.builder.tasks.SearchTaskParameters; 048import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; 049import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; 050import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum; 051import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; 052import ca.uhn.fhir.jpa.util.QueryParameterUtils; 053import ca.uhn.fhir.model.api.IQueryParameterType; 054import ca.uhn.fhir.model.api.Include; 055import ca.uhn.fhir.rest.api.CacheControlDirective; 056import ca.uhn.fhir.rest.api.Constants; 057import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; 058import ca.uhn.fhir.rest.api.SearchTotalModeEnum; 059import ca.uhn.fhir.rest.api.server.IBundleProvider; 060import ca.uhn.fhir.rest.api.server.RequestDetails; 061import ca.uhn.fhir.rest.api.server.SystemRequestDetails; 062import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; 063import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 064import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; 065import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; 066import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; 067import ca.uhn.fhir.util.AsyncUtil; 068import ca.uhn.fhir.util.StopWatch; 069import ca.uhn.fhir.util.UrlUtil; 070import com.google.common.annotations.VisibleForTesting; 071import jakarta.annotation.Nonnull; 072import jakarta.annotation.Nullable; 073import org.apache.commons.lang3.time.DateUtils; 074import org.hl7.fhir.instance.model.api.IBaseResource; 075import org.springframework.beans.factory.BeanFactory; 076import org.springframework.data.domain.PageRequest; 077import org.springframework.data.domain.Pageable; 078import org.springframework.data.domain.Sort; 079import org.springframework.stereotype.Component; 080import org.springframework.transaction.support.TransactionSynchronizationManager; 081 082import java.io.Serial; 083import java.time.Instant; 084import java.time.temporal.ChronoUnit; 085import java.util.Date; 086import java.util.HashSet; 087import java.util.List; 088import java.util.Optional; 089import java.util.Set; 090import java.util.UUID; 091import java.util.concurrent.Callable; 092import java.util.concurrent.ConcurrentHashMap; 093import java.util.concurrent.TimeUnit; 094import java.util.function.Consumer; 095import java.util.stream.Collectors; 096 097import static ca.uhn.fhir.jpa.util.QueryParameterUtils.DEFAULT_SYNC_SIZE; 098import static org.apache.commons.lang3.StringUtils.isBlank; 099import static org.apache.commons.lang3.StringUtils.isNotBlank; 100 101@Component("mySearchCoordinatorSvc") 102public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc<JpaPid> { 103 104 private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchCoordinatorSvcImpl.class); 105 public static final int SEARCH_EXPIRY_OFFSET_MINUTES = 10; 106 107 private final FhirContext myContext; 108 private final JpaStorageSettings myStorageSettings; 109 private final IInterceptorBroadcaster myInterceptorBroadcaster; 110 private final HapiTransactionService myTxService; 111 private final ISearchCacheSvc mySearchCacheSvc; 112 private final ISearchResultCacheSvc mySearchResultCacheSvc; 113 private final DaoRegistry myDaoRegistry; 114 private final SearchBuilderFactory<JpaPid> mySearchBuilderFactory; 115 private final ISynchronousSearchSvc mySynchronousSearchSvc; 116 private final PersistedJpaBundleProviderFactory myPersistedJpaBundleProviderFactory; 117 private final ISearchParamRegistry mySearchParamRegistry; 118 private final SearchStrategyFactory mySearchStrategyFactory; 119 private final ExceptionService myExceptionSvc; 120 private final BeanFactory myBeanFactory; 121 private final IRequestPartitionHelperSvc myRequestPartitionHelperSvc; 122 private ConcurrentHashMap<String, SearchTask> myIdToSearchTask = new ConcurrentHashMap<>(); 123 124 private final Consumer<String> myOnRemoveSearchTask = myIdToSearchTask::remove; 125 126 private final StorageInterceptorHooksFacade myStorageInterceptorHooks; 127 private Integer myLoadingThrottleForUnitTests = null; 128 private long myMaxMillisToWaitForRemoteResults = DateUtils.MILLIS_PER_MINUTE; 129 private boolean myNeverUseLocalSearchForUnitTests; 130 private int mySyncSize = DEFAULT_SYNC_SIZE; 131 132 /** 133 * Constructor 134 */ 135 public SearchCoordinatorSvcImpl( 136 FhirContext theContext, 137 JpaStorageSettings theStorageSettings, 138 IInterceptorBroadcaster theInterceptorBroadcaster, 139 HapiTransactionService theTxService, 140 ISearchCacheSvc theSearchCacheSvc, 141 ISearchResultCacheSvc theSearchResultCacheSvc, 142 DaoRegistry theDaoRegistry, 143 SearchBuilderFactory<JpaPid> theSearchBuilderFactory, 144 ISynchronousSearchSvc theSynchronousSearchSvc, 145 PersistedJpaBundleProviderFactory thePersistedJpaBundleProviderFactory, 146 ISearchParamRegistry theSearchParamRegistry, 147 SearchStrategyFactory theSearchStrategyFactory, 148 ExceptionService theExceptionSvc, 149 BeanFactory theBeanFactory, 150 IRequestPartitionHelperSvc theRequestPartitionHelperSvc) { 151 super(); 152 myContext = theContext; 153 myStorageSettings = theStorageSettings; 154 myInterceptorBroadcaster = theInterceptorBroadcaster; 155 myTxService = theTxService; 156 mySearchCacheSvc = theSearchCacheSvc; 157 mySearchResultCacheSvc = theSearchResultCacheSvc; 158 myDaoRegistry = theDaoRegistry; 159 mySearchBuilderFactory = theSearchBuilderFactory; 160 mySynchronousSearchSvc = theSynchronousSearchSvc; 161 myPersistedJpaBundleProviderFactory = thePersistedJpaBundleProviderFactory; 162 mySearchParamRegistry = theSearchParamRegistry; 163 mySearchStrategyFactory = theSearchStrategyFactory; 164 myExceptionSvc = theExceptionSvc; 165 myBeanFactory = theBeanFactory; 166 myRequestPartitionHelperSvc = theRequestPartitionHelperSvc; 167 168 myStorageInterceptorHooks = new StorageInterceptorHooksFacade(myInterceptorBroadcaster); 169 } 170 171 @VisibleForTesting 172 Set<String> getActiveSearchIds() { 173 return myIdToSearchTask.keySet(); 174 } 175 176 @VisibleForTesting 177 public void setIdToSearchTaskMapForUnitTests(ConcurrentHashMap<String, SearchTask> theIdToSearchTaskMap) { 178 myIdToSearchTask = theIdToSearchTaskMap; 179 } 180 181 @VisibleForTesting 182 public void setLoadingThrottleForUnitTests(Integer theLoadingThrottleForUnitTests) { 183 myLoadingThrottleForUnitTests = theLoadingThrottleForUnitTests; 184 } 185 186 @VisibleForTesting 187 public void setNeverUseLocalSearchForUnitTests(boolean theNeverUseLocalSearchForUnitTests) { 188 myNeverUseLocalSearchForUnitTests = theNeverUseLocalSearchForUnitTests; 189 } 190 191 @VisibleForTesting 192 public void setSyncSizeForUnitTests(int theSyncSize) { 193 mySyncSize = theSyncSize; 194 } 195 196 @Override 197 public void cancelAllActiveSearches() { 198 for (SearchTask next : myIdToSearchTask.values()) { 199 ourLog.info( 200 "Requesting immediate abort of search: {}", next.getSearch().getUuid()); 201 next.requestImmediateAbort(); 202 AsyncUtil.awaitLatchAndIgnoreInterrupt(next.getCompletionLatch(), 30, TimeUnit.SECONDS); 203 } 204 } 205 206 @SuppressWarnings("SameParameterValue") 207 @VisibleForTesting 208 public void setMaxMillisToWaitForRemoteResultsForUnitTest(long theMaxMillisToWaitForRemoteResults) { 209 myMaxMillisToWaitForRemoteResults = theMaxMillisToWaitForRemoteResults; 210 } 211 212 /** 213 * This method is called by the HTTP client processing thread in order to 214 * fetch resources. 215 * <p> 216 * This method must not be called from inside a transaction. The rationale is that 217 * the {@link Search} entity is treated as a piece of shared state across client threads 218 * accessing the same thread, so we need to be able to update that table in a transaction 219 * and commit it right away in order for that to work. Examples of needing to do this 220 * include if two different clients request the same search and are both paging at the 221 * same time, but also includes clients that are hacking the paging links to 222 * fetch multiple pages of a search result in parallel. In both cases we need to only 223 * let one of them actually activate the search, or we will have conflicts. The other thread 224 * just needs to wait until the first one actually fetches more results. 225 */ 226 @Override 227 public List<JpaPid> getResources( 228 final String theUuid, 229 int theFrom, 230 int theTo, 231 @Nullable RequestDetails theRequestDetails, 232 RequestPartitionId theRequestPartitionId) { 233 assert !TransactionSynchronizationManager.isActualTransactionActive(); 234 235 // If we're actively searching right now, don't try to do anything until at least one batch has been 236 // persisted in the DB 237 SearchTask searchTask = myIdToSearchTask.get(theUuid); 238 if (searchTask != null) { 239 searchTask.awaitInitialSync(); 240 } 241 242 ourLog.trace("About to start looking for resources {}-{}", theFrom, theTo); 243 244 Search search; 245 StopWatch sw = new StopWatch(); 246 while (true) { 247 248 if (!myNeverUseLocalSearchForUnitTests) { 249 if (searchTask != null) { 250 ourLog.trace("Local search found"); 251 List<JpaPid> resourcePids = searchTask.getResourcePids(theFrom, theTo); 252 ourLog.trace( 253 "Local search returned {} pids, wanted {}-{} - Search: {}", 254 resourcePids.size(), 255 theFrom, 256 theTo, 257 searchTask.getSearch()); 258 259 /* 260 * Generally, if a search task is open, the fastest possible thing is to just return its results. This 261 * will work most of the time, but can fail if the task hit a search threshold and the client is requesting 262 * results beyond that threashold. In that case, we'll keep going below, since that will trigger another 263 * task. 264 */ 265 if ((searchTask.getSearch().getNumFound() 266 - searchTask.getSearch().getNumBlocked()) 267 >= theTo 268 || resourcePids.size() == (theTo - theFrom)) { 269 return resourcePids; 270 } 271 } 272 } 273 274 Callable<Search> searchCallback = () -> mySearchCacheSvc 275 .fetchByUuid(theUuid, theRequestPartitionId) 276 .orElseThrow(() -> myExceptionSvc.newUnknownSearchException(theUuid)); 277 search = myTxService 278 .withRequest(theRequestDetails) 279 .withRequestPartitionId(theRequestPartitionId) 280 .execute(searchCallback); 281 QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(search); 282 283 if (search.getStatus() == SearchStatusEnum.FINISHED) { 284 ourLog.trace("Search entity marked as finished with {} results", search.getNumFound()); 285 break; 286 } 287 if ((search.getNumFound() - search.getNumBlocked()) >= theTo) { 288 ourLog.trace("Search entity has {} results so far", search.getNumFound()); 289 break; 290 } 291 292 if (sw.getMillis() > myMaxMillisToWaitForRemoteResults) { 293 ourLog.error( 294 "Search {} of type {} for {}{} timed out after {}ms with status {}.", 295 search.getId(), 296 search.getSearchType(), 297 search.getResourceType(), 298 search.getSearchQueryString(), 299 sw.getMillis(), 300 search.getStatus()); 301 throw new InternalErrorException(Msg.code(1163) + "Request timed out after " + sw.getMillis() + "ms"); 302 } 303 304 // If the search was saved in "pass complete mode" it's probably time to 305 // start a new pass 306 if (search.getStatus() == SearchStatusEnum.PASSCMPLET) { 307 ourLog.trace("Going to try to start next search"); 308 Optional<Search> newSearch = 309 mySearchCacheSvc.tryToMarkSearchAsInProgress(search, theRequestPartitionId); 310 if (newSearch.isPresent()) { 311 ourLog.trace("Launching new search"); 312 search = newSearch.get(); 313 String resourceType = search.getResourceType(); 314 SearchParameterMap params = search.getSearchParameterMap() 315 .orElseThrow(() -> new IllegalStateException("No map in PASSCOMPLET search")); 316 IFhirResourceDao<?> resourceDao = myDaoRegistry.getResourceDao(resourceType); 317 318 SearchTaskParameters parameters = new SearchTaskParameters( 319 search, 320 resourceDao, 321 params, 322 resourceType, 323 theRequestDetails, 324 theRequestPartitionId, 325 myOnRemoveSearchTask, 326 mySyncSize); 327 parameters.setLoadingThrottleForUnitTests(myLoadingThrottleForUnitTests); 328 SearchContinuationTask task = 329 (SearchContinuationTask) myBeanFactory.getBean(SearchConfig.CONTINUE_TASK, parameters); 330 myIdToSearchTask.put(search.getUuid(), task); 331 task.call(); 332 } 333 } 334 335 if (!search.getStatus().isDone()) { 336 AsyncUtil.sleep(500); 337 } 338 } 339 340 ourLog.trace("Finished looping"); 341 342 List<JpaPid> pids = fetchResultPids(theUuid, theFrom, theTo, theRequestDetails, search, theRequestPartitionId); 343 344 updateSearchExpiryOrNull(search, theRequestPartitionId); 345 346 ourLog.trace("Fetched {} results", pids.size()); 347 348 return pids; 349 } 350 351 @Nonnull 352 private List<JpaPid> fetchResultPids( 353 String theUuid, 354 int theFrom, 355 int theTo, 356 @Nullable RequestDetails theRequestDetails, 357 Search theSearch, 358 RequestPartitionId theRequestPartitionId) { 359 List<JpaPid> pids = mySearchResultCacheSvc.fetchResultPids( 360 theSearch, theFrom, theTo, theRequestDetails, theRequestPartitionId); 361 if (pids == null) { 362 throw myExceptionSvc.newUnknownSearchException(theUuid); 363 } 364 return pids; 365 } 366 367 private void updateSearchExpiryOrNull(Search theSearch, RequestPartitionId theRequestPartitionId) { 368 // The created time may be null in some unit tests 369 if (theSearch.getCreated() != null) { 370 // start tracking last-access-time for this search when it is more than halfway to expire by created time 371 // we do this to avoid generating excessive write traffic on busy cached searches. 372 long expireAfterMillis = myStorageSettings.getExpireSearchResultsAfterMillis(); 373 long createdCutoff = theSearch.getCreated().getTime() + expireAfterMillis; 374 if (createdCutoff - System.currentTimeMillis() < expireAfterMillis / 2) { 375 theSearch.setExpiryOrNull(DateUtils.addMinutes(new Date(), SEARCH_EXPIRY_OFFSET_MINUTES)); 376 // TODO: A nice future enhancement might be to make this flush be asynchronous 377 mySearchCacheSvc.save(theSearch, theRequestPartitionId); 378 } 379 } 380 } 381 382 @Override 383 public IBundleProvider registerSearch( 384 final IFhirResourceDao<?> theCallingDao, 385 final SearchParameterMap theParams, 386 String theResourceType, 387 CacheControlDirective theCacheControlDirective, 388 @Nullable RequestDetails theRequestDetails) { 389 final String searchUuid = UUID.randomUUID().toString(); 390 391 final String queryString = theParams.toNormalizedQueryString(); 392 ourLog.debug("Registering new search {}", searchUuid); 393 394 RequestPartitionId requestPartitionId = null; 395 if (theRequestDetails instanceof SystemRequestDetails srd) { 396 requestPartitionId = srd.getRequestPartitionId(); 397 } 398 399 // If an explicit request partition wasn't provided, calculate the request 400 // partition after invoking STORAGE_PRESEARCH_REGISTERED just in case any interceptors 401 // made changes which could affect the calculated partition 402 if (requestPartitionId == null) { 403 404 // Invoke any STORAGE_PRESEARCH_PARTITION_SELECTED interceptor hooks 405 NonPersistedSearch search = new NonPersistedSearch(theResourceType); 406 search.setUuid(searchUuid); 407 myStorageInterceptorHooks.callStoragePresearchPartitionSelected(theRequestDetails, theParams, search); 408 409 requestPartitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequestForSearchType( 410 theRequestDetails, theResourceType, theParams); 411 } 412 413 Search search = new Search(); 414 QueryParameterUtils.populateSearchEntity( 415 theParams, theResourceType, searchUuid, queryString, search, requestPartitionId); 416 417 // Invoke any STORAGE_PRESEARCH_REGISTERED interceptor hooks 418 myStorageInterceptorHooks.callStoragePresearchRegistered( 419 theRequestDetails, theParams, search, requestPartitionId); 420 421 validateSearch(theParams); 422 423 Class<? extends IBaseResource> resourceTypeClass = 424 myContext.getResourceDefinition(theResourceType).getImplementingClass(); 425 final ISearchBuilder<JpaPid> sb = mySearchBuilderFactory.newSearchBuilder(theResourceType, resourceTypeClass); 426 sb.setFetchSize(mySyncSize); 427 sb.setRequireTotal(theParams.getCount() != null); 428 429 final Integer loadSynchronousUpTo = getLoadSynchronousUpToOrNull(theCacheControlDirective); 430 boolean isOffsetQuery = theParams.isOffsetQuery(); 431 432 // todo someday - not today. 433 // SearchStrategyFactory.ISearchStrategy searchStrategy = mySearchStrategyFactory.pickStrategy(theResourceType, 434 // theParams, theRequestDetails); 435 // return searchStrategy.get(); 436 437 if (theParams.isLoadSynchronous() || loadSynchronousUpTo != null || isOffsetQuery) { 438 if (mySearchStrategyFactory.isSupportsHSearchDirect(theResourceType, theParams, theRequestDetails)) { 439 ourLog.info("Search {} is using direct load strategy", searchUuid); 440 SearchStrategyFactory.ISearchStrategy direct = mySearchStrategyFactory.makeDirectStrategy( 441 searchUuid, theResourceType, theParams, theRequestDetails); 442 443 try { 444 return direct.get(); 445 } catch (ResourceNotFoundInIndexException theE) { 446 // some resources were not found in index, so we will inform this and resort to JPA search 447 ourLog.warn( 448 "Some resources were not found in index. Make sure all resources were indexed. Resorting to database search."); 449 } 450 } 451 452 // we need a max to fetch for synchronous searches; 453 // otherwise we'll explode memory. 454 Integer maxToLoad = getSynchronousMaxResultsToFetch(theParams, loadSynchronousUpTo); 455 ourLog.debug("Setting a max fetch value of {} for synchronous search", maxToLoad); 456 sb.setMaxResultsToFetch(maxToLoad); 457 458 ourLog.debug("Search {} is loading in synchronous mode", searchUuid); 459 return mySynchronousSearchSvc.executeQuery( 460 theParams, theRequestDetails, searchUuid, sb, loadSynchronousUpTo, requestPartitionId); 461 } 462 463 /* 464 * See if there are any cached searches whose results we can return 465 * instead 466 */ 467 SearchCacheStatusEnum cacheStatus = SearchCacheStatusEnum.MISS; 468 if (theCacheControlDirective != null && theCacheControlDirective.isNoCache()) { 469 cacheStatus = SearchCacheStatusEnum.NOT_TRIED; 470 } 471 472 if (cacheStatus != SearchCacheStatusEnum.NOT_TRIED) { 473 if (theParams.getEverythingMode() == null) { 474 if (myStorageSettings.getReuseCachedSearchResultsForMillis() != null) { 475 PersistedJpaBundleProvider foundSearchProvider = findCachedQuery( 476 theParams, theResourceType, theRequestDetails, queryString, requestPartitionId); 477 if (foundSearchProvider != null) { 478 foundSearchProvider.setCacheStatus(SearchCacheStatusEnum.HIT); 479 return foundSearchProvider; 480 } 481 } 482 } 483 } 484 485 PersistedJpaSearchFirstPageBundleProvider retVal = submitSearch( 486 theCallingDao, theParams, theResourceType, theRequestDetails, sb, requestPartitionId, search); 487 retVal.setCacheStatus(cacheStatus); 488 return retVal; 489 } 490 491 /** 492 * The max results to return if this is a synchronous search. 493 * <p> 494 * We'll look in this order: 495 * * load synchronous up to (on params) 496 * * param count (+ offset) 497 * * StorageSettings fetch size default max 498 * * 499 */ 500 private Integer getSynchronousMaxResultsToFetch(SearchParameterMap theParams, Integer theLoadSynchronousUpTo) { 501 if (theLoadSynchronousUpTo != null) { 502 return theLoadSynchronousUpTo; 503 } 504 505 if (theParams.getCount() != null) { 506 int valToReturn = theParams.getCount() + 1; 507 if (theParams.getOffset() != null) { 508 valToReturn += theParams.getOffset(); 509 } 510 return valToReturn; 511 } 512 513 if (myStorageSettings.getFetchSizeDefaultMaximum() != null) { 514 return myStorageSettings.getFetchSizeDefaultMaximum(); 515 } 516 517 return myStorageSettings.getInternalSynchronousSearchSize(); 518 } 519 520 private void validateSearch(SearchParameterMap theParams) { 521 /* 522 * Having duplicate identical params in the search (e.g. Patient?gender=male&gender=male) is not 523 * technically wrong, but it's inefficient and can slow query execution down. Checking for it also 524 * adds CPU load itself though, so we only check this in an assert to hopefully catch errors in tests. 525 */ 526 assert checkNoDuplicateParameters(theParams) 527 : "Duplicate parameters found in query: " + theParams.toNormalizedQueryString(); 528 529 validateIncludes(theParams.getIncludes(), Constants.PARAM_INCLUDE); 530 validateIncludes(theParams.getRevIncludes(), Constants.PARAM_REVINCLUDE); 531 } 532 533 /** 534 * This method detects whether we have any duplicate lists of parameters and returns 535 * {@literal true} if none are found. For example, the following query would result 536 * in this method returning {@literal false}: 537 * <code>Patient?name=bart,homer&name=bart,homer</code> 538 * <p> 539 * This is not an optimized test, and it's not technically even prohibited to have 540 * duplicates like these in queries so this method should only be called as a 541 * part of an {@literal assert} statement to catch errors in tests. 542 */ 543 private boolean checkNoDuplicateParameters(SearchParameterMap theParams) { 544 HashSet<List<IQueryParameterType>> lists = new HashSet<>(); 545 for (List<List<IQueryParameterType>> andList : theParams.values()) { 546 547 lists.clear(); 548 for (int i = 0; i < andList.size(); i++) { 549 List<IQueryParameterType> orListI = andList.get(i); 550 if (!orListI.isEmpty() && !lists.add(orListI)) { 551 return false; 552 } 553 } 554 } 555 return true; 556 } 557 558 private void validateIncludes(Set<Include> includes, String name) { 559 for (Include next : includes) { 560 String value = next.getValue(); 561 if (value.equals(Constants.INCLUDE_STAR) || isBlank(value)) { 562 continue; 563 } 564 565 String paramType = next.getParamType(); 566 String paramName = next.getParamName(); 567 String paramTargetType = next.getParamTargetType(); 568 569 if (isBlank(paramType) || isBlank(paramName)) { 570 String msg = myContext 571 .getLocalizer() 572 .getMessageSanitized(SearchCoordinatorSvcImpl.class, "invalidInclude", name, value, ""); 573 throw new InvalidRequestException(Msg.code(2018) + msg); 574 } 575 576 if (!myDaoRegistry.isResourceTypeSupported(paramType)) { 577 String resourceTypeMsg = myContext 578 .getLocalizer() 579 .getMessageSanitized(SearchCoordinatorSvcImpl.class, "invalidResourceType", paramType); 580 String msg = myContext 581 .getLocalizer() 582 .getMessage( 583 SearchCoordinatorSvcImpl.class, 584 "invalidInclude", 585 UrlUtil.sanitizeUrlPart(name), 586 UrlUtil.sanitizeUrlPart(value), 587 resourceTypeMsg); // last param is pre-sanitized 588 throw new InvalidRequestException(Msg.code(2017) + msg); 589 } 590 591 if (isNotBlank(paramTargetType) && !myDaoRegistry.isResourceTypeSupported(paramTargetType)) { 592 String resourceTypeMsg = myContext 593 .getLocalizer() 594 .getMessageSanitized(SearchCoordinatorSvcImpl.class, "invalidResourceType", paramTargetType); 595 String msg = myContext 596 .getLocalizer() 597 .getMessage( 598 SearchCoordinatorSvcImpl.class, 599 "invalidInclude", 600 UrlUtil.sanitizeUrlPart(name), 601 UrlUtil.sanitizeUrlPart(value), 602 resourceTypeMsg); // last param is pre-sanitized 603 throw new InvalidRequestException(Msg.code(2016) + msg); 604 } 605 606 if (!Constants.INCLUDE_STAR.equals(paramName) 607 && mySearchParamRegistry.getActiveSearchParam( 608 paramType, paramName, ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH) 609 == null) { 610 List<String> validNames = mySearchParamRegistry 611 .getActiveSearchParams(paramType, ISearchParamRegistry.SearchParamLookupContextEnum.SEARCH) 612 .values() 613 .stream() 614 .filter(t -> t.getParamType() == RestSearchParameterTypeEnum.REFERENCE) 615 .map(t -> UrlUtil.sanitizeUrlPart(t.getName())) 616 .sorted() 617 .collect(Collectors.toList()); 618 String searchParamMessage = myContext 619 .getLocalizer() 620 .getMessage( 621 BaseStorageDao.class, 622 "invalidSearchParameter", 623 UrlUtil.sanitizeUrlPart(paramName), 624 UrlUtil.sanitizeUrlPart(paramType), 625 validNames); 626 String msg = myContext 627 .getLocalizer() 628 .getMessage( 629 SearchCoordinatorSvcImpl.class, 630 "invalidInclude", 631 UrlUtil.sanitizeUrlPart(name), 632 UrlUtil.sanitizeUrlPart(value), 633 searchParamMessage); // last param is pre-sanitized 634 throw new InvalidRequestException(Msg.code(2015) + msg); 635 } 636 } 637 } 638 639 @Override 640 public Optional<Integer> getSearchTotal( 641 String theUuid, @Nullable RequestDetails theRequestDetails, RequestPartitionId theRequestPartitionId) { 642 SearchTask task = myIdToSearchTask.get(theUuid); 643 if (task != null) { 644 return Optional.ofNullable(task.awaitInitialSync()); 645 } 646 647 /* 648 * In case there is no running search, if the total is listed as accurate we know one is coming 649 * so let's wait a bit for it to show up 650 */ 651 Optional<Search> search = myTxService 652 .withRequest(theRequestDetails) 653 .execute(() -> mySearchCacheSvc.fetchByUuid(theUuid, theRequestPartitionId)); 654 if (search.isPresent()) { 655 Optional<SearchParameterMap> searchParameterMap = search.get().getSearchParameterMap(); 656 if (searchParameterMap.isPresent() 657 && searchParameterMap.get().getSearchTotalMode() == SearchTotalModeEnum.ACCURATE) { 658 for (int i = 0; i < 10; i++) { 659 if (search.isPresent()) { 660 QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(search.get()); 661 if (search.get().getTotalCount() != null) { 662 return Optional.of(search.get().getTotalCount()); 663 } 664 } 665 search = mySearchCacheSvc.fetchByUuid(theUuid, theRequestPartitionId); 666 } 667 } 668 } 669 670 return Optional.empty(); 671 } 672 673 @Nonnull 674 private PersistedJpaSearchFirstPageBundleProvider submitSearch( 675 IDao theCallingDao, 676 SearchParameterMap theParams, 677 String theResourceType, 678 RequestDetails theRequestDetails, 679 ISearchBuilder<JpaPid> theSb, 680 RequestPartitionId theRequestPartitionId, 681 Search theSearch) { 682 StopWatch w = new StopWatch(); 683 684 SearchTaskParameters stp = new SearchTaskParameters( 685 theSearch, 686 theCallingDao, 687 theParams, 688 theResourceType, 689 theRequestDetails, 690 theRequestPartitionId, 691 myOnRemoveSearchTask, 692 mySyncSize); 693 stp.setLoadingThrottleForUnitTests(myLoadingThrottleForUnitTests); 694 SearchTask task = (SearchTask) myBeanFactory.getBean(SearchConfig.SEARCH_TASK, stp); 695 myIdToSearchTask.put(theSearch.getUuid(), task); 696 task.call(); 697 698 PersistedJpaSearchFirstPageBundleProvider retVal = myPersistedJpaBundleProviderFactory.newInstanceFirstPage( 699 theRequestDetails, task, theSb, theRequestPartitionId); 700 701 ourLog.debug("Search initial phase completed in {}ms", w.getMillis()); 702 return retVal; 703 } 704 705 @Nullable 706 private PersistedJpaBundleProvider findCachedQuery( 707 SearchParameterMap theParams, 708 String theResourceType, 709 RequestDetails theRequestDetails, 710 String theQueryString, 711 RequestPartitionId theRequestPartitionId) { 712 // May be null 713 return myTxService 714 .withRequest(theRequestDetails) 715 .withRequestPartitionId(theRequestPartitionId) 716 .execute(() -> { 717 IInterceptorBroadcaster compositeBroadcaster = 718 CompositeInterceptorBroadcaster.newCompositeBroadcaster( 719 myInterceptorBroadcaster, theRequestDetails); 720 721 // Interceptor call: STORAGE_PRECHECK_FOR_CACHED_SEARCH 722 723 HookParams params = new HookParams() 724 .add(SearchParameterMap.class, theParams) 725 .add(RequestDetails.class, theRequestDetails) 726 .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); 727 boolean canUseCache = 728 compositeBroadcaster.callHooks(Pointcut.STORAGE_PRECHECK_FOR_CACHED_SEARCH, params); 729 if (!canUseCache) { 730 return null; 731 } 732 733 // Check for a search matching the given hash 734 Search searchToUse = findSearchToUseOrNull(theQueryString, theResourceType, theRequestPartitionId); 735 if (searchToUse == null) { 736 return null; 737 } 738 739 ourLog.debug("Reusing search {} from cache", searchToUse.getUuid()); 740 // Interceptor call: JPA_PERFTRACE_SEARCH_REUSING_CACHED 741 params = new HookParams() 742 .add(SearchParameterMap.class, theParams) 743 .add(RequestDetails.class, theRequestDetails) 744 .addIfMatchesType(ServletRequestDetails.class, theRequestDetails); 745 compositeBroadcaster.callHooks(Pointcut.JPA_PERFTRACE_SEARCH_REUSING_CACHED, params); 746 747 return myPersistedJpaBundleProviderFactory.newInstance(theRequestDetails, searchToUse.getUuid()); 748 }); 749 } 750 751 @Nullable 752 private Search findSearchToUseOrNull( 753 String theQueryString, String theResourceType, RequestPartitionId theRequestPartitionId) { 754 // createdCutoff is in recent past 755 final Instant createdCutoff = 756 Instant.now().minus(myStorageSettings.getReuseCachedSearchResultsForMillis(), ChronoUnit.MILLIS); 757 758 Optional<Search> candidate = mySearchCacheSvc.findCandidatesForReuse( 759 theResourceType, theQueryString, createdCutoff, theRequestPartitionId); 760 return candidate.orElse(null); 761 } 762 763 @Nullable 764 private Integer getLoadSynchronousUpToOrNull(CacheControlDirective theCacheControlDirective) { 765 final Integer loadSynchronousUpTo; 766 if (theCacheControlDirective != null && theCacheControlDirective.isNoStore()) { 767 if (theCacheControlDirective.getMaxResults() != null) { 768 loadSynchronousUpTo = theCacheControlDirective.getMaxResults(); 769 if (loadSynchronousUpTo > myStorageSettings.getCacheControlNoStoreMaxResultsUpperLimit()) { 770 throw new InvalidRequestException(Msg.code(1165) + Constants.HEADER_CACHE_CONTROL + " header " 771 + Constants.CACHE_CONTROL_MAX_RESULTS + " value must not exceed " 772 + myStorageSettings.getCacheControlNoStoreMaxResultsUpperLimit()); 773 } 774 } else { 775 loadSynchronousUpTo = 100; 776 } 777 } else { 778 loadSynchronousUpTo = null; 779 } 780 return loadSynchronousUpTo; 781 } 782 783 /** 784 * Creates a {@link Pageable} using a start and end index 785 */ 786 @SuppressWarnings("WeakerAccess") 787 @Nullable 788 public static Pageable toPage(final int theFromIndex, int theToIndex) { 789 int pageSize = theToIndex - theFromIndex; 790 if (pageSize < 1) { 791 return null; 792 } 793 794 int pageIndex = theFromIndex / pageSize; 795 796 return new PageRequest(pageIndex, pageSize, Sort.unsorted()) { 797 @Serial 798 private static final long serialVersionUID = 1L; 799 800 @Override 801 public long getOffset() { 802 return theFromIndex; 803 } 804 }; 805 } 806}