001/* 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.search.builder; 021 022import ca.uhn.fhir.context.ComboSearchParamType; 023import ca.uhn.fhir.context.FhirContext; 024import ca.uhn.fhir.context.FhirVersionEnum; 025import ca.uhn.fhir.context.RuntimeResourceDefinition; 026import ca.uhn.fhir.context.RuntimeSearchParam; 027import ca.uhn.fhir.i18n.Msg; 028import ca.uhn.fhir.interceptor.api.HookParams; 029import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; 030import ca.uhn.fhir.interceptor.api.Pointcut; 031import ca.uhn.fhir.interceptor.model.RequestPartitionId; 032import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 033import ca.uhn.fhir.jpa.api.dao.DaoRegistry; 034import ca.uhn.fhir.jpa.api.dao.IDao; 035import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; 036import ca.uhn.fhir.jpa.api.svc.IIdHelperService; 037import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean; 038import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider; 039import ca.uhn.fhir.jpa.dao.BaseStorageDao; 040import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; 041import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser; 042import ca.uhn.fhir.jpa.dao.IResultIterator; 043import ca.uhn.fhir.jpa.dao.ISearchBuilder; 044import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao; 045import ca.uhn.fhir.jpa.dao.data.IResourceTagDao; 046import ca.uhn.fhir.jpa.dao.search.ResourceNotFoundInIndexException; 047import ca.uhn.fhir.jpa.entity.ResourceSearchView; 048import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; 049import ca.uhn.fhir.jpa.model.config.PartitionSettings; 050import ca.uhn.fhir.jpa.model.dao.JpaPid; 051import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam; 052import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity; 053import ca.uhn.fhir.jpa.model.entity.ResourceTag; 054import ca.uhn.fhir.jpa.model.search.SearchBuilderLoadIncludesParameters; 055import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; 056import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; 057import ca.uhn.fhir.jpa.search.SearchConstants; 058import ca.uhn.fhir.jpa.search.builder.models.ResolvedSearchQueryExecutor; 059import ca.uhn.fhir.jpa.search.builder.sql.GeneratedSql; 060import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; 061import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryExecutor; 062import ca.uhn.fhir.jpa.search.builder.sql.SqlObjectFactory; 063import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc; 064import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; 065import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper; 066import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil; 067import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper; 068import ca.uhn.fhir.jpa.util.BaseIterator; 069import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener; 070import ca.uhn.fhir.jpa.util.QueryChunker; 071import ca.uhn.fhir.jpa.util.SqlQueryList; 072import ca.uhn.fhir.model.api.IQueryParameterType; 073import ca.uhn.fhir.model.api.Include; 074import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; 075import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum; 076import ca.uhn.fhir.rest.api.Constants; 077import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; 078import ca.uhn.fhir.rest.api.SearchContainedModeEnum; 079import ca.uhn.fhir.rest.api.SortOrderEnum; 080import ca.uhn.fhir.rest.api.SortSpec; 081import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails; 082import ca.uhn.fhir.rest.api.server.RequestDetails; 083import ca.uhn.fhir.rest.param.DateRangeParam; 084import ca.uhn.fhir.rest.param.ParameterUtil; 085import ca.uhn.fhir.rest.param.ReferenceParam; 086import ca.uhn.fhir.rest.param.StringParam; 087import ca.uhn.fhir.rest.param.TokenParam; 088import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 089import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; 090import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; 091import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; 092import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; 093import ca.uhn.fhir.util.StopWatch; 094import ca.uhn.fhir.util.StringUtil; 095import ca.uhn.fhir.util.UrlUtil; 096import com.google.common.collect.Streams; 097import com.healthmarketscience.sqlbuilder.Condition; 098import jakarta.annotation.Nonnull; 099import jakarta.annotation.Nullable; 100import jakarta.persistence.EntityManager; 101import jakarta.persistence.PersistenceContext; 102import jakarta.persistence.PersistenceContextType; 103import jakarta.persistence.Query; 104import jakarta.persistence.Tuple; 105import jakarta.persistence.TypedQuery; 106import jakarta.persistence.criteria.CriteriaBuilder; 107import org.apache.commons.lang3.StringUtils; 108import org.apache.commons.lang3.Validate; 109import org.apache.commons.lang3.math.NumberUtils; 110import org.apache.commons.lang3.tuple.Pair; 111import org.hl7.fhir.instance.model.api.IAnyResource; 112import org.hl7.fhir.instance.model.api.IBaseResource; 113import org.slf4j.Logger; 114import org.slf4j.LoggerFactory; 115import org.springframework.beans.factory.annotation.Autowired; 116import org.springframework.jdbc.core.JdbcTemplate; 117import org.springframework.jdbc.core.SingleColumnRowMapper; 118import org.springframework.transaction.support.TransactionSynchronizationManager; 119 120import java.util.ArrayList; 121import java.util.Collection; 122import java.util.Collections; 123import java.util.HashMap; 124import java.util.HashSet; 125import java.util.Iterator; 126import java.util.List; 127import java.util.Map; 128import java.util.Objects; 129import java.util.Set; 130import java.util.stream.Collectors; 131 132import static ca.uhn.fhir.jpa.model.util.JpaConstants.UNDESIRED_RESOURCE_LINKAGES_FOR_EVERYTHING_ON_PATIENT_INSTANCE; 133import static ca.uhn.fhir.jpa.search.builder.QueryStack.LOCATION_POSITION; 134import static ca.uhn.fhir.jpa.search.builder.QueryStack.SearchForIdsParams.with; 135import static org.apache.commons.lang3.StringUtils.defaultString; 136import static org.apache.commons.lang3.StringUtils.isBlank; 137import static org.apache.commons.lang3.StringUtils.isNotBlank; 138 139/** 140 * The SearchBuilder is responsible for actually forming the SQL query that handles 141 * searches for resources 142 */ 143public class SearchBuilder implements ISearchBuilder<JpaPid> { 144 145 /** 146 * See loadResourcesByPid 147 * for an explanation of why we use the constant 800 148 */ 149 // NB: keep public 150 @Deprecated 151 public static final int MAXIMUM_PAGE_SIZE = SearchConstants.MAX_PAGE_SIZE; 152 153 public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50; 154 public static final String RESOURCE_ID_ALIAS = "resource_id"; 155 public static final String RESOURCE_VERSION_ALIAS = "resource_version"; 156 private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class); 157 private static final JpaPid NO_MORE = JpaPid.fromId(-1L); 158 private static final String MY_TARGET_RESOURCE_PID = "myTargetResourcePid"; 159 private static final String MY_SOURCE_RESOURCE_PID = "mySourceResourcePid"; 160 private static final String MY_TARGET_RESOURCE_TYPE = "myTargetResourceType"; 161 private static final String MY_SOURCE_RESOURCE_TYPE = "mySourceResourceType"; 162 private static final String MY_TARGET_RESOURCE_VERSION = "myTargetResourceVersion"; 163 public static boolean myUseMaxPageSize50ForTest = false; 164 protected final IInterceptorBroadcaster myInterceptorBroadcaster; 165 protected final IResourceTagDao myResourceTagDao; 166 private final String myResourceName; 167 private final Class<? extends IBaseResource> myResourceType; 168 private final HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory; 169 private final SqlObjectFactory mySqlBuilderFactory; 170 private final HibernatePropertiesProvider myDialectProvider; 171 private final ISearchParamRegistry mySearchParamRegistry; 172 private final PartitionSettings myPartitionSettings; 173 private final DaoRegistry myDaoRegistry; 174 private final IResourceSearchViewDao myResourceSearchViewDao; 175 private final FhirContext myContext; 176 private final IIdHelperService<JpaPid> myIdHelperService; 177 private final JpaStorageSettings myStorageSettings; 178 private final IDao myCallingDao; 179 180 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 181 protected EntityManager myEntityManager; 182 183 private CriteriaBuilder myCriteriaBuilder; 184 private SearchParameterMap myParams; 185 private String mySearchUuid; 186 private int myFetchSize; 187 private Integer myMaxResultsToFetch; 188 private Set<JpaPid> myPidSet; 189 private boolean myHasNextIteratorQuery = false; 190 private RequestPartitionId myRequestPartitionId; 191 192 @Autowired(required = false) 193 private IFulltextSearchSvc myFulltextSearchSvc; 194 195 @Autowired(required = false) 196 private IElasticsearchSvc myIElasticsearchSvc; 197 198 @Autowired 199 private IJpaStorageResourceParser myJpaStorageResourceParser; 200 201 /** 202 * Constructor 203 */ 204 public SearchBuilder( 205 IDao theDao, 206 String theResourceName, 207 JpaStorageSettings theStorageSettings, 208 HapiFhirLocalContainerEntityManagerFactoryBean theEntityManagerFactory, 209 SqlObjectFactory theSqlBuilderFactory, 210 HibernatePropertiesProvider theDialectProvider, 211 ISearchParamRegistry theSearchParamRegistry, 212 PartitionSettings thePartitionSettings, 213 IInterceptorBroadcaster theInterceptorBroadcaster, 214 IResourceTagDao theResourceTagDao, 215 DaoRegistry theDaoRegistry, 216 IResourceSearchViewDao theResourceSearchViewDao, 217 FhirContext theContext, 218 IIdHelperService theIdHelperService, 219 Class<? extends IBaseResource> theResourceType) { 220 myCallingDao = theDao; 221 myResourceName = theResourceName; 222 myResourceType = theResourceType; 223 myStorageSettings = theStorageSettings; 224 225 myEntityManagerFactory = theEntityManagerFactory; 226 mySqlBuilderFactory = theSqlBuilderFactory; 227 myDialectProvider = theDialectProvider; 228 mySearchParamRegistry = theSearchParamRegistry; 229 myPartitionSettings = thePartitionSettings; 230 myInterceptorBroadcaster = theInterceptorBroadcaster; 231 myResourceTagDao = theResourceTagDao; 232 myDaoRegistry = theDaoRegistry; 233 myResourceSearchViewDao = theResourceSearchViewDao; 234 myContext = theContext; 235 myIdHelperService = theIdHelperService; 236 } 237 238 @Override 239 public void setMaxResultsToFetch(Integer theMaxResultsToFetch) { 240 myMaxResultsToFetch = theMaxResultsToFetch; 241 } 242 243 private void searchForIdsWithAndOr( 244 SearchQueryBuilder theSearchSqlBuilder, 245 QueryStack theQueryStack, 246 @Nonnull SearchParameterMap theParams, 247 RequestDetails theRequest) { 248 myParams = theParams; 249 250 // Remove any empty parameters 251 theParams.clean(); 252 253 // For DSTU3, pull out near-distance first so when it comes time to evaluate near, we already know the distance 254 if (myContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { 255 Dstu3DistanceHelper.setNearDistance(myResourceType, theParams); 256 } 257 258 // Attempt to lookup via composite unique key. 259 if (isCompositeUniqueSpCandidate()) { 260 attemptComboUniqueSpProcessing(theQueryStack, theParams, theRequest); 261 } 262 263 SearchContainedModeEnum searchContainedMode = theParams.getSearchContainedMode(); 264 265 // Handle _id and _tag last, since they can typically be tacked onto a different parameter 266 List<String> paramNames = myParams.keySet().stream() 267 .filter(t -> !t.equals(IAnyResource.SP_RES_ID)) 268 .filter(t -> !t.equals(Constants.PARAM_TAG)) 269 .collect(Collectors.toList()); 270 if (myParams.containsKey(IAnyResource.SP_RES_ID)) { 271 paramNames.add(IAnyResource.SP_RES_ID); 272 } 273 if (myParams.containsKey(Constants.PARAM_TAG)) { 274 paramNames.add(Constants.PARAM_TAG); 275 } 276 277 // Handle each parameter 278 for (String nextParamName : paramNames) { 279 if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) { 280 // Skip parameters for Subject, Patient, Code and Category for LastN as these will be filtered by 281 // Elasticsearch 282 continue; 283 } 284 List<List<IQueryParameterType>> andOrParams = myParams.get(nextParamName); 285 Condition predicate = theQueryStack.searchForIdsWithAndOr(with().setResourceName(myResourceName) 286 .setParamName(nextParamName) 287 .setAndOrParams(andOrParams) 288 .setRequest(theRequest) 289 .setRequestPartitionId(myRequestPartitionId)); 290 if (predicate != null) { 291 theSearchSqlBuilder.addPredicate(predicate); 292 } 293 } 294 } 295 296 /** 297 * A search is a candidate for Composite Unique SP if unique indexes are enabled, there is no EverythingMode, and the 298 * parameters all have no modifiers. 299 */ 300 private boolean isCompositeUniqueSpCandidate() { 301 return myStorageSettings.isUniqueIndexesEnabled() 302 && myParams.getEverythingMode() == null 303 && myParams.isAllParametersHaveNoModifier(); 304 } 305 306 @SuppressWarnings("ConstantConditions") 307 @Override 308 public Long createCountQuery( 309 SearchParameterMap theParams, 310 String theSearchUuid, 311 RequestDetails theRequest, 312 @Nonnull RequestPartitionId theRequestPartitionId) { 313 314 assert theRequestPartitionId != null; 315 assert TransactionSynchronizationManager.isActualTransactionActive(); 316 317 init(theParams, theSearchUuid, theRequestPartitionId); 318 319 if (checkUseHibernateSearch()) { 320 long count = myFulltextSearchSvc.count(myResourceName, theParams.clone()); 321 return count; 322 } 323 324 List<ISearchQueryExecutor> queries = createQuery(theParams.clone(), null, null, null, true, theRequest, null); 325 if (queries.isEmpty()) { 326 return 0L; 327 } else { 328 return queries.get(0).next(); 329 } 330 } 331 332 /** 333 * @param thePidSet May be null 334 */ 335 @Override 336 public void setPreviouslyAddedResourcePids(@Nonnull List<JpaPid> thePidSet) { 337 myPidSet = new HashSet<>(thePidSet); 338 } 339 340 @SuppressWarnings("ConstantConditions") 341 @Override 342 public IResultIterator<JpaPid> createQuery( 343 SearchParameterMap theParams, 344 SearchRuntimeDetails theSearchRuntimeDetails, 345 RequestDetails theRequest, 346 @Nonnull RequestPartitionId theRequestPartitionId) { 347 assert theRequestPartitionId != null; 348 assert TransactionSynchronizationManager.isActualTransactionActive(); 349 350 init(theParams, theSearchRuntimeDetails.getSearchUuid(), theRequestPartitionId); 351 352 if (myPidSet == null) { 353 myPidSet = new HashSet<>(); 354 } 355 356 return new QueryIterator(theSearchRuntimeDetails, theRequest); 357 } 358 359 private void init(SearchParameterMap theParams, String theSearchUuid, RequestPartitionId theRequestPartitionId) { 360 myCriteriaBuilder = myEntityManager.getCriteriaBuilder(); 361 // we mutate the params. Make a private copy. 362 myParams = theParams.clone(); 363 mySearchUuid = theSearchUuid; 364 myRequestPartitionId = theRequestPartitionId; 365 } 366 367 private List<ISearchQueryExecutor> createQuery( 368 SearchParameterMap theParams, 369 SortSpec sort, 370 Integer theOffset, 371 Integer theMaximumResults, 372 boolean theCountOnlyFlag, 373 RequestDetails theRequest, 374 SearchRuntimeDetails theSearchRuntimeDetails) { 375 376 ArrayList<ISearchQueryExecutor> queries = new ArrayList<>(); 377 378 if (checkUseHibernateSearch()) { 379 // we're going to run at least part of the search against the Fulltext service. 380 381 // Ugh - we have two different return types for now 382 ISearchQueryExecutor fulltextExecutor = null; 383 List<JpaPid> fulltextMatchIds = null; 384 int resultCount = 0; 385 if (myParams.isLastN()) { 386 fulltextMatchIds = executeLastNAgainstIndex(theMaximumResults); 387 resultCount = fulltextMatchIds.size(); 388 } else if (myParams.getEverythingMode() != null) { 389 fulltextMatchIds = queryHibernateSearchForEverythingPids(theRequest); 390 resultCount = fulltextMatchIds.size(); 391 } else { 392 fulltextExecutor = myFulltextSearchSvc.searchNotScrolled( 393 myResourceName, myParams, myMaxResultsToFetch, theRequest); 394 } 395 396 if (fulltextExecutor == null) { 397 fulltextExecutor = SearchQueryExecutors.from(fulltextMatchIds); 398 } 399 400 if (theSearchRuntimeDetails != null) { 401 theSearchRuntimeDetails.setFoundIndexMatchesCount(resultCount); 402 HookParams params = new HookParams() 403 .add(RequestDetails.class, theRequest) 404 .addIfMatchesType(ServletRequestDetails.class, theRequest) 405 .add(SearchRuntimeDetails.class, theSearchRuntimeDetails); 406 CompositeInterceptorBroadcaster.doCallHooks( 407 myInterceptorBroadcaster, 408 theRequest, 409 Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, 410 params); 411 } 412 413 // can we skip the database entirely and return the pid list from here? 414 boolean canSkipDatabase = 415 // if we processed an AND clause, and it returned nothing, then nothing can match. 416 !fulltextExecutor.hasNext() 417 || 418 // Our hibernate search query doesn't respect partitions yet 419 (!myPartitionSettings.isPartitioningEnabled() 420 && 421 // were there AND terms left? Then we still need the db. 422 theParams.isEmpty() 423 && 424 // not every param is a param. :-( 425 theParams.getNearDistanceParam() == null 426 && 427 // todo MB don't we support _lastUpdated and _offset now? 428 theParams.getLastUpdated() == null 429 && theParams.getEverythingMode() == null 430 && theParams.getOffset() == null); 431 432 if (canSkipDatabase) { 433 ourLog.trace("Query finished after HSearch. Skip db query phase"); 434 if (theMaximumResults != null) { 435 fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theMaximumResults); 436 } 437 queries.add(fulltextExecutor); 438 } else { 439 ourLog.trace("Query needs db after HSearch. Chunking."); 440 // Finish the query in the database for the rest of the search parameters, sorting, partitioning, etc. 441 // We break the pids into chunks that fit in the 1k limit for jdbc bind params. 442 new QueryChunker<Long>() 443 .chunk( 444 Streams.stream(fulltextExecutor).collect(Collectors.toList()), 445 t -> doCreateChunkedQueries( 446 theParams, t, theOffset, sort, theCountOnlyFlag, theRequest, queries)); 447 } 448 } else { 449 // do everything in the database. 450 createChunkedQuery( 451 theParams, sort, theOffset, theMaximumResults, theCountOnlyFlag, theRequest, null, queries); 452 } 453 454 return queries; 455 } 456 457 /** 458 * Check to see if query should use Hibernate Search, and error if the query can't continue. 459 * 460 * @return true if the query should first be processed by Hibernate Search 461 * @throws InvalidRequestException if fulltext search is not enabled but the query requires it - _content or _text 462 */ 463 private boolean checkUseHibernateSearch() { 464 boolean fulltextEnabled = (myFulltextSearchSvc != null) && !myFulltextSearchSvc.isDisabled(); 465 466 if (!fulltextEnabled) { 467 failIfUsed(Constants.PARAM_TEXT); 468 failIfUsed(Constants.PARAM_CONTENT); 469 } 470 471 // someday we'll want a query planner to figure out if we _should_ or _must_ use the ft index, not just if we 472 // can. 473 return fulltextEnabled 474 && myParams != null 475 && myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE 476 && myFulltextSearchSvc.supportsSomeOf(myParams); 477 } 478 479 private void failIfUsed(String theParamName) { 480 if (myParams.containsKey(theParamName)) { 481 throw new InvalidRequestException(Msg.code(1192) 482 + "Fulltext search is not enabled on this service, can not process parameter: " + theParamName); 483 } 484 } 485 486 private List<JpaPid> executeLastNAgainstIndex(Integer theMaximumResults) { 487 // Can we use our hibernate search generated index on resource to support lastN?: 488 if (myStorageSettings.isAdvancedHSearchIndexing()) { 489 if (myFulltextSearchSvc == null) { 490 throw new InvalidRequestException(Msg.code(2027) 491 + "LastN operation is not enabled on this service, can not process this request"); 492 } 493 return myFulltextSearchSvc.lastN(myParams, theMaximumResults).stream() 494 .map(lastNResourceId -> myIdHelperService.resolveResourcePersistentIds( 495 myRequestPartitionId, myResourceName, String.valueOf(lastNResourceId))) 496 .collect(Collectors.toList()); 497 } else { 498 throw new InvalidRequestException( 499 Msg.code(2033) + "LastN operation is not enabled on this service, can not process this request"); 500 } 501 } 502 503 private List<JpaPid> queryHibernateSearchForEverythingPids(RequestDetails theRequestDetails) { 504 JpaPid pid = null; 505 if (myParams.get(IAnyResource.SP_RES_ID) != null) { 506 String idParamValue; 507 IQueryParameterType idParam = 508 myParams.get(IAnyResource.SP_RES_ID).get(0).get(0); 509 if (idParam instanceof TokenParam) { 510 TokenParam idParm = (TokenParam) idParam; 511 idParamValue = idParm.getValue(); 512 } else { 513 StringParam idParm = (StringParam) idParam; 514 idParamValue = idParm.getValue(); 515 } 516 517 pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue); 518 } 519 List<JpaPid> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid, theRequestDetails); 520 return pids; 521 } 522 523 private void doCreateChunkedQueries( 524 SearchParameterMap theParams, 525 List<Long> thePids, 526 Integer theOffset, 527 SortSpec sort, 528 boolean theCount, 529 RequestDetails theRequest, 530 ArrayList<ISearchQueryExecutor> theQueries) { 531 if (thePids.size() < getMaximumPageSize()) { 532 normalizeIdListForLastNInClause(thePids); 533 } 534 createChunkedQuery(theParams, sort, theOffset, thePids.size(), theCount, theRequest, thePids, theQueries); 535 } 536 537 /** 538 * Combs through the params for any _id parameters and extracts the PIDs for them 539 * 540 * @param theTargetPids 541 */ 542 private void extractTargetPidsFromIdParams( 543 HashSet<Long> theTargetPids, List<ISearchQueryExecutor> theSearchQueryExecutors) { 544 // get all the IQueryParameterType objects 545 // for _id -> these should all be StringParam values 546 HashSet<String> ids = new HashSet<>(); 547 List<List<IQueryParameterType>> params = myParams.get(IAnyResource.SP_RES_ID); 548 for (List<IQueryParameterType> paramList : params) { 549 for (IQueryParameterType param : paramList) { 550 if (param instanceof StringParam) { 551 // we expect all _id values to be StringParams 552 ids.add(((StringParam) param).getValue()); 553 } else if (param instanceof TokenParam) { 554 ids.add(((TokenParam) param).getValue()); 555 } else { 556 // we do not expect the _id parameter to be a non-string value 557 throw new IllegalArgumentException( 558 Msg.code(1193) + "_id parameter must be a StringParam or TokenParam"); 559 } 560 } 561 } 562 563 // fetch our target Pids 564 // this will throw if an id is not found 565 Map<String, JpaPid> idToPid = myIdHelperService.resolveResourcePersistentIds( 566 myRequestPartitionId, myResourceName, new ArrayList<>(ids)); 567 568 // add the pids to targetPids 569 for (JpaPid pid : idToPid.values()) { 570 theTargetPids.add(pid.getId()); 571 } 572 573 // add the target pids to our executors as the first 574 // results iterator to go through 575 theSearchQueryExecutors.add(new ResolvedSearchQueryExecutor(new ArrayList<>(theTargetPids))); 576 } 577 578 private void createChunkedQuery( 579 SearchParameterMap theParams, 580 SortSpec sort, 581 Integer theOffset, 582 Integer theMaximumResults, 583 boolean theCountOnlyFlag, 584 RequestDetails theRequest, 585 List<Long> thePidList, 586 List<ISearchQueryExecutor> theSearchQueryExecutors) { 587 String sqlBuilderResourceName = myParams.getEverythingMode() == null ? myResourceName : null; 588 SearchQueryBuilder sqlBuilder = new SearchQueryBuilder( 589 myContext, 590 myStorageSettings, 591 myPartitionSettings, 592 myRequestPartitionId, 593 sqlBuilderResourceName, 594 mySqlBuilderFactory, 595 myDialectProvider, 596 theCountOnlyFlag); 597 QueryStack queryStack3 = new QueryStack( 598 theParams, myStorageSettings, myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings); 599 600 if (theParams.keySet().size() > 1 601 || theParams.getSort() != null 602 || theParams.keySet().contains(Constants.PARAM_HAS) 603 || isPotentiallyContainedReferenceParameterExistsAtRoot(theParams)) { 604 List<RuntimeSearchParam> activeComboParams = 605 mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet()); 606 if (activeComboParams.isEmpty()) { 607 sqlBuilder.setNeedResourceTableRoot(true); 608 } 609 } 610 611 JdbcTemplate jdbcTemplate = new JdbcTemplate(myEntityManagerFactory.getDataSource()); 612 jdbcTemplate.setFetchSize(myFetchSize); 613 if (theMaximumResults != null) { 614 jdbcTemplate.setMaxRows(theMaximumResults); 615 } 616 617 if (myParams.getEverythingMode() != null) { 618 HashSet<Long> targetPids = new HashSet<>(); 619 if (myParams.get(IAnyResource.SP_RES_ID) != null) { 620 // will add an initial search executor for 621 // _id params 622 extractTargetPidsFromIdParams(targetPids, theSearchQueryExecutors); 623 } else { 624 // For Everything queries, we make the query root by the ResourceLink table, since this query 625 // is basically a reverse-include search. For type/Everything (as opposed to instance/Everything) 626 // the one problem with this approach is that it doesn't catch Patients that have absolutely 627 // nothing linked to them. So we do one additional query to make sure we catch those too. 628 SearchQueryBuilder fetchPidsSqlBuilder = new SearchQueryBuilder( 629 myContext, 630 myStorageSettings, 631 myPartitionSettings, 632 myRequestPartitionId, 633 myResourceName, 634 mySqlBuilderFactory, 635 myDialectProvider, 636 theCountOnlyFlag); 637 GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theOffset, myMaxResultsToFetch); 638 String sql = allTargetsSql.getSql(); 639 Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]); 640 641 List<Long> output = jdbcTemplate.query(sql, args, new SingleColumnRowMapper<>(Long.class)); 642 643 // we add a search executor to fetch unlinked patients first 644 theSearchQueryExecutors.add(new ResolvedSearchQueryExecutor(output)); 645 } 646 647 List<String> typeSourceResources = new ArrayList<>(); 648 if (myParams.get(Constants.PARAM_TYPE) != null) { 649 typeSourceResources.addAll(extractTypeSourceResourcesFromParams()); 650 } 651 652 queryStack3.addPredicateEverythingOperation( 653 myResourceName, typeSourceResources, targetPids.toArray(new Long[0])); 654 } else { 655 /* 656 * If we're doing a filter, always use the resource table as the root - This avoids the possibility of 657 * specific filters with ORs as their root from working around the natural resource type / deletion 658 * status / partition IDs built into queries. 659 */ 660 if (theParams.containsKey(Constants.PARAM_FILTER)) { 661 Condition partitionIdPredicate = sqlBuilder 662 .getOrCreateResourceTablePredicateBuilder() 663 .createPartitionIdPredicate(myRequestPartitionId); 664 if (partitionIdPredicate != null) { 665 sqlBuilder.addPredicate(partitionIdPredicate); 666 } 667 } 668 669 // Normal search 670 searchForIdsWithAndOr(sqlBuilder, queryStack3, myParams, theRequest); 671 } 672 673 // If we haven't added any predicates yet, we're doing a search for all resources. Make sure we add the 674 // partition ID predicate in that case. 675 if (!sqlBuilder.haveAtLeastOnePredicate()) { 676 Condition partitionIdPredicate = sqlBuilder 677 .getOrCreateResourceTablePredicateBuilder() 678 .createPartitionIdPredicate(myRequestPartitionId); 679 if (partitionIdPredicate != null) { 680 sqlBuilder.addPredicate(partitionIdPredicate); 681 } 682 } 683 684 // Add PID list predicate for full text search and/or lastn operation 685 if (thePidList != null && thePidList.size() > 0) { 686 sqlBuilder.addResourceIdsPredicate(thePidList); 687 } 688 689 // Last updated 690 DateRangeParam lu = myParams.getLastUpdated(); 691 if (lu != null && !lu.isEmpty()) { 692 Condition lastUpdatedPredicates = sqlBuilder.addPredicateLastUpdated(lu); 693 sqlBuilder.addPredicate(lastUpdatedPredicates); 694 } 695 696 /* 697 * Exclude the pids already in the previous iterator. This is an optimization, as opposed 698 * to something needed to guarantee correct results. 699 * 700 * Why do we need it? Suppose for example, a query like: 701 * Observation?category=foo,bar,baz 702 * And suppose you have many resources that have all 3 of these category codes. In this case 703 * the SQL query will probably return the same PIDs multiple times, and if this happens enough 704 * we may exhaust the query results without getting enough distinct results back. When that 705 * happens we re-run the query with a larger limit. Excluding results we already know about 706 * tries to ensure that we get new unique results. 707 * 708 * The challenge with that though is that lots of DBs have an issue with too many 709 * parameters in one query. So we only do this optimization if there aren't too 710 * many results. 711 */ 712 if (myHasNextIteratorQuery) { 713 if (myPidSet.size() + sqlBuilder.countBindVariables() < 900) { 714 sqlBuilder.excludeResourceIdsPredicate(myPidSet); 715 } 716 } 717 718 /* 719 * If offset is present, we want deduplicate the results by using GROUP BY 720 */ 721 if (theOffset != null) { 722 queryStack3.addGrouping(); 723 queryStack3.setUseAggregate(true); 724 } 725 726 /* 727 * Sort 728 * 729 * If we have a sort, we wrap the criteria search (the search that actually 730 * finds the appropriate resources) in an outer search which is then sorted 731 */ 732 if (sort != null) { 733 assert !theCountOnlyFlag; 734 735 createSort(queryStack3, sort, theParams); 736 } 737 738 /* 739 * Now perform the search 740 */ 741 GeneratedSql generatedSql = sqlBuilder.generate(theOffset, myMaxResultsToFetch); 742 if (!generatedSql.isMatchNothing()) { 743 SearchQueryExecutor executor = 744 mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, myMaxResultsToFetch); 745 theSearchQueryExecutors.add(executor); 746 } 747 } 748 749 private Collection<String> extractTypeSourceResourcesFromParams() { 750 751 List<List<IQueryParameterType>> listOfList = myParams.get(Constants.PARAM_TYPE); 752 753 // first off, let's flatten the list of list 754 List<IQueryParameterType> iQueryParameterTypesList = 755 listOfList.stream().flatMap(List::stream).collect(Collectors.toList()); 756 757 // then, extract all elements of each CSV into one big list 758 List<String> resourceTypes = iQueryParameterTypesList.stream() 759 .map(param -> ((StringParam) param).getValue()) 760 .map(csvString -> List.of(csvString.split(","))) 761 .flatMap(List::stream) 762 .collect(Collectors.toList()); 763 764 Set<String> knownResourceTypes = myContext.getResourceTypes(); 765 766 // remove leading/trailing whitespaces if any and remove duplicates 767 Set<String> retVal = new HashSet<>(); 768 769 for (String type : resourceTypes) { 770 String trimmed = type.trim(); 771 if (!knownResourceTypes.contains(trimmed)) { 772 throw new ResourceNotFoundException( 773 Msg.code(2197) + "Unknown resource type '" + trimmed + "' in _type parameter."); 774 } 775 retVal.add(trimmed); 776 } 777 778 return retVal; 779 } 780 781 private boolean isPotentiallyContainedReferenceParameterExistsAtRoot(SearchParameterMap theParams) { 782 return myStorageSettings.isIndexOnContainedResources() 783 && theParams.values().stream() 784 .flatMap(Collection::stream) 785 .flatMap(Collection::stream) 786 .anyMatch(t -> t instanceof ReferenceParam); 787 } 788 789 private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) { 790 /* 791 The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying 792 numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info: 793 https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage. 794 795 Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of 796 arguments never exceeds the maximum specified below. 797 */ 798 int listSize = lastnResourceIds.size(); 799 800 if (listSize > 1 && listSize < 10) { 801 padIdListWithPlaceholders(lastnResourceIds, 10); 802 } else if (listSize > 10 && listSize < 50) { 803 padIdListWithPlaceholders(lastnResourceIds, 50); 804 } else if (listSize > 50 && listSize < 100) { 805 padIdListWithPlaceholders(lastnResourceIds, 100); 806 } else if (listSize > 100 && listSize < 200) { 807 padIdListWithPlaceholders(lastnResourceIds, 200); 808 } else if (listSize > 200 && listSize < 500) { 809 padIdListWithPlaceholders(lastnResourceIds, 500); 810 } else if (listSize > 500 && listSize < 800) { 811 padIdListWithPlaceholders(lastnResourceIds, 800); 812 } 813 814 return lastnResourceIds; 815 } 816 817 private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) { 818 while (theIdList.size() < preferredListSize) { 819 theIdList.add(-1L); 820 } 821 } 822 823 private void createSort(QueryStack theQueryStack, SortSpec theSort, SearchParameterMap theParams) { 824 if (theSort == null || isBlank(theSort.getParamName())) { 825 return; 826 } 827 828 boolean ascending = (theSort.getOrder() == null) || (theSort.getOrder() == SortOrderEnum.ASC); 829 830 if (IAnyResource.SP_RES_ID.equals(theSort.getParamName())) { 831 832 theQueryStack.addSortOnResourceId(ascending); 833 834 } else if (Constants.PARAM_PID.equals(theSort.getParamName())) { 835 836 theQueryStack.addSortOnResourcePID(ascending); 837 838 } else if (Constants.PARAM_LASTUPDATED.equals(theSort.getParamName())) { 839 840 theQueryStack.addSortOnLastUpdated(ascending); 841 842 } else { 843 844 RuntimeSearchParam param = null; 845 846 if (param == null) { 847 // do we have a composition param defined for the whole chain? 848 param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName()); 849 } 850 851 /* 852 * If we have a sort like _sort=subject.name and we have an 853 * uplifted refchain for that combination we can do it more efficiently 854 * by using the index associated with the uplifted refchain. In this case, 855 * we need to find the actual target search parameter (corresponding 856 * to "name" in this example) so that we know what datatype it is. 857 */ 858 String paramName = theSort.getParamName(); 859 if (param == null && myStorageSettings.isIndexOnUpliftedRefchains()) { 860 String[] chains = StringUtils.split(paramName, '.'); 861 if (chains.length == 2) { 862 863 // Given: Encounter?_sort=Patient:subject.name 864 String referenceParam = chains[0]; // subject 865 String referenceParamTargetType = null; // Patient 866 String targetParam = chains[1]; // name 867 868 int colonIdx = referenceParam.indexOf(':'); 869 if (colonIdx > -1) { 870 referenceParamTargetType = referenceParam.substring(0, colonIdx); 871 referenceParam = referenceParam.substring(colonIdx + 1); 872 } 873 RuntimeSearchParam outerParam = 874 mySearchParamRegistry.getActiveSearchParam(myResourceName, referenceParam); 875 if (outerParam == null) { 876 throwInvalidRequestExceptionForUnknownSortParameter(myResourceName, referenceParam); 877 } 878 879 if (outerParam.hasUpliftRefchain(targetParam)) { 880 for (String nextTargetType : outerParam.getTargets()) { 881 if (referenceParamTargetType != null && !referenceParamTargetType.equals(nextTargetType)) { 882 continue; 883 } 884 RuntimeSearchParam innerParam = 885 mySearchParamRegistry.getActiveSearchParam(nextTargetType, targetParam); 886 if (innerParam != null) { 887 param = innerParam; 888 break; 889 } 890 } 891 } 892 } 893 } 894 895 int colonIdx = paramName.indexOf(':'); 896 String referenceTargetType = null; 897 if (colonIdx > -1) { 898 referenceTargetType = paramName.substring(0, colonIdx); 899 paramName = paramName.substring(colonIdx + 1); 900 } 901 902 int dotIdx = paramName.indexOf('.'); 903 String chainName = null; 904 if (param == null && dotIdx > -1) { 905 chainName = paramName.substring(dotIdx + 1); 906 paramName = paramName.substring(0, dotIdx); 907 if (chainName.contains(".")) { 908 String msg = myContext 909 .getLocalizer() 910 .getMessageSanitized( 911 BaseStorageDao.class, 912 "invalidSortParameterTooManyChains", 913 paramName + "." + chainName); 914 throw new InvalidRequestException(Msg.code(2286) + msg); 915 } 916 } 917 918 if (param == null) { 919 param = mySearchParamRegistry.getActiveSearchParam(myResourceName, paramName); 920 } 921 922 if (param == null) { 923 throwInvalidRequestExceptionForUnknownSortParameter(getResourceName(), paramName); 924 } 925 926 if (isNotBlank(chainName) && param.getParamType() != RestSearchParameterTypeEnum.REFERENCE) { 927 throw new InvalidRequestException( 928 Msg.code(2285) + "Invalid chain, " + paramName + " is not a reference SearchParameter"); 929 } 930 931 switch (param.getParamType()) { 932 case STRING: 933 theQueryStack.addSortOnString(myResourceName, paramName, ascending); 934 break; 935 case DATE: 936 theQueryStack.addSortOnDate(myResourceName, paramName, ascending); 937 break; 938 case REFERENCE: 939 theQueryStack.addSortOnResourceLink( 940 myResourceName, referenceTargetType, paramName, chainName, ascending, theParams); 941 break; 942 case TOKEN: 943 theQueryStack.addSortOnToken(myResourceName, paramName, ascending); 944 break; 945 case NUMBER: 946 theQueryStack.addSortOnNumber(myResourceName, paramName, ascending); 947 break; 948 case URI: 949 theQueryStack.addSortOnUri(myResourceName, paramName, ascending); 950 break; 951 case QUANTITY: 952 theQueryStack.addSortOnQuantity(myResourceName, paramName, ascending); 953 break; 954 case COMPOSITE: 955 List<RuntimeSearchParam> compositeList = 956 JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, param); 957 if (compositeList == null) { 958 throw new InvalidRequestException(Msg.code(1195) + "The composite _sort parameter " + paramName 959 + " is not defined by the resource " + myResourceName); 960 } 961 if (compositeList.size() != 2) { 962 throw new InvalidRequestException(Msg.code(1196) + "The composite _sort parameter " + paramName 963 + " must have 2 composite types declared in parameter annotation, found " 964 + compositeList.size()); 965 } 966 RuntimeSearchParam left = compositeList.get(0); 967 RuntimeSearchParam right = compositeList.get(1); 968 969 createCompositeSort(theQueryStack, left.getParamType(), left.getName(), ascending); 970 createCompositeSort(theQueryStack, right.getParamType(), right.getName(), ascending); 971 972 break; 973 case SPECIAL: 974 if (LOCATION_POSITION.equals(param.getPath())) { 975 theQueryStack.addSortOnCoordsNear(paramName, ascending, theParams); 976 break; 977 } 978 throw new InvalidRequestException( 979 Msg.code(2306) + "This server does not support _sort specifications of type " 980 + param.getParamType() + " - Can't serve _sort=" + paramName); 981 982 case HAS: 983 default: 984 throw new InvalidRequestException( 985 Msg.code(1197) + "This server does not support _sort specifications of type " 986 + param.getParamType() + " - Can't serve _sort=" + paramName); 987 } 988 } 989 990 // Recurse 991 createSort(theQueryStack, theSort.getChain(), theParams); 992 } 993 994 private void throwInvalidRequestExceptionForUnknownSortParameter(String theResourceName, String theParamName) { 995 Collection<String> validSearchParameterNames = 996 mySearchParamRegistry.getValidSearchParameterNamesIncludingMeta(theResourceName); 997 String msg = myContext 998 .getLocalizer() 999 .getMessageSanitized( 1000 BaseStorageDao.class, 1001 "invalidSortParameter", 1002 theParamName, 1003 theResourceName, 1004 validSearchParameterNames); 1005 throw new InvalidRequestException(Msg.code(1194) + msg); 1006 } 1007 1008 private void createCompositeSort( 1009 QueryStack theQueryStack, 1010 RestSearchParameterTypeEnum theParamType, 1011 String theParamName, 1012 boolean theAscending) { 1013 1014 switch (theParamType) { 1015 case STRING: 1016 theQueryStack.addSortOnString(myResourceName, theParamName, theAscending); 1017 break; 1018 case DATE: 1019 theQueryStack.addSortOnDate(myResourceName, theParamName, theAscending); 1020 break; 1021 case TOKEN: 1022 theQueryStack.addSortOnToken(myResourceName, theParamName, theAscending); 1023 break; 1024 case QUANTITY: 1025 theQueryStack.addSortOnQuantity(myResourceName, theParamName, theAscending); 1026 break; 1027 case NUMBER: 1028 case REFERENCE: 1029 case COMPOSITE: 1030 case URI: 1031 case HAS: 1032 case SPECIAL: 1033 default: 1034 throw new InvalidRequestException( 1035 Msg.code(1198) + "Don't know how to handle composite parameter with type of " + theParamType 1036 + " on _sort=" + theParamName); 1037 } 1038 } 1039 1040 private void doLoadPids( 1041 Collection<JpaPid> thePids, 1042 Collection<JpaPid> theIncludedPids, 1043 List<IBaseResource> theResourceListToPopulate, 1044 boolean theForHistoryOperation, 1045 Map<JpaPid, Integer> thePosition) { 1046 1047 Map<Long, Long> resourcePidToVersion = null; 1048 for (JpaPid next : thePids) { 1049 if (next.getVersion() != null && myStorageSettings.isRespectVersionsForSearchIncludes()) { 1050 if (resourcePidToVersion == null) { 1051 resourcePidToVersion = new HashMap<>(); 1052 } 1053 resourcePidToVersion.put((next).getId(), next.getVersion()); 1054 } 1055 } 1056 1057 List<Long> versionlessPids = JpaPid.toLongList(thePids); 1058 if (versionlessPids.size() < getMaximumPageSize()) { 1059 versionlessPids = normalizeIdListForLastNInClause(versionlessPids); 1060 } 1061 1062 // -- get the resource from the searchView 1063 Collection<ResourceSearchView> resourceSearchViewList = 1064 myResourceSearchViewDao.findByResourceIds(versionlessPids); 1065 1066 // -- preload all tags with tag definition if any 1067 Map<Long, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList); 1068 1069 for (IBaseResourceEntity next : resourceSearchViewList) { 1070 if (next.getDeleted() != null) { 1071 continue; 1072 } 1073 1074 Class<? extends IBaseResource> resourceType = 1075 myContext.getResourceDefinition(next.getResourceType()).getImplementingClass(); 1076 1077 JpaPid resourceId = JpaPid.fromId(next.getResourceId()); 1078 1079 /* 1080 * If a specific version is requested via an include, we'll replace the current version 1081 * with the specific desired version. This is not the most efficient thing, given that 1082 * we're loading the current version and then turning around and throwing it away again. 1083 * This could be optimized and probably should be, but it's not critical given that 1084 * this only applies to includes, which don't tend to be massive in numbers. 1085 */ 1086 if (resourcePidToVersion != null) { 1087 Long version = resourcePidToVersion.get(next.getResourceId()); 1088 resourceId.setVersion(version); 1089 if (version != null && !version.equals(next.getVersion())) { 1090 IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(resourceType); 1091 next = (IBaseResourceEntity) 1092 dao.readEntity(next.getIdDt().withVersion(Long.toString(version)), null); 1093 } 1094 } 1095 1096 IBaseResource resource = null; 1097 if (next != null) { 1098 resource = myJpaStorageResourceParser.toResource( 1099 resourceType, next, tagMap.get(next.getId()), theForHistoryOperation); 1100 } 1101 if (resource == null) { 1102 ourLog.warn( 1103 "Unable to find resource {}/{}/_history/{} in database", 1104 next.getResourceType(), 1105 next.getIdDt().getIdPart(), 1106 next.getVersion()); 1107 continue; 1108 } 1109 1110 Integer index = thePosition.get(resourceId); 1111 if (index == null) { 1112 ourLog.warn("Got back unexpected resource PID {}", resourceId); 1113 continue; 1114 } 1115 1116 if (theIncludedPids.contains(resourceId)) { 1117 ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put(resource, BundleEntrySearchModeEnum.INCLUDE); 1118 } else { 1119 ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put(resource, BundleEntrySearchModeEnum.MATCH); 1120 } 1121 1122 theResourceListToPopulate.set(index, resource); 1123 } 1124 } 1125 1126 private Map<Long, Collection<ResourceTag>> getResourceTagMap( 1127 Collection<? extends IBaseResourceEntity> theResourceSearchViewList) { 1128 1129 List<Long> idList = new ArrayList<>(theResourceSearchViewList.size()); 1130 1131 // -- find all resource has tags 1132 for (IBaseResourceEntity resource : theResourceSearchViewList) { 1133 if (resource.isHasTags()) idList.add(resource.getId()); 1134 } 1135 1136 return getPidToTagMap(idList); 1137 } 1138 1139 @Nonnull 1140 private Map<Long, Collection<ResourceTag>> getPidToTagMap(List<Long> thePidList) { 1141 Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>(); 1142 1143 // -- no tags 1144 if (thePidList.size() == 0) return tagMap; 1145 1146 // -- get all tags for the idList 1147 Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(thePidList); 1148 1149 // -- build the map, key = resourceId, value = list of ResourceTag 1150 JpaPid resourceId; 1151 Collection<ResourceTag> tagCol; 1152 for (ResourceTag tag : tagList) { 1153 1154 resourceId = JpaPid.fromId(tag.getResourceId()); 1155 tagCol = tagMap.get(resourceId.getId()); 1156 if (tagCol == null) { 1157 tagCol = new ArrayList<>(); 1158 tagCol.add(tag); 1159 tagMap.put(resourceId.getId(), tagCol); 1160 } else { 1161 tagCol.add(tag); 1162 } 1163 } 1164 1165 return tagMap; 1166 } 1167 1168 @Override 1169 public void loadResourcesByPid( 1170 Collection<JpaPid> thePids, 1171 Collection<JpaPid> theIncludedPids, 1172 List<IBaseResource> theResourceListToPopulate, 1173 boolean theForHistoryOperation, 1174 RequestDetails theDetails) { 1175 if (thePids.isEmpty()) { 1176 ourLog.debug("The include pids are empty"); 1177 // return; 1178 } 1179 1180 // Dupes will cause a crash later anyhow, but this is expensive so only do it 1181 // when running asserts 1182 assert new HashSet<>(thePids).size() == thePids.size() : "PID list contains duplicates: " + thePids; 1183 1184 Map<JpaPid, Integer> position = new HashMap<>(); 1185 for (JpaPid next : thePids) { 1186 position.put(next, theResourceListToPopulate.size()); 1187 theResourceListToPopulate.add(null); 1188 } 1189 1190 // Can we fast track this loading by checking elastic search? 1191 if (isLoadingFromElasticSearchSupported(thePids)) { 1192 try { 1193 theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids)); 1194 return; 1195 1196 } catch (ResourceNotFoundInIndexException theE) { 1197 // some resources were not found in index, so we will inform this and resort to JPA search 1198 ourLog.warn( 1199 "Some resources were not found in index. Make sure all resources were indexed. Resorting to database search."); 1200 } 1201 } 1202 1203 // We only chunk because some jdbc drivers can't handle long param lists. 1204 new QueryChunker<JpaPid>() 1205 .chunk( 1206 thePids, 1207 t -> doLoadPids( 1208 t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position)); 1209 } 1210 1211 /** 1212 * Check if we can load the resources from Hibernate Search instead of the database. 1213 * We assume this is faster. 1214 * <p> 1215 * Hibernate Search only stores the current version, and only if enabled. 1216 * 1217 * @param thePids the pids to check for versioned references 1218 * @return can we fetch from Hibernate Search? 1219 */ 1220 private boolean isLoadingFromElasticSearchSupported(Collection<JpaPid> thePids) { 1221 // is storage enabled? 1222 return myStorageSettings.isStoreResourceInHSearchIndex() 1223 && myStorageSettings.isAdvancedHSearchIndexing() 1224 && 1225 // we don't support history 1226 thePids.stream().noneMatch(p -> p.getVersion() != null) 1227 && 1228 // skip the complexity for metadata in dstu2 1229 myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3); 1230 } 1231 1232 private List<IBaseResource> loadResourcesFromElasticSearch(Collection<JpaPid> thePids) { 1233 // Do we use the fulltextsvc via hibernate-search to load resources or be backwards compatible with older ES 1234 // only impl 1235 // to handle lastN? 1236 if (myStorageSettings.isAdvancedHSearchIndexing() && myStorageSettings.isStoreResourceInHSearchIndex()) { 1237 List<Long> pidList = thePids.stream().map(pid -> (pid).getId()).collect(Collectors.toList()); 1238 1239 List<IBaseResource> resources = myFulltextSearchSvc.getResources(pidList); 1240 return resources; 1241 } else if (!Objects.isNull(myParams) && myParams.isLastN()) { 1242 // legacy LastN implementation 1243 return myIElasticsearchSvc.getObservationResources(thePids); 1244 } else { 1245 return Collections.emptyList(); 1246 } 1247 } 1248 1249 /** 1250 * THIS SHOULD RETURN HASHSET and not just Set because we add to it later 1251 * so it can't be Collections.emptySet() or some such thing. 1252 * The JpaPid returned will have resource type populated. 1253 */ 1254 @Override 1255 public Set<JpaPid> loadIncludes( 1256 FhirContext theContext, 1257 EntityManager theEntityManager, 1258 Collection<JpaPid> theMatches, 1259 Collection<Include> theIncludes, 1260 boolean theReverseMode, 1261 DateRangeParam theLastUpdated, 1262 String theSearchIdOrDescription, 1263 RequestDetails theRequest, 1264 Integer theMaxCount) { 1265 SearchBuilderLoadIncludesParameters<JpaPid> parameters = new SearchBuilderLoadIncludesParameters<>(); 1266 parameters.setFhirContext(theContext); 1267 parameters.setEntityManager(theEntityManager); 1268 parameters.setMatches(theMatches); 1269 parameters.setIncludeFilters(theIncludes); 1270 parameters.setReverseMode(theReverseMode); 1271 parameters.setLastUpdated(theLastUpdated); 1272 parameters.setSearchIdOrDescription(theSearchIdOrDescription); 1273 parameters.setRequestDetails(theRequest); 1274 parameters.setMaxCount(theMaxCount); 1275 return loadIncludes(parameters); 1276 } 1277 1278 @Override 1279 public Set<JpaPid> loadIncludes(SearchBuilderLoadIncludesParameters<JpaPid> theParameters) { 1280 Collection<JpaPid> matches = theParameters.getMatches(); 1281 Collection<Include> currentIncludes = theParameters.getIncludeFilters(); 1282 boolean reverseMode = theParameters.isReverseMode(); 1283 EntityManager entityManager = theParameters.getEntityManager(); 1284 Integer maxCount = theParameters.getMaxCount(); 1285 FhirContext fhirContext = theParameters.getFhirContext(); 1286 DateRangeParam lastUpdated = theParameters.getLastUpdated(); 1287 RequestDetails request = theParameters.getRequestDetails(); 1288 String searchIdOrDescription = theParameters.getSearchIdOrDescription(); 1289 List<String> desiredResourceTypes = theParameters.getDesiredResourceTypes(); 1290 boolean hasDesiredResourceTypes = desiredResourceTypes != null && !desiredResourceTypes.isEmpty(); 1291 if (CompositeInterceptorBroadcaster.hasHooks( 1292 Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, theParameters.getRequestDetails())) { 1293 CurrentThreadCaptureQueriesListener.startCapturing(); 1294 } 1295 if (matches.size() == 0) { 1296 return new HashSet<>(); 1297 } 1298 if (currentIncludes == null || currentIncludes.isEmpty()) { 1299 return new HashSet<>(); 1300 } 1301 String searchPidFieldName = reverseMode ? MY_TARGET_RESOURCE_PID : MY_SOURCE_RESOURCE_PID; 1302 String findPidFieldName = reverseMode ? MY_SOURCE_RESOURCE_PID : MY_TARGET_RESOURCE_PID; 1303 String findResourceTypeFieldName = reverseMode ? MY_SOURCE_RESOURCE_TYPE : MY_TARGET_RESOURCE_TYPE; 1304 String findVersionFieldName = null; 1305 if (!reverseMode && myStorageSettings.isRespectVersionsForSearchIncludes()) { 1306 findVersionFieldName = MY_TARGET_RESOURCE_VERSION; 1307 } 1308 1309 List<JpaPid> nextRoundMatches = new ArrayList<>(matches); 1310 HashSet<JpaPid> allAdded = new HashSet<>(); 1311 HashSet<JpaPid> original = new HashSet<>(matches); 1312 ArrayList<Include> includes = new ArrayList<>(currentIncludes); 1313 1314 int roundCounts = 0; 1315 StopWatch w = new StopWatch(); 1316 1317 boolean addedSomeThisRound; 1318 do { 1319 roundCounts++; 1320 1321 HashSet<JpaPid> pidsToInclude = new HashSet<>(); 1322 1323 for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) { 1324 Include nextInclude = iter.next(); 1325 if (nextInclude.isRecurse() == false) { 1326 iter.remove(); 1327 } 1328 1329 // Account for _include=* 1330 boolean matchAll = "*".equals(nextInclude.getValue()); 1331 1332 // Account for _include=[resourceType]:* 1333 String wantResourceType = null; 1334 if (!matchAll) { 1335 if ("*".equals(nextInclude.getParamName())) { 1336 wantResourceType = nextInclude.getParamType(); 1337 matchAll = true; 1338 } 1339 } 1340 1341 if (matchAll) { 1342 StringBuilder sqlBuilder = new StringBuilder(); 1343 sqlBuilder.append("SELECT r.").append(findPidFieldName); 1344 sqlBuilder.append(", r.").append(findResourceTypeFieldName); 1345 if (findVersionFieldName != null) { 1346 sqlBuilder.append(", r.").append(findVersionFieldName); 1347 } 1348 sqlBuilder.append(" FROM ResourceLink r WHERE "); 1349 1350 sqlBuilder.append("r."); 1351 sqlBuilder.append(searchPidFieldName); // (rev mode) target_resource_id | source_resource_id 1352 sqlBuilder.append(" IN (:target_pids)"); 1353 1354 /* 1355 * We need to set the resource type in 2 cases only: 1356 * 1) we are in $everything mode 1357 * (where we only want to fetch specific resource types, regardless of what is 1358 * available to fetch) 1359 * 2) we are doing revincludes 1360 * 1361 * Technically if the request is a qualified star (e.g. _include=Observation:*) we 1362 * should always be checking the source resource type on the resource link. We don't 1363 * actually index that column though by default, so in order to try and be efficient 1364 * we don't actually include it for includes (but we do for revincludes). This is 1365 * because for an include, it doesn't really make sense to include a different 1366 * resource type than the one you are searching on. 1367 */ 1368 if (wantResourceType != null 1369 && (reverseMode || (myParams != null && myParams.getEverythingMode() != null))) { 1370 // because mySourceResourceType is not part of the HFJ_RES_LINK 1371 // index, this might not be the most optimal performance. 1372 // but it is for an $everything operation (and maybe we should update the index) 1373 sqlBuilder.append(" AND r.mySourceResourceType = :want_resource_type"); 1374 } else { 1375 wantResourceType = null; 1376 } 1377 1378 // When calling $everything on a Patient instance, we don't want to recurse into new Patient 1379 // resources 1380 // (e.g. via Provenance, List, or Group) when in an $everything operation 1381 if (myParams != null 1382 && myParams.getEverythingMode() == SearchParameterMap.EverythingModeEnum.PATIENT_INSTANCE) { 1383 sqlBuilder.append(" AND r.myTargetResourceType != 'Patient'"); 1384 sqlBuilder.append(UNDESIRED_RESOURCE_LINKAGES_FOR_EVERYTHING_ON_PATIENT_INSTANCE.stream() 1385 .collect(Collectors.joining("', '", " AND r.mySourceResourceType NOT IN ('", "')"))); 1386 } 1387 if (hasDesiredResourceTypes) { 1388 sqlBuilder.append(" AND r.myTargetResourceType IN (:desired_target_resource_types)"); 1389 } 1390 1391 String sql = sqlBuilder.toString(); 1392 List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize()); 1393 for (Collection<JpaPid> nextPartition : partitions) { 1394 TypedQuery<?> q = entityManager.createQuery(sql, Object[].class); 1395 q.setParameter("target_pids", JpaPid.toLongList(nextPartition)); 1396 if (wantResourceType != null) { 1397 q.setParameter("want_resource_type", wantResourceType); 1398 } 1399 if (maxCount != null) { 1400 q.setMaxResults(maxCount); 1401 } 1402 if (hasDesiredResourceTypes) { 1403 q.setParameter("desired_target_resource_types", desiredResourceTypes); 1404 } 1405 List<?> results = q.getResultList(); 1406 for (Object nextRow : results) { 1407 if (nextRow == null) { 1408 // This can happen if there are outgoing references which are canonical or point to 1409 // other servers 1410 continue; 1411 } 1412 1413 Long version = null; 1414 Long resourceLink = (Long) ((Object[]) nextRow)[0]; 1415 String resourceType = (String) ((Object[]) nextRow)[1]; 1416 if (findVersionFieldName != null) { 1417 version = (Long) ((Object[]) nextRow)[2]; 1418 } 1419 1420 if (resourceLink != null) { 1421 JpaPid pid = 1422 JpaPid.fromIdAndVersionAndResourceType(resourceLink, version, resourceType); 1423 pidsToInclude.add(pid); 1424 } 1425 } 1426 } 1427 } else { 1428 List<String> paths; 1429 1430 // Start replace 1431 RuntimeSearchParam param; 1432 String resType = nextInclude.getParamType(); 1433 if (isBlank(resType)) { 1434 continue; 1435 } 1436 RuntimeResourceDefinition def = fhirContext.getResourceDefinition(resType); 1437 if (def == null) { 1438 ourLog.warn("Unknown resource type in include/revinclude=" + nextInclude.getValue()); 1439 continue; 1440 } 1441 1442 String paramName = nextInclude.getParamName(); 1443 if (isNotBlank(paramName)) { 1444 param = mySearchParamRegistry.getActiveSearchParam(resType, paramName); 1445 } else { 1446 param = null; 1447 } 1448 if (param == null) { 1449 ourLog.warn("Unknown param name in include/revinclude=" + nextInclude.getValue()); 1450 continue; 1451 } 1452 1453 paths = param.getPathsSplitForResourceType(resType); 1454 // end replace 1455 1456 Set<String> targetResourceTypes = computeTargetResourceTypes(nextInclude, param); 1457 1458 for (String nextPath : paths) { 1459 String findPidFieldSqlColumn = findPidFieldName.equals(MY_SOURCE_RESOURCE_PID) 1460 ? "src_resource_id" 1461 : "target_resource_id"; 1462 String fieldsToLoad = "r." + findPidFieldSqlColumn + " AS " + RESOURCE_ID_ALIAS; 1463 if (findVersionFieldName != null) { 1464 fieldsToLoad += ", r.target_resource_version AS " + RESOURCE_VERSION_ALIAS; 1465 } 1466 1467 // Query for includes lookup has 2 cases 1468 // Case 1: Where target_resource_id is available in hfj_res_link table for local references 1469 // Case 2: Where target_resource_id is null in hfj_res_link table and referred by a canonical 1470 // url in target_resource_url 1471 1472 // Case 1: 1473 Map<String, Object> localReferenceQueryParams = new HashMap<>(); 1474 1475 String searchPidFieldSqlColumn = searchPidFieldName.equals(MY_TARGET_RESOURCE_PID) 1476 ? "target_resource_id" 1477 : "src_resource_id"; 1478 StringBuilder localReferenceQuery = 1479 new StringBuilder("SELECT " + fieldsToLoad + " FROM hfj_res_link r " 1480 + " WHERE r.src_path = :src_path AND " 1481 + " r.target_resource_id IS NOT NULL AND " 1482 + " r." 1483 + searchPidFieldSqlColumn + " IN (:target_pids) "); 1484 localReferenceQueryParams.put("src_path", nextPath); 1485 // we loop over target_pids later. 1486 if (targetResourceTypes != null) { 1487 if (targetResourceTypes.size() == 1) { 1488 localReferenceQuery.append(" AND r.target_resource_type = :target_resource_type "); 1489 localReferenceQueryParams.put( 1490 "target_resource_type", 1491 targetResourceTypes.iterator().next()); 1492 } else { 1493 localReferenceQuery.append(" AND r.target_resource_type in (:target_resource_types) "); 1494 localReferenceQueryParams.put("target_resource_types", targetResourceTypes); 1495 } 1496 } 1497 1498 // Case 2: 1499 Pair<String, Map<String, Object>> canonicalQuery = buildCanonicalUrlQuery( 1500 findVersionFieldName, searchPidFieldSqlColumn, targetResourceTypes); 1501 1502 // @formatter:on 1503 1504 String sql = localReferenceQuery + " UNION " + canonicalQuery.getLeft(); 1505 1506 List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize()); 1507 for (Collection<JpaPid> nextPartition : partitions) { 1508 Query q = entityManager.createNativeQuery(sql, Tuple.class); 1509 q.setParameter("target_pids", JpaPid.toLongList(nextPartition)); 1510 localReferenceQueryParams.forEach(q::setParameter); 1511 canonicalQuery.getRight().forEach(q::setParameter); 1512 1513 if (maxCount != null) { 1514 q.setMaxResults(maxCount); 1515 } 1516 @SuppressWarnings("unchecked") 1517 List<Tuple> results = q.getResultList(); 1518 for (Tuple result : results) { 1519 if (result != null) { 1520 Long resourceId = 1521 NumberUtils.createLong(String.valueOf(result.get(RESOURCE_ID_ALIAS))); 1522 Long resourceVersion = null; 1523 if (findVersionFieldName != null && result.get(RESOURCE_VERSION_ALIAS) != null) { 1524 resourceVersion = NumberUtils.createLong( 1525 String.valueOf(result.get(RESOURCE_VERSION_ALIAS))); 1526 } 1527 pidsToInclude.add(JpaPid.fromIdAndVersion(resourceId, resourceVersion)); 1528 } 1529 } 1530 } 1531 } 1532 } 1533 } 1534 1535 nextRoundMatches.clear(); 1536 for (JpaPid next : pidsToInclude) { 1537 if (!original.contains(next) && !allAdded.contains(next)) { 1538 nextRoundMatches.add(next); 1539 } 1540 } 1541 1542 addedSomeThisRound = allAdded.addAll(pidsToInclude); 1543 1544 if (maxCount != null && allAdded.size() >= maxCount) { 1545 break; 1546 } 1547 1548 } while (!includes.isEmpty() && !nextRoundMatches.isEmpty() && addedSomeThisRound); 1549 1550 allAdded.removeAll(original); 1551 1552 ourLog.info( 1553 "Loaded {} {} in {} rounds and {} ms for search {}", 1554 allAdded.size(), 1555 reverseMode ? "_revincludes" : "_includes", 1556 roundCounts, 1557 w.getMillisAndRestart(), 1558 searchIdOrDescription); 1559 1560 if (CompositeInterceptorBroadcaster.hasHooks( 1561 Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, request)) { 1562 callRawSqlHookWithCurrentThreadQueries(request); 1563 } 1564 // Interceptor call: STORAGE_PREACCESS_RESOURCES 1565 // This can be used to remove results from the search result details before 1566 // the user has a chance to know that they were in the results 1567 if (!allAdded.isEmpty()) { 1568 1569 if (CompositeInterceptorBroadcaster.hasHooks( 1570 Pointcut.STORAGE_PREACCESS_RESOURCES, myInterceptorBroadcaster, request)) { 1571 List<JpaPid> includedPidList = new ArrayList<>(allAdded); 1572 JpaPreResourceAccessDetails accessDetails = 1573 new JpaPreResourceAccessDetails(includedPidList, () -> this); 1574 HookParams params = new HookParams() 1575 .add(IPreResourceAccessDetails.class, accessDetails) 1576 .add(RequestDetails.class, request) 1577 .addIfMatchesType(ServletRequestDetails.class, request); 1578 CompositeInterceptorBroadcaster.doCallHooks( 1579 myInterceptorBroadcaster, request, Pointcut.STORAGE_PREACCESS_RESOURCES, params); 1580 1581 for (int i = includedPidList.size() - 1; i >= 0; i--) { 1582 if (accessDetails.isDontReturnResourceAtIndex(i)) { 1583 JpaPid value = includedPidList.remove(i); 1584 if (value != null) { 1585 allAdded.remove(value); 1586 } 1587 } 1588 } 1589 } 1590 } 1591 1592 return allAdded; 1593 } 1594 1595 /** 1596 * Given a 1597 * @param request 1598 */ 1599 private void callRawSqlHookWithCurrentThreadQueries(RequestDetails request) { 1600 SqlQueryList capturedQueries = CurrentThreadCaptureQueriesListener.getCurrentQueueAndStopCapturing(); 1601 HookParams params = new HookParams() 1602 .add(RequestDetails.class, request) 1603 .addIfMatchesType(ServletRequestDetails.class, request) 1604 .add(SqlQueryList.class, capturedQueries); 1605 CompositeInterceptorBroadcaster.doCallHooks( 1606 myInterceptorBroadcaster, request, Pointcut.JPA_PERFTRACE_RAW_SQL, params); 1607 } 1608 1609 @Nullable 1610 private static Set<String> computeTargetResourceTypes(Include nextInclude, RuntimeSearchParam param) { 1611 String targetResourceType = defaultString(nextInclude.getParamTargetType(), null); 1612 boolean haveTargetTypesDefinedByParam = param.hasTargets(); 1613 Set<String> targetResourceTypes; 1614 if (targetResourceType != null) { 1615 targetResourceTypes = Set.of(targetResourceType); 1616 } else if (haveTargetTypesDefinedByParam) { 1617 targetResourceTypes = param.getTargets(); 1618 } else { 1619 // all types! 1620 targetResourceTypes = null; 1621 } 1622 return targetResourceTypes; 1623 } 1624 1625 @Nonnull 1626 private Pair<String, Map<String, Object>> buildCanonicalUrlQuery( 1627 String theVersionFieldName, String thePidFieldSqlColumn, Set<String> theTargetResourceTypes) { 1628 String fieldsToLoadFromSpidxUriTable = "rUri.res_id"; 1629 if (theVersionFieldName != null) { 1630 // canonical-uri references aren't versioned, but we need to match the column count for the UNION 1631 fieldsToLoadFromSpidxUriTable += ", NULL"; 1632 } 1633 // The logical join will be by hfj_spidx_uri on sp_name='uri' and sp_uri=target_resource_url. 1634 // But sp_name isn't indexed, so we use hash_identity instead. 1635 if (theTargetResourceTypes == null) { 1636 // hash_identity includes the resource type. So a null wildcard must be replaced with a list of all types. 1637 theTargetResourceTypes = myDaoRegistry.getRegisteredDaoTypes(); 1638 } 1639 assert !theTargetResourceTypes.isEmpty(); 1640 1641 Set<Long> identityHashesForTypes = theTargetResourceTypes.stream() 1642 .map(type -> BaseResourceIndexedSearchParam.calculateHashIdentity( 1643 myPartitionSettings, myRequestPartitionId, type, "url")) 1644 .collect(Collectors.toSet()); 1645 1646 Map<String, Object> canonicalUriQueryParams = new HashMap<>(); 1647 StringBuilder canonicalUrlQuery = new StringBuilder( 1648 "SELECT " + fieldsToLoadFromSpidxUriTable + " FROM hfj_res_link r " + " JOIN hfj_spidx_uri rUri ON ( "); 1649 // join on hash_identity and sp_uri - indexed in IDX_SP_URI_HASH_IDENTITY_V2 1650 if (theTargetResourceTypes.size() == 1) { 1651 canonicalUrlQuery.append(" rUri.hash_identity = :uri_identity_hash "); 1652 canonicalUriQueryParams.put( 1653 "uri_identity_hash", identityHashesForTypes.iterator().next()); 1654 } else { 1655 canonicalUrlQuery.append(" rUri.hash_identity in (:uri_identity_hashes) "); 1656 canonicalUriQueryParams.put("uri_identity_hashes", identityHashesForTypes); 1657 } 1658 1659 canonicalUrlQuery.append(" AND r.target_resource_url = rUri.sp_uri )" + " WHERE r.src_path = :src_path AND " 1660 + " r.target_resource_id IS NULL AND " 1661 + " r." 1662 + thePidFieldSqlColumn + " IN (:target_pids) "); 1663 return Pair.of(canonicalUrlQuery.toString(), canonicalUriQueryParams); 1664 } 1665 1666 private List<Collection<JpaPid>> partition(Collection<JpaPid> theNextRoundMatches, int theMaxLoad) { 1667 if (theNextRoundMatches.size() <= theMaxLoad) { 1668 return Collections.singletonList(theNextRoundMatches); 1669 } else { 1670 1671 List<Collection<JpaPid>> retVal = new ArrayList<>(); 1672 Collection<JpaPid> current = null; 1673 for (JpaPid next : theNextRoundMatches) { 1674 if (current == null) { 1675 current = new ArrayList<>(theMaxLoad); 1676 retVal.add(current); 1677 } 1678 1679 current.add(next); 1680 1681 if (current.size() >= theMaxLoad) { 1682 current = null; 1683 } 1684 } 1685 1686 return retVal; 1687 } 1688 } 1689 1690 private void attemptComboUniqueSpProcessing( 1691 QueryStack theQueryStack3, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) { 1692 RuntimeSearchParam comboParam = null; 1693 List<String> comboParamNames = null; 1694 List<RuntimeSearchParam> exactMatchParams = 1695 mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet()); 1696 if (exactMatchParams.size() > 0) { 1697 comboParam = exactMatchParams.get(0); 1698 comboParamNames = new ArrayList<>(theParams.keySet()); 1699 } 1700 1701 if (comboParam == null) { 1702 List<RuntimeSearchParam> candidateComboParams = 1703 mySearchParamRegistry.getActiveComboSearchParams(myResourceName); 1704 for (RuntimeSearchParam nextCandidate : candidateComboParams) { 1705 List<String> nextCandidateParamNames = 1706 JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, nextCandidate).stream() 1707 .map(t -> t.getName()) 1708 .collect(Collectors.toList()); 1709 if (theParams.keySet().containsAll(nextCandidateParamNames)) { 1710 comboParam = nextCandidate; 1711 comboParamNames = nextCandidateParamNames; 1712 break; 1713 } 1714 } 1715 } 1716 1717 if (comboParam != null) { 1718 // Since we're going to remove elements below 1719 theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList)); 1720 1721 StringBuilder sb = new StringBuilder(); 1722 sb.append(myResourceName); 1723 sb.append("?"); 1724 1725 boolean first = true; 1726 1727 Collections.sort(comboParamNames); 1728 for (String nextParamName : comboParamNames) { 1729 List<List<IQueryParameterType>> nextValues = theParams.get(nextParamName); 1730 1731 // TODO Hack to fix weird IOOB on the next stanza until James comes back and makes sense of this. 1732 if (nextValues.isEmpty()) { 1733 ourLog.error( 1734 "query parameter {} is unexpectedly empty. Encountered while considering {} index for {}", 1735 nextParamName, 1736 comboParam.getName(), 1737 theRequest.getCompleteUrl()); 1738 sb = null; 1739 break; 1740 } 1741 1742 if (nextValues.get(0).size() != 1) { 1743 sb = null; 1744 break; 1745 } 1746 1747 // Reference params are only eligible for using a composite index if they 1748 // are qualified 1749 RuntimeSearchParam nextParamDef = 1750 mySearchParamRegistry.getActiveSearchParam(myResourceName, nextParamName); 1751 if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.REFERENCE) { 1752 ReferenceParam param = (ReferenceParam) nextValues.get(0).get(0); 1753 if (isBlank(param.getResourceType())) { 1754 sb = null; 1755 break; 1756 } 1757 } 1758 1759 List<? extends IQueryParameterType> nextAnd = nextValues.remove(0); 1760 IQueryParameterType nextOr = nextAnd.remove(0); 1761 String nextOrValue = nextOr.getValueAsQueryToken(myContext); 1762 1763 if (comboParam.getComboSearchParamType() == ComboSearchParamType.NON_UNIQUE) { 1764 if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.STRING) { 1765 nextOrValue = StringUtil.normalizeStringForSearchIndexing(nextOrValue); 1766 } 1767 } 1768 1769 if (first) { 1770 first = false; 1771 } else { 1772 sb.append('&'); 1773 } 1774 1775 nextParamName = UrlUtil.escapeUrlParam(nextParamName); 1776 nextOrValue = UrlUtil.escapeUrlParam(nextOrValue); 1777 1778 sb.append(nextParamName).append('=').append(nextOrValue); 1779 } 1780 1781 if (sb != null) { 1782 String indexString = sb.toString(); 1783 ourLog.debug( 1784 "Checking for {} combo index for query: {}", comboParam.getComboSearchParamType(), indexString); 1785 1786 // Interceptor broadcast: JPA_PERFTRACE_INFO 1787 StorageProcessingMessage msg = new StorageProcessingMessage() 1788 .setMessage("Using " + comboParam.getComboSearchParamType() + " index for query for search: " 1789 + indexString); 1790 HookParams params = new HookParams() 1791 .add(RequestDetails.class, theRequest) 1792 .addIfMatchesType(ServletRequestDetails.class, theRequest) 1793 .add(StorageProcessingMessage.class, msg); 1794 CompositeInterceptorBroadcaster.doCallHooks( 1795 myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INFO, params); 1796 1797 switch (comboParam.getComboSearchParamType()) { 1798 case UNIQUE: 1799 theQueryStack3.addPredicateCompositeUnique(indexString, myRequestPartitionId); 1800 break; 1801 case NON_UNIQUE: 1802 theQueryStack3.addPredicateCompositeNonUnique(indexString, myRequestPartitionId); 1803 break; 1804 } 1805 1806 // Remove any empty parameters remaining after this 1807 theParams.clean(); 1808 } 1809 } 1810 } 1811 1812 private <T> void ensureSubListsAreWritable(List<List<T>> theListOfLists) { 1813 for (int i = 0; i < theListOfLists.size(); i++) { 1814 List<T> oldSubList = theListOfLists.get(i); 1815 if (!(oldSubList instanceof ArrayList)) { 1816 List<T> newSubList = new ArrayList<>(oldSubList); 1817 theListOfLists.set(i, newSubList); 1818 } 1819 } 1820 } 1821 1822 @Override 1823 public void setFetchSize(int theFetchSize) { 1824 myFetchSize = theFetchSize; 1825 } 1826 1827 public SearchParameterMap getParams() { 1828 return myParams; 1829 } 1830 1831 public CriteriaBuilder getBuilder() { 1832 return myCriteriaBuilder; 1833 } 1834 1835 public Class<? extends IBaseResource> getResourceType() { 1836 return myResourceType; 1837 } 1838 1839 public String getResourceName() { 1840 return myResourceName; 1841 } 1842 1843 /** 1844 * IncludesIterator, used to recursively fetch resources from the provided list of PIDs 1845 */ 1846 public class IncludesIterator extends BaseIterator<JpaPid> implements Iterator<JpaPid> { 1847 1848 private final RequestDetails myRequest; 1849 private final Set<JpaPid> myCurrentPids; 1850 private Iterator<JpaPid> myCurrentIterator; 1851 private JpaPid myNext; 1852 1853 IncludesIterator(Set<JpaPid> thePidSet, RequestDetails theRequest) { 1854 myCurrentPids = new HashSet<>(thePidSet); 1855 myCurrentIterator = null; 1856 myRequest = theRequest; 1857 } 1858 1859 private void fetchNext() { 1860 while (myNext == null) { 1861 1862 if (myCurrentIterator == null) { 1863 Set<Include> includes = new HashSet<>(); 1864 if (myParams.containsKey(Constants.PARAM_TYPE)) { 1865 for (List<IQueryParameterType> typeList : myParams.get(Constants.PARAM_TYPE)) { 1866 for (IQueryParameterType type : typeList) { 1867 String queryString = ParameterUtil.unescape(type.getValueAsQueryToken(myContext)); 1868 for (String resourceType : queryString.split(",")) { 1869 String rt = resourceType.trim(); 1870 if (isNotBlank(rt)) { 1871 includes.add(new Include(rt + ":*", true)); 1872 } 1873 } 1874 } 1875 } 1876 } 1877 if (includes.isEmpty()) { 1878 includes.add(new Include("*", true)); 1879 } 1880 Set<JpaPid> newPids = loadIncludes( 1881 myContext, 1882 myEntityManager, 1883 myCurrentPids, 1884 includes, 1885 false, 1886 getParams().getLastUpdated(), 1887 mySearchUuid, 1888 myRequest, 1889 null); 1890 myCurrentIterator = newPids.iterator(); 1891 } 1892 1893 if (myCurrentIterator.hasNext()) { 1894 myNext = myCurrentIterator.next(); 1895 } else { 1896 myNext = NO_MORE; 1897 } 1898 } 1899 } 1900 1901 @Override 1902 public boolean hasNext() { 1903 fetchNext(); 1904 return !NO_MORE.equals(myNext); 1905 } 1906 1907 @Override 1908 public JpaPid next() { 1909 fetchNext(); 1910 JpaPid retVal = myNext; 1911 myNext = null; 1912 return retVal; 1913 } 1914 } 1915 1916 /** 1917 * Basic Query iterator, used to fetch the results of a query. 1918 */ 1919 private final class QueryIterator extends BaseIterator<JpaPid> implements IResultIterator<JpaPid> { 1920 1921 private final SearchRuntimeDetails mySearchRuntimeDetails; 1922 private final RequestDetails myRequest; 1923 private final boolean myHaveRawSqlHooks; 1924 private final boolean myHavePerfTraceFoundIdHook; 1925 private final SortSpec mySort; 1926 private final Integer myOffset; 1927 private boolean myFirst = true; 1928 private IncludesIterator myIncludesIterator; 1929 /** 1930 * The next JpaPid value of the next result in this query. 1931 * Will not be null if fetched using getNext() 1932 */ 1933 private JpaPid myNext; 1934 /** 1935 * The current query result iterator running sql and supplying PIDs 1936 * @see #myQueryList 1937 */ 1938 private ISearchQueryExecutor myResultsIterator; 1939 1940 private boolean myFetchIncludesForEverythingOperation; 1941 /** 1942 * The count of resources skipped because they were seen in earlier results 1943 */ 1944 private int mySkipCount = 0; 1945 /** 1946 * The count of resources that are new in this search 1947 * (ie, not cached in previous searches) 1948 */ 1949 private int myNonSkipCount = 0; 1950 1951 /** 1952 * The list of queries to use to find all results. 1953 * Normal JPA queries will normally have a single entry. 1954 * Queries that involve Hibernate Search/Elastisearch may have 1955 * multiple queries because of chunking. 1956 * The $everything operation also jams some extra results in. 1957 */ 1958 private List<ISearchQueryExecutor> myQueryList = new ArrayList<>(); 1959 1960 private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) { 1961 mySearchRuntimeDetails = theSearchRuntimeDetails; 1962 mySort = myParams.getSort(); 1963 myOffset = myParams.getOffset(); 1964 myRequest = theRequest; 1965 1966 // everything requires fetching recursively all related resources 1967 if (myParams.getEverythingMode() != null) { 1968 myFetchIncludesForEverythingOperation = true; 1969 } 1970 1971 myHavePerfTraceFoundIdHook = CompositeInterceptorBroadcaster.hasHooks( 1972 Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, myInterceptorBroadcaster, myRequest); 1973 myHaveRawSqlHooks = CompositeInterceptorBroadcaster.hasHooks( 1974 Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, myRequest); 1975 } 1976 1977 private void fetchNext() { 1978 try { 1979 if (myHaveRawSqlHooks) { 1980 CurrentThreadCaptureQueriesListener.startCapturing(); 1981 } 1982 1983 // If we don't have a query yet, create one 1984 if (myResultsIterator == null) { 1985 if (myMaxResultsToFetch == null) { 1986 if (myParams.getLoadSynchronousUpTo() != null) { 1987 myMaxResultsToFetch = myParams.getLoadSynchronousUpTo(); 1988 } else if (myParams.getOffset() != null && myParams.getCount() != null) { 1989 myMaxResultsToFetch = myParams.getCount(); 1990 } else { 1991 myMaxResultsToFetch = myStorageSettings.getFetchSizeDefaultMaximum(); 1992 } 1993 } 1994 1995 /* 1996 * assigns the results iterator 1997 * and populates the myQueryList. 1998 */ 1999 initializeIteratorQuery(myOffset, myMaxResultsToFetch); 2000 } 2001 2002 if (myNext == null) { 2003 // no next means we need a new query (if one is available) 2004 while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) { 2005 // Update iterator with next chunk if necessary. 2006 if (!myResultsIterator.hasNext()) { 2007 retrieveNextIteratorQuery(); 2008 2009 // if our new results iterator is also empty 2010 // we're done here 2011 if (!myResultsIterator.hasNext()) { 2012 break; 2013 } 2014 } 2015 2016 Long nextLong = myResultsIterator.next(); 2017 if (myHavePerfTraceFoundIdHook) { 2018 HookParams params = new HookParams() 2019 .add(Integer.class, System.identityHashCode(this)) 2020 .add(Object.class, nextLong); 2021 CompositeInterceptorBroadcaster.doCallHooks( 2022 myInterceptorBroadcaster, 2023 myRequest, 2024 Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, 2025 params); 2026 } 2027 2028 if (nextLong != null) { 2029 JpaPid next = JpaPid.fromId(nextLong); 2030 if (myPidSet.add(next)) { 2031 myNext = next; 2032 myNonSkipCount++; 2033 break; 2034 } else { 2035 mySkipCount++; 2036 } 2037 } 2038 2039 if (!myResultsIterator.hasNext()) { 2040 if (myMaxResultsToFetch != null && (mySkipCount + myNonSkipCount == myMaxResultsToFetch)) { 2041 if (mySkipCount > 0 && myNonSkipCount == 0) { 2042 2043 sendProcessingMsgAndFirePerformanceHook(); 2044 2045 myMaxResultsToFetch += 1000; 2046 initializeIteratorQuery(myOffset, myMaxResultsToFetch); 2047 } 2048 } 2049 } 2050 } 2051 } 2052 2053 if (myNext == null) { 2054 // if we got here, it means the current JpaPid has already been processed, 2055 // and we will decide (here) if we need to fetch related resources recursively 2056 if (myFetchIncludesForEverythingOperation) { 2057 myIncludesIterator = new IncludesIterator(myPidSet, myRequest); 2058 myFetchIncludesForEverythingOperation = false; 2059 } 2060 if (myIncludesIterator != null) { 2061 while (myIncludesIterator.hasNext()) { 2062 JpaPid next = myIncludesIterator.next(); 2063 if (next != null) 2064 if (myPidSet.add(next)) { 2065 myNext = next; 2066 break; 2067 } 2068 } 2069 if (myNext == null) { 2070 myNext = NO_MORE; 2071 } 2072 } else { 2073 myNext = NO_MORE; 2074 } 2075 } 2076 2077 mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size()); 2078 2079 } finally { 2080 // search finished - fire hooks 2081 if (myHaveRawSqlHooks) { 2082 callRawSqlHookWithCurrentThreadQueries(myRequest); 2083 } 2084 } 2085 2086 if (myFirst) { 2087 HookParams params = new HookParams() 2088 .add(RequestDetails.class, myRequest) 2089 .addIfMatchesType(ServletRequestDetails.class, myRequest) 2090 .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); 2091 CompositeInterceptorBroadcaster.doCallHooks( 2092 myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FIRST_RESULT_LOADED, params); 2093 myFirst = false; 2094 } 2095 2096 if (NO_MORE.equals(myNext)) { 2097 HookParams params = new HookParams() 2098 .add(RequestDetails.class, myRequest) 2099 .addIfMatchesType(ServletRequestDetails.class, myRequest) 2100 .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); 2101 CompositeInterceptorBroadcaster.doCallHooks( 2102 myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_SELECT_COMPLETE, params); 2103 } 2104 } 2105 2106 private void sendProcessingMsgAndFirePerformanceHook() { 2107 StorageProcessingMessage message = new StorageProcessingMessage(); 2108 String msg = "Pass completed with no matching results seeking rows " 2109 + myPidSet.size() + "-" + mySkipCount 2110 + ". This indicates an inefficient query! Retrying with new max count of " 2111 + myMaxResultsToFetch; 2112 ourLog.warn(msg); 2113 message.setMessage(msg); 2114 HookParams params = new HookParams() 2115 .add(RequestDetails.class, myRequest) 2116 .addIfMatchesType(ServletRequestDetails.class, myRequest) 2117 .add(StorageProcessingMessage.class, message); 2118 CompositeInterceptorBroadcaster.doCallHooks( 2119 myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_WARNING, params); 2120 } 2121 2122 private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToFetch) { 2123 if (myQueryList.isEmpty()) { 2124 // Capture times for Lucene/Elasticsearch queries as well 2125 mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); 2126 myQueryList = createQuery( 2127 myParams, mySort, theOffset, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails); 2128 } 2129 2130 mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); 2131 2132 retrieveNextIteratorQuery(); 2133 2134 mySkipCount = 0; 2135 myNonSkipCount = 0; 2136 } 2137 2138 private void retrieveNextIteratorQuery() { 2139 close(); 2140 if (myQueryList != null && myQueryList.size() > 0) { 2141 myResultsIterator = myQueryList.remove(0); 2142 myHasNextIteratorQuery = true; 2143 } else { 2144 myResultsIterator = SearchQueryExecutor.emptyExecutor(); 2145 myHasNextIteratorQuery = false; 2146 } 2147 } 2148 2149 @Override 2150 public boolean hasNext() { 2151 if (myNext == null) { 2152 fetchNext(); 2153 } 2154 return !NO_MORE.equals(myNext); 2155 } 2156 2157 @Override 2158 public JpaPid next() { 2159 fetchNext(); 2160 JpaPid retVal = myNext; 2161 myNext = null; 2162 Validate.isTrue(!NO_MORE.equals(retVal), "No more elements"); 2163 return retVal; 2164 } 2165 2166 @Override 2167 public int getSkippedCount() { 2168 return mySkipCount; 2169 } 2170 2171 @Override 2172 public int getNonSkippedCount() { 2173 return myNonSkipCount; 2174 } 2175 2176 @Override 2177 public Collection<JpaPid> getNextResultBatch(long theBatchSize) { 2178 Collection<JpaPid> batch = new ArrayList<>(); 2179 while (this.hasNext() && batch.size() < theBatchSize) { 2180 batch.add(this.next()); 2181 } 2182 return batch; 2183 } 2184 2185 @Override 2186 public void close() { 2187 if (myResultsIterator != null) { 2188 myResultsIterator.close(); 2189 } 2190 myResultsIterator = null; 2191 } 2192 } 2193 2194 public static int getMaximumPageSize() { 2195 if (myUseMaxPageSize50ForTest) { 2196 return MAXIMUM_PAGE_SIZE_FOR_TESTING; 2197 } else { 2198 return MAXIMUM_PAGE_SIZE; 2199 } 2200 } 2201 2202 public static void setMaxPageSize50ForTest(boolean theIsTest) { 2203 myUseMaxPageSize50ForTest = theIsTest; 2204 } 2205}