
001package ca.uhn.fhir.jpa.search.builder; 002 003/* 004 * #%L 005 * HAPI FHIR JPA Server 006 * %% 007 * Copyright (C) 2014 - 2023 Smile CDR, Inc. 008 * %% 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 * #L% 021 */ 022 023import ca.uhn.fhir.context.ComboSearchParamType; 024import ca.uhn.fhir.context.FhirContext; 025import ca.uhn.fhir.context.FhirVersionEnum; 026import ca.uhn.fhir.context.RuntimeResourceDefinition; 027import ca.uhn.fhir.context.RuntimeSearchParam; 028import ca.uhn.fhir.i18n.Msg; 029import ca.uhn.fhir.interceptor.api.HookParams; 030import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; 031import ca.uhn.fhir.interceptor.api.Pointcut; 032import ca.uhn.fhir.interceptor.model.RequestPartitionId; 033import ca.uhn.fhir.jpa.api.config.DaoConfig; 034import ca.uhn.fhir.jpa.api.dao.DaoRegistry; 035import ca.uhn.fhir.jpa.api.dao.IDao; 036import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; 037import ca.uhn.fhir.jpa.api.svc.IIdHelperService; 038import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean; 039import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider; 040import ca.uhn.fhir.jpa.dao.BaseStorageDao; 041import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc; 042import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser; 043import ca.uhn.fhir.jpa.dao.IResultIterator; 044import ca.uhn.fhir.jpa.dao.ISearchBuilder; 045import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao; 046import ca.uhn.fhir.jpa.dao.data.IResourceTagDao; 047import ca.uhn.fhir.jpa.dao.search.ResourceNotFoundInIndexException; 048import ca.uhn.fhir.jpa.entity.ResourceSearchView; 049import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails; 050import ca.uhn.fhir.jpa.model.config.PartitionSettings; 051import ca.uhn.fhir.jpa.model.dao.JpaPid; 052import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity; 053import ca.uhn.fhir.jpa.model.entity.ModelConfig; 054import ca.uhn.fhir.jpa.model.entity.ResourceTag; 055import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails; 056import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; 057import ca.uhn.fhir.jpa.search.SearchConstants; 058import ca.uhn.fhir.jpa.search.builder.sql.GeneratedSql; 059import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder; 060import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryExecutor; 061import ca.uhn.fhir.jpa.search.builder.sql.SqlObjectFactory; 062import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc; 063import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; 064import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper; 065import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil; 066import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper; 067import ca.uhn.fhir.jpa.util.BaseIterator; 068import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener; 069import ca.uhn.fhir.jpa.util.QueryChunker; 070import ca.uhn.fhir.jpa.util.QueryParameterUtils; 071import ca.uhn.fhir.jpa.util.SqlQueryList; 072import ca.uhn.fhir.model.api.IQueryParameterType; 073import ca.uhn.fhir.model.api.IResource; 074import ca.uhn.fhir.model.api.Include; 075import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum; 076import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum; 077import ca.uhn.fhir.rest.api.Constants; 078import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum; 079import ca.uhn.fhir.rest.api.SearchContainedModeEnum; 080import ca.uhn.fhir.rest.api.SortOrderEnum; 081import ca.uhn.fhir.rest.api.SortSpec; 082import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails; 083import ca.uhn.fhir.rest.api.server.RequestDetails; 084import ca.uhn.fhir.rest.param.DateRangeParam; 085import ca.uhn.fhir.rest.param.ReferenceParam; 086import ca.uhn.fhir.rest.param.StringParam; 087import ca.uhn.fhir.rest.param.TokenParam; 088import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 089import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; 090import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; 091import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; 092import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; 093import ca.uhn.fhir.util.StopWatch; 094import ca.uhn.fhir.util.StringUtil; 095import ca.uhn.fhir.util.UrlUtil; 096import com.google.common.collect.Streams; 097import com.healthmarketscience.sqlbuilder.Condition; 098import org.apache.commons.lang3.Validate; 099import org.apache.commons.lang3.math.NumberUtils; 100import org.hl7.fhir.instance.model.api.IAnyResource; 101import org.hl7.fhir.instance.model.api.IBaseResource; 102import org.slf4j.Logger; 103import org.slf4j.LoggerFactory; 104import org.springframework.beans.factory.annotation.Autowired; 105import org.springframework.jdbc.core.JdbcTemplate; 106import org.springframework.jdbc.core.SingleColumnRowMapper; 107import org.springframework.transaction.support.TransactionSynchronizationManager; 108 109import javax.annotation.Nonnull; 110import javax.persistence.EntityManager; 111import javax.persistence.PersistenceContext; 112import javax.persistence.PersistenceContextType; 113import javax.persistence.Query; 114import javax.persistence.Tuple; 115import javax.persistence.TypedQuery; 116import javax.persistence.criteria.CriteriaBuilder; 117import java.util.ArrayList; 118import java.util.Collection; 119import java.util.Collections; 120import java.util.HashMap; 121import java.util.HashSet; 122import java.util.Iterator; 123import java.util.List; 124import java.util.Map; 125import java.util.Objects; 126import java.util.Optional; 127import java.util.Set; 128import java.util.stream.Collectors; 129 130import static org.apache.commons.lang3.StringUtils.defaultString; 131import static org.apache.commons.lang3.StringUtils.isBlank; 132import static org.apache.commons.lang3.StringUtils.isNotBlank; 133 134/** 135 * The SearchBuilder is responsible for actually forming the SQL query that handles 136 * searches for resources 137 */ 138public class SearchBuilder implements ISearchBuilder<JpaPid> { 139 140 /** 141 * See loadResourcesByPid 142 * for an explanation of why we use the constant 800 143 */ 144 // NB: keep public 145 @Deprecated 146 public static final int MAXIMUM_PAGE_SIZE = SearchConstants.MAX_PAGE_SIZE; 147 public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50; 148 public static final String RESOURCE_ID_ALIAS = "resource_id"; 149 public static final String RESOURCE_VERSION_ALIAS = "resource_version"; 150 private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class); 151 private static final JpaPid NO_MORE = JpaPid.fromId(-1L); 152 private static final String MY_TARGET_RESOURCE_PID = "myTargetResourcePid"; 153 private static final String MY_SOURCE_RESOURCE_PID = "mySourceResourcePid"; 154 private static final String MY_TARGET_RESOURCE_TYPE = "myTargetResourceType"; 155 private static final String MY_SOURCE_RESOURCE_TYPE = "mySourceResourceType"; 156 private static final String MY_TARGET_RESOURCE_VERSION = "myTargetResourceVersion"; 157 public static boolean myUseMaxPageSize50ForTest = false; 158 protected final IInterceptorBroadcaster myInterceptorBroadcaster; 159 protected final IResourceTagDao myResourceTagDao; 160 private final String myResourceName; 161 private final Class<? extends IBaseResource> myResourceType; 162 private final HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory; 163 private final SqlObjectFactory mySqlBuilderFactory; 164 private final HibernatePropertiesProvider myDialectProvider; 165 private final ModelConfig myModelConfig; 166 private final ISearchParamRegistry mySearchParamRegistry; 167 private final PartitionSettings myPartitionSettings; 168 private final DaoRegistry myDaoRegistry; 169 private final IResourceSearchViewDao myResourceSearchViewDao; 170 private final FhirContext myContext; 171 private final IIdHelperService<JpaPid> myIdHelperService; 172 private final DaoConfig myDaoConfig; 173 private final IDao myCallingDao; 174 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 175 protected EntityManager myEntityManager; 176 private List<JpaPid> myAlsoIncludePids; 177 private CriteriaBuilder myCriteriaBuilder; 178 private SearchParameterMap myParams; 179 private String mySearchUuid; 180 private int myFetchSize; 181 private Integer myMaxResultsToFetch; 182 private Set<JpaPid> myPidSet; 183 private boolean myHasNextIteratorQuery = false; 184 private RequestPartitionId myRequestPartitionId; 185 @Autowired(required = false) 186 private IFulltextSearchSvc myFulltextSearchSvc; 187 @Autowired(required = false) 188 private IElasticsearchSvc myIElasticsearchSvc; 189 @Autowired 190 private IJpaStorageResourceParser myJpaStorageResourceParser; 191 192 /** 193 * Constructor 194 */ 195 public SearchBuilder( 196 IDao theDao, 197 String theResourceName, 198 DaoConfig theDaoConfig, 199 HapiFhirLocalContainerEntityManagerFactoryBean theEntityManagerFactory, 200 SqlObjectFactory theSqlBuilderFactory, 201 HibernatePropertiesProvider theDialectProvider, 202 ModelConfig theModelConfig, 203 ISearchParamRegistry theSearchParamRegistry, 204 PartitionSettings thePartitionSettings, 205 IInterceptorBroadcaster theInterceptorBroadcaster, 206 IResourceTagDao theResourceTagDao, 207 DaoRegistry theDaoRegistry, 208 IResourceSearchViewDao theResourceSearchViewDao, 209 FhirContext theContext, 210 IIdHelperService theIdHelperService, 211 Class<? extends IBaseResource> theResourceType 212 ) { 213 myCallingDao = theDao; 214 myResourceName = theResourceName; 215 myResourceType = theResourceType; 216 myDaoConfig = theDaoConfig; 217 218 myEntityManagerFactory = theEntityManagerFactory; 219 mySqlBuilderFactory = theSqlBuilderFactory; 220 myDialectProvider = theDialectProvider; 221 myModelConfig = theModelConfig; 222 mySearchParamRegistry = theSearchParamRegistry; 223 myPartitionSettings = thePartitionSettings; 224 myInterceptorBroadcaster = theInterceptorBroadcaster; 225 myResourceTagDao = theResourceTagDao; 226 myDaoRegistry = theDaoRegistry; 227 myResourceSearchViewDao = theResourceSearchViewDao; 228 myContext = theContext; 229 myIdHelperService = theIdHelperService; 230 } 231 232 @Override 233 public void setMaxResultsToFetch(Integer theMaxResultsToFetch) { 234 myMaxResultsToFetch = theMaxResultsToFetch; 235 } 236 237 private void searchForIdsWithAndOr(SearchQueryBuilder theSearchSqlBuilder, QueryStack theQueryStack, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) { 238 myParams = theParams; 239 240 // Remove any empty parameters 241 theParams.clean(); 242 243 // For DSTU3, pull out near-distance first so when it comes time to evaluate near, we already know the distance 244 if (myContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) { 245 Dstu3DistanceHelper.setNearDistance(myResourceType, theParams); 246 } 247 248 // Attempt to lookup via composite unique key. 249 if (isCompositeUniqueSpCandidate()) { 250 attemptComboUniqueSpProcessing(theQueryStack, theParams, theRequest); 251 } 252 253 SearchContainedModeEnum searchContainedMode = theParams.getSearchContainedMode(); 254 255 // Handle _id and _tag last, since they can typically be tacked onto a different parameter 256 List<String> paramNames = myParams.keySet().stream().filter(t -> !t.equals(IAnyResource.SP_RES_ID)) 257 .filter(t -> !t.equals(Constants.PARAM_TAG)).collect(Collectors.toList()); 258 if (myParams.containsKey(IAnyResource.SP_RES_ID)) { 259 paramNames.add(IAnyResource.SP_RES_ID); 260 } 261 if (myParams.containsKey(Constants.PARAM_TAG)) { 262 paramNames.add(Constants.PARAM_TAG); 263 } 264 265 // Handle each parameter 266 for (String nextParamName : paramNames) { 267 if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) { 268 // Skip parameters for Subject, Patient, Code and Category for LastN as these will be filtered by Elasticsearch 269 continue; 270 } 271 List<List<IQueryParameterType>> andOrParams = myParams.get(nextParamName); 272 Condition predicate = theQueryStack.searchForIdsWithAndOr(null, myResourceName, nextParamName, andOrParams, theRequest, myRequestPartitionId, searchContainedMode); 273 if (predicate != null) { 274 theSearchSqlBuilder.addPredicate(predicate); 275 } 276 } 277 } 278 279 /** 280 * A search is a candidate for Composite Unique SP if unique indexes are enabled, there is no EverythingMode, and the 281 * parameters all have no modifiers. 282 */ 283 private boolean isCompositeUniqueSpCandidate() { 284 return myDaoConfig.isUniqueIndexesEnabled() && 285 myParams.getEverythingMode() == null && 286 myParams.isAllParametersHaveNoModifier(); 287 } 288 289 @SuppressWarnings("ConstantConditions") 290 @Override 291 public Long createCountQuery(SearchParameterMap theParams, String theSearchUuid, 292 RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) { 293 294 assert theRequestPartitionId != null; 295 assert TransactionSynchronizationManager.isActualTransactionActive(); 296 297 init(theParams, theSearchUuid, theRequestPartitionId); 298 299 if (checkUseHibernateSearch()) { 300 long count = myFulltextSearchSvc.count(myResourceName, theParams.clone()); 301 return count; 302 } 303 304 List<ISearchQueryExecutor> queries = createQuery(theParams.clone(), null, null, null, true, theRequest, null); 305 if (queries.isEmpty()) { 306 return 0L; 307 } else { 308 return queries.get(0).next(); 309 } 310 } 311 312 /** 313 * @param thePidSet May be null 314 */ 315 @Override 316 public void setPreviouslyAddedResourcePids(@Nonnull List<JpaPid> thePidSet) { 317 myPidSet = new HashSet<>(thePidSet); 318 } 319 320 @SuppressWarnings("ConstantConditions") 321 @Override 322 public IResultIterator createQuery(SearchParameterMap theParams, SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) { 323 assert theRequestPartitionId != null; 324 assert TransactionSynchronizationManager.isActualTransactionActive(); 325 326 init(theParams, theSearchRuntimeDetails.getSearchUuid(), theRequestPartitionId); 327 328 if (myPidSet == null) { 329 myPidSet = new HashSet<>(); 330 } 331 332 return new QueryIterator(theSearchRuntimeDetails, theRequest); 333 } 334 335 336 private void init(SearchParameterMap theParams, String theSearchUuid, RequestPartitionId theRequestPartitionId) { 337 myCriteriaBuilder = myEntityManager.getCriteriaBuilder(); 338 // we mutate the params. Make a private copy. 339 myParams = theParams.clone(); 340 mySearchUuid = theSearchUuid; 341 myRequestPartitionId = theRequestPartitionId; 342 } 343 344 private List<ISearchQueryExecutor> createQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCountOnlyFlag, RequestDetails theRequest, 345 SearchRuntimeDetails theSearchRuntimeDetails) { 346 347 ArrayList<ISearchQueryExecutor> queries = new ArrayList<>(); 348 349 if (checkUseHibernateSearch()) { 350 // we're going to run at least part of the search against the Fulltext service. 351 352 // Ugh - we have two different return types for now 353 ISearchQueryExecutor fulltextExecutor = null; 354 List<JpaPid> fulltextMatchIds = null; 355 int resultCount = 0; 356 if (myParams.isLastN()) { 357 fulltextMatchIds = executeLastNAgainstIndex(theMaximumResults); 358 resultCount = fulltextMatchIds.size(); 359 } else if (myParams.getEverythingMode() != null) { 360 fulltextMatchIds = queryHibernateSearchForEverythingPids(); 361 resultCount = fulltextMatchIds.size(); 362 } else { 363 fulltextExecutor = myFulltextSearchSvc.searchNotScrolled(myResourceName, myParams, myMaxResultsToFetch); 364 } 365 366 if (fulltextExecutor == null) { 367 fulltextExecutor = SearchQueryExecutors.from(fulltextMatchIds); 368 } 369 370 if (theSearchRuntimeDetails != null) { 371 theSearchRuntimeDetails.setFoundIndexMatchesCount(resultCount); 372 HookParams params = new HookParams() 373 .add(RequestDetails.class, theRequest) 374 .addIfMatchesType(ServletRequestDetails.class, theRequest) 375 .add(SearchRuntimeDetails.class, theSearchRuntimeDetails); 376 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params); 377 } 378 379 // can we skip the database entirely and return the pid list from here? 380 boolean canSkipDatabase = 381 // if we processed an AND clause, and it returned nothing, then nothing can match. 382 !fulltextExecutor.hasNext() || 383 // Our hibernate search query doesn't respect partitions yet 384 (!myPartitionSettings.isPartitioningEnabled() && 385 // were there AND terms left? Then we still need the db. 386 theParams.isEmpty() && 387 // not every param is a param. :-( 388 theParams.getNearDistanceParam() == null && 389 // todo MB don't we support _lastUpdated and _offset now? 390 theParams.getLastUpdated() == null && 391 theParams.getEverythingMode() == null && 392 theParams.getOffset() == null 393 ); 394 395 if (canSkipDatabase) { 396 ourLog.trace("Query finished after HSearch. Skip db query phase"); 397 if (theMaximumResults != null) { 398 fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theMaximumResults); 399 } 400 queries.add(fulltextExecutor); 401 } else { 402 ourLog.trace("Query needs db after HSearch. Chunking."); 403 // Finish the query in the database for the rest of the search parameters, sorting, partitioning, etc. 404 // We break the pids into chunks that fit in the 1k limit for jdbc bind params. 405 new QueryChunker<Long>() 406 .chunk(Streams.stream(fulltextExecutor).collect(Collectors.toList()), t -> doCreateChunkedQueries(theParams, t, theOffset, sort, theCountOnlyFlag, theRequest, queries)); 407 } 408 } else { 409 // do everything in the database. 410 Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, theMaximumResults, theCountOnlyFlag, theRequest, null); 411 query.ifPresent(queries::add); 412 } 413 414 return queries; 415 } 416 417 /** 418 * Check to see if query should use Hibernate Search, and error if the query can't continue. 419 * 420 * @return true if the query should first be processed by Hibernate Search 421 * @throws InvalidRequestException if fulltext search is not enabled but the query requires it - _content or _text 422 */ 423 private boolean checkUseHibernateSearch() { 424 boolean fulltextEnabled = (myFulltextSearchSvc != null) && !myFulltextSearchSvc.isDisabled(); 425 426 if (!fulltextEnabled) { 427 failIfUsed(Constants.PARAM_TEXT); 428 failIfUsed(Constants.PARAM_CONTENT); 429 } 430 431 // someday we'll want a query planner to figure out if we _should_ or _must_ use the ft index, not just if we can. 432 return fulltextEnabled && myParams != null && 433 myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE && 434 myFulltextSearchSvc.supportsSomeOf(myParams); 435 } 436 437 private void failIfUsed(String theParamName) { 438 if (myParams.containsKey(theParamName)) { 439 throw new InvalidRequestException(Msg.code(1192) + "Fulltext search is not enabled on this service, can not process parameter: " + theParamName); 440 } 441 } 442 443 private List<JpaPid> executeLastNAgainstIndex(Integer theMaximumResults) { 444 // Can we use our hibernate search generated index on resource to support lastN?: 445 if (myDaoConfig.isAdvancedHSearchIndexing()) { 446 if (myFulltextSearchSvc == null) { 447 throw new InvalidRequestException(Msg.code(2027) + "LastN operation is not enabled on this service, can not process this request"); 448 } 449 return myFulltextSearchSvc.lastN(myParams, theMaximumResults) 450 .stream().map(lastNResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, String.valueOf(lastNResourceId))) 451 .collect(Collectors.toList()); 452 } else { 453 if (myIElasticsearchSvc == null) { 454 throw new InvalidRequestException(Msg.code(2033) + "LastN operation is not enabled on this service, can not process this request"); 455 } 456 // use the dedicated observation ES/Lucene index to support lastN query 457 return myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults).stream() 458 .map(lastnResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId)) 459 .collect(Collectors.toList()); 460 } 461 } 462 463 private List<JpaPid> queryHibernateSearchForEverythingPids() { 464 JpaPid pid = null; 465 if (myParams.get(IAnyResource.SP_RES_ID) != null) { 466 String idParamValue; 467 IQueryParameterType idParam = myParams.get(IAnyResource.SP_RES_ID).get(0).get(0); 468 if (idParam instanceof TokenParam) { 469 TokenParam idParm = (TokenParam) idParam; 470 idParamValue = idParm.getValue(); 471 } else { 472 StringParam idParm = (StringParam) idParam; 473 idParamValue = idParm.getValue(); 474 } 475 476 pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue); 477 } 478 List<JpaPid> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid); 479 return pids; 480 } 481 482 private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<ISearchQueryExecutor> theQueries) { 483 if (thePids.size() < getMaximumPageSize()) { 484 normalizeIdListForLastNInClause(thePids); 485 } 486 Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, thePids.size(), theCount, theRequest, thePids); 487 query.ifPresent(t -> theQueries.add(t)); 488 } 489 490 /** 491 * Combs through the params for any _id parameters and extracts the PIDs for them 492 * 493 * @param theTargetPids 494 */ 495 private void extractTargetPidsFromIdParams(HashSet<Long> theTargetPids) { 496 // get all the IQueryParameterType objects 497 // for _id -> these should all be StringParam values 498 HashSet<String> ids = new HashSet<>(); 499 List<List<IQueryParameterType>> params = myParams.get(IAnyResource.SP_RES_ID); 500 for (List<IQueryParameterType> paramList : params) { 501 for (IQueryParameterType param : paramList) { 502 if (param instanceof StringParam) { 503 // we expect all _id values to be StringParams 504 ids.add(((StringParam) param).getValue()); 505 } else if (param instanceof TokenParam) { 506 ids.add(((TokenParam) param).getValue()); 507 } else { 508 // we do not expect the _id parameter to be a non-string value 509 throw new IllegalArgumentException(Msg.code(1193) + "_id parameter must be a StringParam or TokenParam"); 510 } 511 } 512 } 513 514 // fetch our target Pids 515 // this will throw if an id is not found 516 Map<String, JpaPid> idToPid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, 517 myResourceName, 518 new ArrayList<>(ids)); 519 if (myAlsoIncludePids == null) { 520 myAlsoIncludePids = new ArrayList<>(); 521 } 522 523 // add the pids to targetPids 524 for (JpaPid pid : idToPid.values()) { 525 myAlsoIncludePids.add(pid); 526 theTargetPids.add(pid.getId()); 527 } 528 } 529 530 private Optional<SearchQueryExecutor> createChunkedQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCountOnlyFlag, RequestDetails theRequest, List<Long> thePidList) { 531 String sqlBuilderResourceName = myParams.getEverythingMode() == null ? myResourceName : null; 532 SearchQueryBuilder sqlBuilder = new SearchQueryBuilder(myContext, myDaoConfig.getModelConfig(), myPartitionSettings, myRequestPartitionId, sqlBuilderResourceName, mySqlBuilderFactory, myDialectProvider, theCountOnlyFlag); 533 QueryStack queryStack3 = new QueryStack(theParams, myDaoConfig, myDaoConfig.getModelConfig(), myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings); 534 535 if (theParams.keySet().size() > 1 || theParams.getSort() != null || theParams.keySet().contains(Constants.PARAM_HAS) || isPotentiallyContainedReferenceParameterExistsAtRoot(theParams)) { 536 List<RuntimeSearchParam> activeComboParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet()); 537 if (activeComboParams.isEmpty()) { 538 sqlBuilder.setNeedResourceTableRoot(true); 539 } 540 } 541 542 JdbcTemplate jdbcTemplate = new JdbcTemplate(myEntityManagerFactory.getDataSource()); 543 jdbcTemplate.setFetchSize(myFetchSize); 544 if (theMaximumResults != null) { 545 jdbcTemplate.setMaxRows(theMaximumResults); 546 } 547 548 if (myParams.getEverythingMode() != null) { 549 HashSet<Long> targetPids = new HashSet<>(); 550 if (myParams.get(IAnyResource.SP_RES_ID) != null) { 551 extractTargetPidsFromIdParams(targetPids); 552 } else { 553 // For Everything queries, we make the query root by the ResourceLink table, since this query 554 // is basically a reverse-include search. For type/Everything (as opposed to instance/Everything) 555 // the one problem with this approach is that it doesn't catch Patients that have absolutely 556 // nothing linked to them. So we do one additional query to make sure we catch those too. 557 SearchQueryBuilder fetchPidsSqlBuilder = new SearchQueryBuilder(myContext, myDaoConfig.getModelConfig(), myPartitionSettings, myRequestPartitionId, myResourceName, mySqlBuilderFactory, myDialectProvider, theCountOnlyFlag); 558 GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theOffset, myMaxResultsToFetch); 559 String sql = allTargetsSql.getSql(); 560 Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]); 561 List<Long> output = jdbcTemplate.query(sql, args, new SingleColumnRowMapper<>(Long.class)); 562 if (myAlsoIncludePids == null) { 563 myAlsoIncludePids = new ArrayList<>(output.size()); 564 } 565 myAlsoIncludePids.addAll(JpaPid.fromLongList(output)); 566 567 } 568 569 List<String> typeSourceResources = new ArrayList<>(); 570 if (myParams.get(Constants.PARAM_TYPE) != null) { 571 typeSourceResources.addAll(extractTypeSourceResourcesFromParams()); 572 } 573 574 queryStack3.addPredicateEverythingOperation(myResourceName, typeSourceResources, targetPids.toArray(new Long[0])); 575 } else { 576 /* 577 * If we're doing a filter, always use the resource table as the root - This avoids the possibility of 578 * specific filters with ORs as their root from working around the natural resource type / deletion 579 * status / partition IDs built into queries. 580 */ 581 if (theParams.containsKey(Constants.PARAM_FILTER)) { 582 Condition partitionIdPredicate = sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId); 583 if (partitionIdPredicate != null) { 584 sqlBuilder.addPredicate(partitionIdPredicate); 585 } 586 } 587 588 // Normal search 589 searchForIdsWithAndOr(sqlBuilder, queryStack3, myParams, theRequest); 590 } 591 592 // If we haven't added any predicates yet, we're doing a search for all resources. Make sure we add the 593 // partition ID predicate in that case. 594 if (!sqlBuilder.haveAtLeastOnePredicate()) { 595 Condition partitionIdPredicate = sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId); 596 if (partitionIdPredicate != null) { 597 sqlBuilder.addPredicate(partitionIdPredicate); 598 } 599 } 600 601 // Add PID list predicate for full text search and/or lastn operation 602 if (thePidList != null && thePidList.size() > 0) { 603 sqlBuilder.addResourceIdsPredicate(thePidList); 604 } 605 606 // Last updated 607 DateRangeParam lu = myParams.getLastUpdated(); 608 if (lu != null && !lu.isEmpty()) { 609 Condition lastUpdatedPredicates = sqlBuilder.addPredicateLastUpdated(lu); 610 sqlBuilder.addPredicate(lastUpdatedPredicates); 611 } 612 613 /* 614 * Exclude the pids already in the previous iterator. This is an optimization, as opposed 615 * to something needed to guarantee correct results. 616 * 617 * Why do we need it? Suppose for example, a query like: 618 * Observation?category=foo,bar,baz 619 * And suppose you have many resources that have all 3 of these category codes. In this case 620 * the SQL query will probably return the same PIDs multiple times, and if this happens enough 621 * we may exhaust the query results without getting enough distinct results back. When that 622 * happens we re-run the query with a larger limit. Excluding results we already know about 623 * tries to ensure that we get new unique results. 624 * 625 * The challenge with that though is that lots of DBs have an issue with too many 626 * parameters in one query. So we only do this optimization if there aren't too 627 * many results. 628 */ 629 if (myHasNextIteratorQuery) { 630 if (myPidSet.size() + sqlBuilder.countBindVariables() < 900) { 631 sqlBuilder.excludeResourceIdsPredicate(myPidSet); 632 } 633 } 634 635 /* 636 * If offset is present, we want deduplicate the results by using GROUP BY 637 */ 638 if (theOffset != null) { 639 queryStack3.addGrouping(); 640 queryStack3.setUseAggregate(true); 641 } 642 643 /* 644 * Sort 645 * 646 * If we have a sort, we wrap the criteria search (the search that actually 647 * finds the appropriate resources) in an outer search which is then sorted 648 */ 649 if (sort != null) { 650 assert !theCountOnlyFlag; 651 652 createSort(queryStack3, sort); 653 } 654 655 656 /* 657 * Now perform the search 658 */ 659 GeneratedSql generatedSql = sqlBuilder.generate(theOffset, myMaxResultsToFetch); 660 if (generatedSql.isMatchNothing()) { 661 return Optional.empty(); 662 } 663 664 SearchQueryExecutor executor = mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, myMaxResultsToFetch); 665 return Optional.of(executor); 666 } 667 668 private Collection<String> extractTypeSourceResourcesFromParams() { 669 670 List<List<IQueryParameterType>> listOfList = myParams.get(Constants.PARAM_TYPE); 671 672 // first off, let's flatten the list of list 673 List<IQueryParameterType> iQueryParameterTypesList = listOfList.stream().flatMap(List::stream).collect(Collectors.toList()); 674 675 // then, extract all elements of each CSV into one big list 676 List<String> resourceTypes = iQueryParameterTypesList 677 .stream() 678 .map(param -> ((StringParam) param).getValue()) 679 .map(csvString -> List.of(csvString.split(","))) 680 .flatMap(List::stream).collect(Collectors.toList()); 681 682 Set<String> knownResourceTypes = myContext.getResourceTypes(); 683 684 // remove leading/trailing whitespaces if any and remove duplicates 685 Set<String> retVal = new HashSet<>(); 686 687 for (String type : resourceTypes) { 688 String trimmed = type.trim(); 689 if (!knownResourceTypes.contains(trimmed)) { 690 throw new ResourceNotFoundException(Msg.code(2197) + "Unknown resource type '" + trimmed + "' in _type parameter."); 691 } 692 retVal.add(trimmed); 693 } 694 695 return retVal; 696 } 697 698 private boolean isPotentiallyContainedReferenceParameterExistsAtRoot(SearchParameterMap theParams) { 699 return myModelConfig.isIndexOnContainedResources() && theParams.values().stream() 700 .flatMap(Collection::stream) 701 .flatMap(Collection::stream) 702 .anyMatch(t -> t instanceof ReferenceParam); 703 } 704 705 private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) { 706 /* 707 The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying 708 numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info: 709 https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage. 710 711 Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of 712 arguments never exceeds the maximum specified below. 713 */ 714 int listSize = lastnResourceIds.size(); 715 716 if (listSize > 1 && listSize < 10) { 717 padIdListWithPlaceholders(lastnResourceIds, 10); 718 } else if (listSize > 10 && listSize < 50) { 719 padIdListWithPlaceholders(lastnResourceIds, 50); 720 } else if (listSize > 50 && listSize < 100) { 721 padIdListWithPlaceholders(lastnResourceIds, 100); 722 } else if (listSize > 100 && listSize < 200) { 723 padIdListWithPlaceholders(lastnResourceIds, 200); 724 } else if (listSize > 200 && listSize < 500) { 725 padIdListWithPlaceholders(lastnResourceIds, 500); 726 } else if (listSize > 500 && listSize < 800) { 727 padIdListWithPlaceholders(lastnResourceIds, 800); 728 } 729 730 return lastnResourceIds; 731 } 732 733 private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) { 734 while (theIdList.size() < preferredListSize) { 735 theIdList.add(-1L); 736 } 737 } 738 739 private void createSort(QueryStack theQueryStack, SortSpec theSort) { 740 if (theSort == null || isBlank(theSort.getParamName())) { 741 return; 742 } 743 744 boolean ascending = (theSort.getOrder() == null) || (theSort.getOrder() == SortOrderEnum.ASC); 745 746 if (IAnyResource.SP_RES_ID.equals(theSort.getParamName())) { 747 748 theQueryStack.addSortOnResourceId(ascending); 749 750 } else if (Constants.PARAM_LASTUPDATED.equals(theSort.getParamName())) { 751 752 theQueryStack.addSortOnLastUpdated(ascending); 753 754 } else { 755 756 RuntimeSearchParam param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName()); 757 if (param == null) { 758 String msg = myContext.getLocalizer().getMessageSanitized(BaseStorageDao.class, "invalidSortParameter", theSort.getParamName(), getResourceName(), mySearchParamRegistry.getValidSearchParameterNamesIncludingMeta(getResourceName())); 759 throw new InvalidRequestException(Msg.code(1194) + msg); 760 } 761 762 switch (param.getParamType()) { 763 case STRING: 764 theQueryStack.addSortOnString(myResourceName, theSort.getParamName(), ascending); 765 break; 766 case DATE: 767 theQueryStack.addSortOnDate(myResourceName, theSort.getParamName(), ascending); 768 break; 769 case REFERENCE: 770 theQueryStack.addSortOnResourceLink(myResourceName, theSort.getParamName(), ascending); 771 break; 772 case TOKEN: 773 theQueryStack.addSortOnToken(myResourceName, theSort.getParamName(), ascending); 774 break; 775 case NUMBER: 776 theQueryStack.addSortOnNumber(myResourceName, theSort.getParamName(), ascending); 777 break; 778 case URI: 779 theQueryStack.addSortOnUri(myResourceName, theSort.getParamName(), ascending); 780 break; 781 case QUANTITY: 782 theQueryStack.addSortOnQuantity(myResourceName, theSort.getParamName(), ascending); 783 break; 784 case COMPOSITE: 785 List<RuntimeSearchParam> compositeList = JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, param); 786 if (compositeList == null) { 787 throw new InvalidRequestException(Msg.code(1195) + "The composite _sort parameter " + theSort.getParamName() + " is not defined by the resource " + myResourceName); 788 } 789 if (compositeList.size() != 2) { 790 throw new InvalidRequestException(Msg.code(1196) + "The composite _sort parameter " + theSort.getParamName() 791 + " must have 2 composite types declared in parameter annotation, found " 792 + compositeList.size()); 793 } 794 RuntimeSearchParam left = compositeList.get(0); 795 RuntimeSearchParam right = compositeList.get(1); 796 797 createCompositeSort(theQueryStack, left.getParamType(), left.getName(), ascending); 798 createCompositeSort(theQueryStack, right.getParamType(), right.getName(), ascending); 799 800 break; 801 case SPECIAL: 802 case HAS: 803 default: 804 throw new InvalidRequestException(Msg.code(1197) + "This server does not support _sort specifications of type " + param.getParamType() + " - Can't serve _sort=" + theSort.getParamName()); 805 } 806 807 } 808 809 // Recurse 810 createSort(theQueryStack, theSort.getChain()); 811 812 } 813 814 private void createCompositeSort(QueryStack theQueryStack, RestSearchParameterTypeEnum theParamType, String theParamName, boolean theAscending) { 815 816 switch (theParamType) { 817 case STRING: 818 theQueryStack.addSortOnString(myResourceName, theParamName, theAscending); 819 break; 820 case DATE: 821 theQueryStack.addSortOnDate(myResourceName, theParamName, theAscending); 822 break; 823 case TOKEN: 824 theQueryStack.addSortOnToken(myResourceName, theParamName, theAscending); 825 break; 826 case QUANTITY: 827 theQueryStack.addSortOnQuantity(myResourceName, theParamName, theAscending); 828 break; 829 case NUMBER: 830 case REFERENCE: 831 case COMPOSITE: 832 case URI: 833 case HAS: 834 case SPECIAL: 835 default: 836 throw new InvalidRequestException(Msg.code(1198) + "Don't know how to handle composite parameter with type of " + theParamType + " on _sort=" + theParamName); 837 } 838 839 } 840 841 private void doLoadPids(Collection<JpaPid> thePids, Collection<JpaPid> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation, 842 Map<JpaPid, Integer> thePosition) { 843 844 Map<Long, Long> resourcePidToVersion = null; 845 for (JpaPid next : thePids) { 846 if (next.getVersion() != null && myModelConfig.isRespectVersionsForSearchIncludes()) { 847 if (resourcePidToVersion == null) { 848 resourcePidToVersion = new HashMap<>(); 849 } 850 resourcePidToVersion.put((next).getId(), next.getVersion()); 851 } 852 } 853 854 List<Long> versionlessPids = JpaPid.toLongList(thePids); 855 if (versionlessPids.size() < getMaximumPageSize()) { 856 versionlessPids = normalizeIdListForLastNInClause(versionlessPids); 857 } 858 859 // -- get the resource from the searchView 860 Collection<ResourceSearchView> resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(versionlessPids); 861 862 //-- preload all tags with tag definition if any 863 Map<Long, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList); 864 865 for (IBaseResourceEntity next : resourceSearchViewList) { 866 if (next.getDeleted() != null) { 867 continue; 868 } 869 870 Class<? extends IBaseResource> resourceType = myContext.getResourceDefinition(next.getResourceType()).getImplementingClass(); 871 872 JpaPid resourceId = JpaPid.fromId(next.getResourceId()); 873 874 /* 875 * If a specific version is requested via an include, we'll replace the current version 876 * with the specific desired version. This is not the most efficient thing, given that 877 * we're loading the current version and then turning around and throwing it away again. 878 * This could be optimized and probably should be, but it's not critical given that 879 * this only applies to includes, which don't tend to be massive in numbers. 880 */ 881 if (resourcePidToVersion != null) { 882 Long version = resourcePidToVersion.get(next.getResourceId()); 883 resourceId.setVersion(version); 884 if (version != null && !version.equals(next.getVersion())) { 885 IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(resourceType); 886 next = dao.readEntity(next.getIdDt().withVersion(Long.toString(version)), null); 887 } 888 } 889 890 IBaseResource resource = null; 891 if (next != null) { 892 resource = myJpaStorageResourceParser.toResource(resourceType, next, tagMap.get(next.getId()), theForHistoryOperation); 893 } 894 if (resource == null) { 895 ourLog.warn("Unable to find resource {}/{}/_history/{} in database", next.getResourceType(), next.getIdDt().getIdPart(), next.getVersion()); 896 continue; 897 } 898 899 Integer index = thePosition.get(resourceId); 900 if (index == null) { 901 ourLog.warn("Got back unexpected resource PID {}", resourceId); 902 continue; 903 } 904 905 if (theIncludedPids.contains(resourceId)) { 906 ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put(resource, BundleEntrySearchModeEnum.INCLUDE); 907 } else { 908 ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put(resource, BundleEntrySearchModeEnum.MATCH); 909 } 910 911 theResourceListToPopulate.set(index, resource); 912 } 913 } 914 915 private Map<Long, Collection<ResourceTag>> getResourceTagMap(Collection<? extends IBaseResourceEntity> theResourceSearchViewList) { 916 917 List<Long> idList = new ArrayList<>(theResourceSearchViewList.size()); 918 919 //-- find all resource has tags 920 for (IBaseResourceEntity resource : theResourceSearchViewList) { 921 if (resource.isHasTags()) 922 idList.add(resource.getId()); 923 } 924 925 return getPidToTagMap(idList); 926 } 927 928 @Nonnull 929 private Map<Long, Collection<ResourceTag>> getPidToTagMap(List<Long> thePidList) { 930 Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>(); 931 932 //-- no tags 933 if (thePidList.size() == 0) 934 return tagMap; 935 936 //-- get all tags for the idList 937 Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(thePidList); 938 939 //-- build the map, key = resourceId, value = list of ResourceTag 940 JpaPid resourceId; 941 Collection<ResourceTag> tagCol; 942 for (ResourceTag tag : tagList) { 943 944 resourceId = JpaPid.fromId(tag.getResourceId()); 945 tagCol = tagMap.get(resourceId.getId()); 946 if (tagCol == null) { 947 tagCol = new ArrayList<>(); 948 tagCol.add(tag); 949 tagMap.put(resourceId.getId(), tagCol); 950 } else { 951 tagCol.add(tag); 952 } 953 } 954 955 return tagMap; 956 } 957 958 @Override 959 public void loadResourcesByPid(Collection<JpaPid> thePids, Collection<JpaPid> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation, RequestDetails theDetails) { 960 if (thePids.isEmpty()) { 961 ourLog.debug("The include pids are empty"); 962 // return; 963 } 964 965 // Dupes will cause a crash later anyhow, but this is expensive so only do it 966 // when running asserts 967 assert new HashSet<>(thePids).size() == thePids.size() : "PID list contains duplicates: " + thePids; 968 969 Map<JpaPid, Integer> position = new HashMap<>(); 970 for (JpaPid next : thePids) { 971 position.put(next, theResourceListToPopulate.size()); 972 theResourceListToPopulate.add(null); 973 } 974 975 // Can we fast track this loading by checking elastic search? 976 if (isLoadingFromElasticSearchSupported(thePids)) { 977 try { 978 theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids)); 979 return; 980 981 } catch (ResourceNotFoundInIndexException theE) { 982 // some resources were not found in index, so we will inform this and resort to JPA search 983 ourLog.warn("Some resources were not found in index. Make sure all resources were indexed. Resorting to database search."); 984 } 985 } 986 987 // We only chunk because some jdbc drivers can't handle long param lists. 988 new QueryChunker<JpaPid>().chunk(thePids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position)); 989 } 990 991 /** 992 * Check if we can load the resources from Hibernate Search instead of the database. 993 * We assume this is faster. 994 * <p> 995 * Hibernate Search only stores the current version, and only if enabled. 996 * 997 * @param thePids the pids to check for versioned references 998 * @return can we fetch from Hibernate Search? 999 */ 1000 private boolean isLoadingFromElasticSearchSupported(Collection<JpaPid> thePids) { 1001 // is storage enabled? 1002 return myDaoConfig.isStoreResourceInHSearchIndex() && 1003 myDaoConfig.isAdvancedHSearchIndexing() && 1004 // we don't support history 1005 thePids.stream().noneMatch(p -> p.getVersion() != null) && 1006 // skip the complexity for metadata in dstu2 1007 myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3); 1008 } 1009 1010 private List<IBaseResource> loadResourcesFromElasticSearch(Collection<JpaPid> thePids) { 1011 // Do we use the fulltextsvc via hibernate-search to load resources or be backwards compatible with older ES only impl 1012 // to handle lastN? 1013 if (myDaoConfig.isAdvancedHSearchIndexing() && myDaoConfig.isStoreResourceInHSearchIndex()) { 1014 List<Long> pidList = thePids.stream().map(pid -> (pid).getId()).collect(Collectors.toList()); 1015 1016 List<IBaseResource> resources = myFulltextSearchSvc.getResources(pidList); 1017 return resources; 1018 } else if (!Objects.isNull(myParams) && myParams.isLastN()) { 1019 // legacy LastN implementation 1020 return myIElasticsearchSvc.getObservationResources(thePids); 1021 } else { 1022 return Collections.emptyList(); 1023 } 1024 } 1025 1026 /** 1027 * THIS SHOULD RETURN HASHSET and not just Set because we add to it later 1028 * so it can't be Collections.emptySet() or some such thing. 1029 * The JpaPid returned will have resource type populated. 1030 */ 1031 @Override 1032 public Set<JpaPid> loadIncludes(FhirContext theContext, EntityManager theEntityManager, Collection<JpaPid> theMatches, Collection<Include> theIncludes, 1033 boolean theReverseMode, DateRangeParam theLastUpdated, String theSearchIdOrDescription, RequestDetails theRequest, Integer theMaxCount) { 1034 if (theMatches.size() == 0) { 1035 return new HashSet<>(); 1036 } 1037 if (theIncludes == null || theIncludes.isEmpty()) { 1038 return new HashSet<>(); 1039 } 1040 String searchPidFieldName = theReverseMode ? MY_TARGET_RESOURCE_PID : MY_SOURCE_RESOURCE_PID; 1041 String findPidFieldName = theReverseMode ? MY_SOURCE_RESOURCE_PID : MY_TARGET_RESOURCE_PID; 1042 String findResourceTypeFieldName = theReverseMode ? MY_SOURCE_RESOURCE_TYPE : MY_TARGET_RESOURCE_TYPE; 1043 String findVersionFieldName = null; 1044 if (!theReverseMode && myModelConfig.isRespectVersionsForSearchIncludes()) { 1045 findVersionFieldName = MY_TARGET_RESOURCE_VERSION; 1046 } 1047 1048 List<JpaPid> nextRoundMatches = new ArrayList<>(theMatches); 1049 HashSet<JpaPid> allAdded = new HashSet<>(); 1050 HashSet<JpaPid> original = new HashSet<>(theMatches); 1051 ArrayList<Include> includes = new ArrayList<>(theIncludes); 1052 1053 int roundCounts = 0; 1054 StopWatch w = new StopWatch(); 1055 1056 boolean addedSomeThisRound; 1057 do { 1058 roundCounts++; 1059 1060 HashSet<JpaPid> pidsToInclude = new HashSet<>(); 1061 1062 for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) { 1063 Include nextInclude = iter.next(); 1064 if (nextInclude.isRecurse() == false) { 1065 iter.remove(); 1066 } 1067 1068 // Account for _include=* 1069 boolean matchAll = "*".equals(nextInclude.getValue()); 1070 1071 // Account for _include=[resourceType]:* 1072 String wantResourceType = null; 1073 if (!matchAll) { 1074 if ("*".equals(nextInclude.getParamName())) { 1075 wantResourceType = nextInclude.getParamType(); 1076 matchAll = true; 1077 } 1078 } 1079 1080 if (matchAll) { 1081 StringBuilder sqlBuilder = new StringBuilder(); 1082 sqlBuilder.append("SELECT r.").append(findPidFieldName); 1083 sqlBuilder.append(", r.").append(findResourceTypeFieldName); 1084 if (findVersionFieldName != null) { 1085 sqlBuilder.append(", r." + findVersionFieldName); 1086 } 1087 sqlBuilder.append(" FROM ResourceLink r WHERE "); 1088 1089 sqlBuilder.append("r."); 1090 sqlBuilder.append(searchPidFieldName); 1091 sqlBuilder.append(" IN (:target_pids)"); 1092 1093 // Technically if the request is a qualified star (e.g. _include=Observation:*) we 1094 // should always be checking the source resource type on the resource link. We don't 1095 // actually index that column though by default, so in order to try and be efficient 1096 // we don't actually include it for includes (but we do for revincludes). This is 1097 // because for an include it doesn't really make sense to include a different 1098 // resource type than the one you are searching on. 1099 if (wantResourceType != null && theReverseMode) { 1100 sqlBuilder.append(" AND r.mySourceResourceType = :want_resource_type"); 1101 } else { 1102 wantResourceType = null; 1103 } 1104 1105 String sql = sqlBuilder.toString(); 1106 List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize()); 1107 for (Collection<JpaPid> nextPartition : partitions) { 1108 TypedQuery<?> q = theEntityManager.createQuery(sql, Object[].class); 1109 q.setParameter("target_pids", JpaPid.toLongList(nextPartition)); 1110 if (wantResourceType != null) { 1111 q.setParameter("want_resource_type", wantResourceType); 1112 } 1113 if (theMaxCount != null) { 1114 q.setMaxResults(theMaxCount); 1115 } 1116 List<?> results = q.getResultList(); 1117 for (Object nextRow : results) { 1118 if (nextRow == null) { 1119 // This can happen if there are outgoing references which are canonical or point to 1120 // other servers 1121 continue; 1122 } 1123 1124 Long version = null; 1125 Long resourceLink = (Long) ((Object[]) nextRow)[0]; 1126 String resourceType = (String) ((Object[]) nextRow)[1]; 1127 if (findVersionFieldName != null) { 1128 version = (Long) ((Object[]) nextRow)[2]; 1129 } 1130 1131 if (resourceLink != null) { 1132 JpaPid pid = JpaPid.fromIdAndVersionAndResourceType(resourceLink, version, resourceType); 1133 pidsToInclude.add(pid); 1134 } 1135 } 1136 } 1137 } else { 1138 1139 List<String> paths; 1140 1141 // Start replace 1142 RuntimeSearchParam param; 1143 String resType = nextInclude.getParamType(); 1144 if (isBlank(resType)) { 1145 continue; 1146 } 1147 RuntimeResourceDefinition def = theContext.getResourceDefinition(resType); 1148 if (def == null) { 1149 ourLog.warn("Unknown resource type in include/revinclude=" + nextInclude.getValue()); 1150 continue; 1151 } 1152 1153 String paramName = nextInclude.getParamName(); 1154 if (isNotBlank(paramName)) { 1155 param = mySearchParamRegistry.getActiveSearchParam(resType, paramName); 1156 } else { 1157 param = null; 1158 } 1159 if (param == null) { 1160 ourLog.warn("Unknown param name in include/revinclude=" + nextInclude.getValue()); 1161 continue; 1162 } 1163 1164 paths = param.getPathsSplitForResourceType(resType); 1165 // end replace 1166 1167 String targetResourceType = defaultString(nextInclude.getParamTargetType(), null); 1168 for (String nextPath : paths) { 1169 boolean haveTargetTypesDefinedByParam = param.hasTargets(); 1170 String findPidFieldSqlColumn = findPidFieldName.equals(MY_SOURCE_RESOURCE_PID) ? "src_resource_id" : "target_resource_id"; 1171 String fieldsToLoad = "r." + findPidFieldSqlColumn + " AS " + RESOURCE_ID_ALIAS; 1172 if (findVersionFieldName != null) { 1173 fieldsToLoad += ", r.target_resource_version AS " + RESOURCE_VERSION_ALIAS; 1174 } 1175 1176 // Query for includes lookup has consider 2 cases 1177 // Case 1: Where target_resource_id is available in hfj_res_link table for local references 1178 // Case 2: Where target_resource_id is null in hfj_res_link table and referred by a canonical url in target_resource_url 1179 1180 // Case 1: 1181 String searchPidFieldSqlColumn = searchPidFieldName.equals(MY_TARGET_RESOURCE_PID) ? "target_resource_id" : "src_resource_id"; 1182 StringBuilder resourceIdBasedQuery = new StringBuilder("SELECT " + fieldsToLoad + 1183 " FROM hfj_res_link r " + 1184 " WHERE r.src_path = :src_path AND " + 1185 " r.target_resource_id IS NOT NULL AND " + 1186 " r." + searchPidFieldSqlColumn + " IN (:target_pids) "); 1187 if (targetResourceType != null) { 1188 resourceIdBasedQuery.append(" AND r.target_resource_type = :target_resource_type "); 1189 } else if (haveTargetTypesDefinedByParam) { 1190 resourceIdBasedQuery.append(" AND r.target_resource_type in (:target_resource_types) "); 1191 } 1192 1193 // Case 2: 1194 String fieldsToLoadFromSpidxUriTable = "rUri.res_id"; 1195 // to match the fields loaded in union 1196 if (fieldsToLoad.split(",").length > 1) { 1197 for (int i = 0; i < fieldsToLoad.split(",").length - 1; i++) { 1198 fieldsToLoadFromSpidxUriTable += ", NULL"; 1199 } 1200 } 1201 //@formatter:off 1202 StringBuilder resourceUrlBasedQuery = new StringBuilder("SELECT " + fieldsToLoadFromSpidxUriTable + 1203 " FROM hfj_res_link r " + 1204 " JOIN hfj_spidx_uri rUri ON ( " + 1205 " r.target_resource_url = rUri.sp_uri AND " + 1206 " rUri.sp_name = 'url' "); 1207 1208 if (targetResourceType != null) { 1209 resourceUrlBasedQuery.append(" AND rUri.res_type = :target_resource_type "); 1210 1211 } else if (haveTargetTypesDefinedByParam) { 1212 resourceUrlBasedQuery.append(" AND rUri.res_type IN (:target_resource_types) "); 1213 } 1214 1215 resourceUrlBasedQuery.append(" ) "); 1216 resourceUrlBasedQuery.append( 1217 " WHERE r.src_path = :src_path AND " + 1218 " r.target_resource_id IS NULL AND " + 1219 " r." + searchPidFieldSqlColumn + " IN (:target_pids) "); 1220 //@formatter:on 1221 1222 String sql = resourceIdBasedQuery + " UNION " + resourceUrlBasedQuery; 1223 1224 List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize()); 1225 for (Collection<JpaPid> nextPartition : partitions) { 1226 Query q = theEntityManager.createNativeQuery(sql, Tuple.class); 1227 q.setParameter("src_path", nextPath); 1228 q.setParameter("target_pids", JpaPid.toLongList(nextPartition)); 1229 if (targetResourceType != null) { 1230 q.setParameter("target_resource_type", targetResourceType); 1231 } else if (haveTargetTypesDefinedByParam) { 1232 q.setParameter("target_resource_types", param.getTargets()); 1233 } 1234 1235 if (theMaxCount != null) { 1236 q.setMaxResults(theMaxCount); 1237 } 1238 List<Tuple> results = q.getResultList(); 1239 for (Tuple result : results) { 1240 if (result != null) { 1241 Long resourceId = NumberUtils.createLong(String.valueOf(result.get(RESOURCE_ID_ALIAS))); 1242 Long resourceVersion = null; 1243 if (findVersionFieldName != null && result.get(RESOURCE_VERSION_ALIAS) != null) { 1244 resourceVersion = NumberUtils.createLong(String.valueOf(result.get(RESOURCE_VERSION_ALIAS))); 1245 } 1246 pidsToInclude.add(JpaPid.fromIdAndVersion(resourceId, resourceVersion)); 1247 } 1248 } 1249 } 1250 } 1251 } 1252 } 1253 1254 if (theReverseMode) { 1255 if (theLastUpdated != null && (theLastUpdated.getLowerBoundAsInstant() != null || theLastUpdated.getUpperBoundAsInstant() != null)) { 1256 pidsToInclude = new HashSet<>(QueryParameterUtils.filterResourceIdsByLastUpdated(theEntityManager, theLastUpdated, pidsToInclude)); 1257 } 1258 } 1259 1260 nextRoundMatches.clear(); 1261 for (JpaPid next : pidsToInclude) { 1262 if (original.contains(next) == false && allAdded.contains(next) == false) { 1263 nextRoundMatches.add(next); 1264 } 1265 } 1266 1267 addedSomeThisRound = allAdded.addAll(pidsToInclude); 1268 1269 if (theMaxCount != null && allAdded.size() >= theMaxCount) { 1270 break; 1271 } 1272 1273 } while (includes.size() > 0 && nextRoundMatches.size() > 0 && addedSomeThisRound); 1274 1275 allAdded.removeAll(original); 1276 1277 ourLog.info("Loaded {} {} in {} rounds and {} ms for search {}", allAdded.size(), theReverseMode ? "_revincludes" : "_includes", roundCounts, w.getMillisAndRestart(), theSearchIdOrDescription); 1278 1279 // Interceptor call: STORAGE_PREACCESS_RESOURCES 1280 // This can be used to remove results from the search result details before 1281 // the user has a chance to know that they were in the results 1282 if (allAdded.size() > 0) { 1283 1284 if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_PREACCESS_RESOURCES, myInterceptorBroadcaster, theRequest)) { 1285 List<JpaPid> includedPidList = new ArrayList<>(allAdded); 1286 JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(includedPidList, () -> this); 1287 HookParams params = new HookParams() 1288 .add(IPreResourceAccessDetails.class, accessDetails) 1289 .add(RequestDetails.class, theRequest) 1290 .addIfMatchesType(ServletRequestDetails.class, theRequest); 1291 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params); 1292 1293 for (int i = includedPidList.size() - 1; i >= 0; i--) { 1294 if (accessDetails.isDontReturnResourceAtIndex(i)) { 1295 JpaPid value = includedPidList.remove(i); 1296 if (value != null) { 1297 allAdded.remove(value); 1298 } 1299 } 1300 } 1301 } 1302 } 1303 1304 return allAdded; 1305 } 1306 1307 private List<Collection<JpaPid>> partition(Collection<JpaPid> theNextRoundMatches, int theMaxLoad) { 1308 if (theNextRoundMatches.size() <= theMaxLoad) { 1309 return Collections.singletonList(theNextRoundMatches); 1310 } else { 1311 1312 List<Collection<JpaPid>> retVal = new ArrayList<>(); 1313 Collection<JpaPid> current = null; 1314 for (JpaPid next : theNextRoundMatches) { 1315 if (current == null) { 1316 current = new ArrayList<>(theMaxLoad); 1317 retVal.add(current); 1318 } 1319 1320 current.add(next); 1321 1322 if (current.size() >= theMaxLoad) { 1323 current = null; 1324 } 1325 } 1326 1327 return retVal; 1328 } 1329 } 1330 1331 private void attemptComboUniqueSpProcessing(QueryStack theQueryStack3, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) { 1332 RuntimeSearchParam comboParam = null; 1333 List<String> comboParamNames = null; 1334 List<RuntimeSearchParam> exactMatchParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet()); 1335 if (exactMatchParams.size() > 0) { 1336 comboParam = exactMatchParams.get(0); 1337 comboParamNames = new ArrayList<>(theParams.keySet()); 1338 } 1339 1340 if (comboParam == null) { 1341 List<RuntimeSearchParam> candidateComboParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName); 1342 for (RuntimeSearchParam nextCandidate : candidateComboParams) { 1343 List<String> nextCandidateParamNames = JpaParamUtil 1344 .resolveComponentParameters(mySearchParamRegistry, nextCandidate) 1345 .stream() 1346 .map(t -> t.getName()) 1347 .collect(Collectors.toList()); 1348 if (theParams.keySet().containsAll(nextCandidateParamNames)) { 1349 comboParam = nextCandidate; 1350 comboParamNames = nextCandidateParamNames; 1351 break; 1352 } 1353 } 1354 } 1355 1356 if (comboParam != null) { 1357 // Since we're going to remove elements below 1358 theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList)); 1359 1360 StringBuilder sb = new StringBuilder(); 1361 sb.append(myResourceName); 1362 sb.append("?"); 1363 1364 boolean first = true; 1365 1366 Collections.sort(comboParamNames); 1367 for (String nextParamName : comboParamNames) { 1368 List<List<IQueryParameterType>> nextValues = theParams.get(nextParamName); 1369 1370 // TODO Hack to fix weird IOOB on the next stanza until James comes back and makes sense of this. 1371 if (nextValues.isEmpty()) { 1372 ourLog.error("query parameter {} is unexpectedly empty. Encountered while considering {} index for {}", nextParamName, comboParam.getName(), theRequest.getCompleteUrl()); 1373 sb = null; 1374 break; 1375 } 1376 1377 if (nextValues.get(0).size() != 1) { 1378 sb = null; 1379 break; 1380 } 1381 1382 // Reference params are only eligible for using a composite index if they 1383 // are qualified 1384 RuntimeSearchParam nextParamDef = mySearchParamRegistry.getActiveSearchParam(myResourceName, nextParamName); 1385 if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.REFERENCE) { 1386 ReferenceParam param = (ReferenceParam) nextValues.get(0).get(0); 1387 if (isBlank(param.getResourceType())) { 1388 sb = null; 1389 break; 1390 } 1391 } 1392 1393 List<? extends IQueryParameterType> nextAnd = nextValues.remove(0); 1394 IQueryParameterType nextOr = nextAnd.remove(0); 1395 String nextOrValue = nextOr.getValueAsQueryToken(myContext); 1396 1397 if (comboParam.getComboSearchParamType() == ComboSearchParamType.NON_UNIQUE) { 1398 if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.STRING) { 1399 nextOrValue = StringUtil.normalizeStringForSearchIndexing(nextOrValue); 1400 } 1401 } 1402 1403 if (first) { 1404 first = false; 1405 } else { 1406 sb.append('&'); 1407 } 1408 1409 nextParamName = UrlUtil.escapeUrlParam(nextParamName); 1410 nextOrValue = UrlUtil.escapeUrlParam(nextOrValue); 1411 1412 sb.append(nextParamName).append('=').append(nextOrValue); 1413 1414 } 1415 1416 if (sb != null) { 1417 String indexString = sb.toString(); 1418 ourLog.debug("Checking for {} combo index for query: {}", comboParam.getComboSearchParamType(), indexString); 1419 1420 // Interceptor broadcast: JPA_PERFTRACE_INFO 1421 StorageProcessingMessage msg = new StorageProcessingMessage() 1422 .setMessage("Using " + comboParam.getComboSearchParamType() + " index for query for search: " + indexString); 1423 HookParams params = new HookParams() 1424 .add(RequestDetails.class, theRequest) 1425 .addIfMatchesType(ServletRequestDetails.class, theRequest) 1426 .add(StorageProcessingMessage.class, msg); 1427 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INFO, params); 1428 1429 switch (comboParam.getComboSearchParamType()) { 1430 case UNIQUE: 1431 theQueryStack3.addPredicateCompositeUnique(indexString, myRequestPartitionId); 1432 break; 1433 case NON_UNIQUE: 1434 theQueryStack3.addPredicateCompositeNonUnique(indexString, myRequestPartitionId); 1435 break; 1436 } 1437 1438 // Remove any empty parameters remaining after this 1439 theParams.clean(); 1440 } 1441 } 1442 } 1443 1444 private <T> void ensureSubListsAreWritable(List<List<T>> theListOfLists) { 1445 for (int i = 0; i < theListOfLists.size(); i++) { 1446 List<T> oldSubList = theListOfLists.get(i); 1447 if (!(oldSubList instanceof ArrayList)) { 1448 List<T> newSubList = new ArrayList<>(oldSubList); 1449 theListOfLists.set(i, newSubList); 1450 } 1451 } 1452 } 1453 1454 @Override 1455 public void setFetchSize(int theFetchSize) { 1456 myFetchSize = theFetchSize; 1457 } 1458 1459 public SearchParameterMap getParams() { 1460 return myParams; 1461 } 1462 1463 public CriteriaBuilder getBuilder() { 1464 return myCriteriaBuilder; 1465 } 1466 1467 public Class<? extends IBaseResource> getResourceType() { 1468 return myResourceType; 1469 } 1470 1471 public String getResourceName() { 1472 return myResourceName; 1473 } 1474 1475 public class IncludesIterator extends BaseIterator<JpaPid> implements Iterator<JpaPid> { 1476 1477 private final RequestDetails myRequest; 1478 private final Set<JpaPid> myCurrentPids; 1479 private Iterator<JpaPid> myCurrentIterator; 1480 private JpaPid myNext; 1481 1482 IncludesIterator(Set<JpaPid> thePidSet, RequestDetails theRequest) { 1483 myCurrentPids = new HashSet<>(thePidSet); 1484 myCurrentIterator = null; 1485 myRequest = theRequest; 1486 } 1487 1488 private void fetchNext() { 1489 while (myNext == null) { 1490 1491 if (myCurrentIterator == null) { 1492 Set<Include> includes = Collections.singleton(new Include("*", true)); 1493 Set<JpaPid> newPids = loadIncludes(myContext, myEntityManager, myCurrentPids, includes, false, getParams().getLastUpdated(), mySearchUuid, myRequest, null); 1494 myCurrentIterator = newPids.iterator(); 1495 } 1496 1497 if (myCurrentIterator.hasNext()) { 1498 myNext = myCurrentIterator.next(); 1499 } else { 1500 myNext = NO_MORE; 1501 } 1502 1503 } 1504 } 1505 1506 @Override 1507 public boolean hasNext() { 1508 fetchNext(); 1509 return !NO_MORE.equals(myNext); 1510 } 1511 1512 @Override 1513 public JpaPid next() { 1514 fetchNext(); 1515 JpaPid retVal = myNext; 1516 myNext = null; 1517 return retVal; 1518 } 1519 1520 } 1521 1522 private final class QueryIterator extends BaseIterator<JpaPid> implements IResultIterator<JpaPid> { 1523 1524 private final SearchRuntimeDetails mySearchRuntimeDetails; 1525 private final RequestDetails myRequest; 1526 private final boolean myHaveRawSqlHooks; 1527 private final boolean myHavePerfTraceFoundIdHook; 1528 private final SortSpec mySort; 1529 private final Integer myOffset; 1530 private boolean myFirst = true; 1531 private IncludesIterator myIncludesIterator; 1532 private JpaPid myNext; 1533 private ISearchQueryExecutor myResultsIterator; 1534 private boolean myFetchIncludesForEverythingOperation; 1535 private int mySkipCount = 0; 1536 private int myNonSkipCount = 0; 1537 private List<ISearchQueryExecutor> myQueryList = new ArrayList<>(); 1538 1539 private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) { 1540 mySearchRuntimeDetails = theSearchRuntimeDetails; 1541 mySort = myParams.getSort(); 1542 myOffset = myParams.getOffset(); 1543 myRequest = theRequest; 1544 1545 // Includes are processed inline for $everything query when we don't have a '_type' specified 1546 if (myParams.getEverythingMode() != null && !myParams.containsKey(Constants.PARAM_TYPE)) { 1547 myFetchIncludesForEverythingOperation = true; 1548 } 1549 1550 myHavePerfTraceFoundIdHook = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, myInterceptorBroadcaster, myRequest); 1551 myHaveRawSqlHooks = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, myRequest); 1552 1553 } 1554 1555 private void fetchNext() { 1556 1557 try { 1558 if (myHaveRawSqlHooks) { 1559 CurrentThreadCaptureQueriesListener.startCapturing(); 1560 } 1561 1562 // If we don't have a query yet, create one 1563 if (myResultsIterator == null) { 1564 if (myMaxResultsToFetch == null) { 1565 if (myParams.getLoadSynchronousUpTo() != null) { 1566 myMaxResultsToFetch = myParams.getLoadSynchronousUpTo(); 1567 } else if (myParams.getOffset() != null && myParams.getCount() != null) { 1568 myMaxResultsToFetch = myParams.getCount(); 1569 } else { 1570 myMaxResultsToFetch = myDaoConfig.getFetchSizeDefaultMaximum(); 1571 } 1572 } 1573 1574 initializeIteratorQuery(myOffset, myMaxResultsToFetch); 1575 1576 if (myAlsoIncludePids == null) { 1577 myAlsoIncludePids = new ArrayList<>(); 1578 } 1579 } 1580 1581 if (myNext == null) { 1582 1583 1584 for (Iterator<JpaPid> myPreResultsIterator = myAlsoIncludePids.iterator(); myPreResultsIterator.hasNext(); ) { 1585 JpaPid next = myPreResultsIterator.next(); 1586 if (next != null) 1587 if (myPidSet.add(next)) { 1588 myNext = next; 1589 break; 1590 } 1591 } 1592 1593 if (myNext == null) { 1594 while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) { 1595 // Update iterator with next chunk if necessary. 1596 if (!myResultsIterator.hasNext()) { 1597 retrieveNextIteratorQuery(); 1598 } 1599 1600 Long nextLong = myResultsIterator.next(); 1601 if (myHavePerfTraceFoundIdHook) { 1602 HookParams params = new HookParams() 1603 .add(Integer.class, System.identityHashCode(this)) 1604 .add(Object.class, nextLong); 1605 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, params); 1606 } 1607 1608 if (nextLong != null) { 1609 JpaPid next = JpaPid.fromId(nextLong); 1610 if (myPidSet.add(next)) { 1611 myNext = next; 1612 myNonSkipCount++; 1613 break; 1614 } else { 1615 mySkipCount++; 1616 } 1617 } 1618 1619 if (!myResultsIterator.hasNext()) { 1620 if (myMaxResultsToFetch != null && (mySkipCount + myNonSkipCount == myMaxResultsToFetch)) { 1621 if (mySkipCount > 0 && myNonSkipCount == 0) { 1622 1623 StorageProcessingMessage message = new StorageProcessingMessage(); 1624 String msg = "Pass completed with no matching results seeking rows " + myPidSet.size() + "-" + mySkipCount + ". This indicates an inefficient query! Retrying with new max count of " + myMaxResultsToFetch; 1625 ourLog.warn(msg); 1626 message.setMessage(msg); 1627 HookParams params = new HookParams() 1628 .add(RequestDetails.class, myRequest) 1629 .addIfMatchesType(ServletRequestDetails.class, myRequest) 1630 .add(StorageProcessingMessage.class, message); 1631 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_WARNING, params); 1632 1633 myMaxResultsToFetch += 1000; 1634 initializeIteratorQuery(myOffset, myMaxResultsToFetch); 1635 } 1636 } 1637 } 1638 } 1639 } 1640 1641 if (myNext == null) { 1642 if (myFetchIncludesForEverythingOperation) { 1643 myIncludesIterator = new IncludesIterator(myPidSet, myRequest); 1644 myFetchIncludesForEverythingOperation = false; 1645 } 1646 if (myIncludesIterator != null) { 1647 while (myIncludesIterator.hasNext()) { 1648 JpaPid next = myIncludesIterator.next(); 1649 if (next != null) 1650 if (myPidSet.add(next)) { 1651 myNext = next; 1652 break; 1653 } 1654 } 1655 if (myNext == null) { 1656 myNext = NO_MORE; 1657 } 1658 } else { 1659 myNext = NO_MORE; 1660 } 1661 } 1662 1663 } // if we need to fetch the next result 1664 1665 mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size()); 1666 1667 } finally { 1668 if (myHaveRawSqlHooks) { 1669 SqlQueryList capturedQueries = CurrentThreadCaptureQueriesListener.getCurrentQueueAndStopCapturing(); 1670 HookParams params = new HookParams() 1671 .add(RequestDetails.class, myRequest) 1672 .addIfMatchesType(ServletRequestDetails.class, myRequest) 1673 .add(SqlQueryList.class, capturedQueries); 1674 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_RAW_SQL, params); 1675 } 1676 } 1677 1678 if (myFirst) { 1679 HookParams params = new HookParams() 1680 .add(RequestDetails.class, myRequest) 1681 .addIfMatchesType(ServletRequestDetails.class, myRequest) 1682 .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); 1683 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FIRST_RESULT_LOADED, params); 1684 myFirst = false; 1685 } 1686 1687 if (NO_MORE.equals(myNext)) { 1688 HookParams params = new HookParams() 1689 .add(RequestDetails.class, myRequest) 1690 .addIfMatchesType(ServletRequestDetails.class, myRequest) 1691 .add(SearchRuntimeDetails.class, mySearchRuntimeDetails); 1692 CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_SELECT_COMPLETE, params); 1693 } 1694 1695 } 1696 1697 private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToFetch) { 1698 if (myQueryList.isEmpty()) { 1699 // Capture times for Lucene/Elasticsearch queries as well 1700 mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); 1701 myQueryList = createQuery(myParams, mySort, theOffset, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails); 1702 } 1703 1704 mySearchRuntimeDetails.setQueryStopwatch(new StopWatch()); 1705 1706 retrieveNextIteratorQuery(); 1707 1708 mySkipCount = 0; 1709 myNonSkipCount = 0; 1710 } 1711 1712 private void retrieveNextIteratorQuery() { 1713 close(); 1714 if (myQueryList != null && myQueryList.size() > 0) { 1715 myResultsIterator = myQueryList.remove(0); 1716 myHasNextIteratorQuery = true; 1717 } else { 1718 myResultsIterator = SearchQueryExecutor.emptyExecutor(); 1719 myHasNextIteratorQuery = false; 1720 } 1721 1722 } 1723 1724 @Override 1725 public boolean hasNext() { 1726 if (myNext == null) { 1727 fetchNext(); 1728 } 1729 return !NO_MORE.equals(myNext); 1730 } 1731 1732 @Override 1733 public JpaPid next() { 1734 fetchNext(); 1735 JpaPid retVal = myNext; 1736 myNext = null; 1737 Validate.isTrue(!NO_MORE.equals(retVal), "No more elements"); 1738 return retVal; 1739 } 1740 1741 @Override 1742 public int getSkippedCount() { 1743 return mySkipCount; 1744 } 1745 1746 @Override 1747 public int getNonSkippedCount() { 1748 return myNonSkipCount; 1749 } 1750 1751 @Override 1752 public Collection<JpaPid> getNextResultBatch(long theBatchSize) { 1753 Collection<JpaPid> batch = new ArrayList<>(); 1754 while (this.hasNext() && batch.size() < theBatchSize) { 1755 batch.add(this.next()); 1756 } 1757 return batch; 1758 } 1759 1760 @Override 1761 public void close() { 1762 if (myResultsIterator != null) { 1763 myResultsIterator.close(); 1764 } 1765 myResultsIterator = null; 1766 } 1767 1768 } 1769 1770 public static int getMaximumPageSize() { 1771 if (myUseMaxPageSize50ForTest) { 1772 return MAXIMUM_PAGE_SIZE_FOR_TESTING; 1773 } else { 1774 return MAXIMUM_PAGE_SIZE; 1775 } 1776 } 1777 1778 public static void setMaxPageSize50ForTest(boolean theIsTest) { 1779 myUseMaxPageSize50ForTest = theIsTest; 1780 } 1781 1782}