001package ca.uhn.fhir.jpa.search.builder;
002
003/*
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2023 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 * http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.context.ComboSearchParamType;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.context.FhirVersionEnum;
026import ca.uhn.fhir.context.RuntimeResourceDefinition;
027import ca.uhn.fhir.context.RuntimeSearchParam;
028import ca.uhn.fhir.i18n.Msg;
029import ca.uhn.fhir.interceptor.api.HookParams;
030import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
031import ca.uhn.fhir.interceptor.api.Pointcut;
032import ca.uhn.fhir.interceptor.model.RequestPartitionId;
033import ca.uhn.fhir.jpa.api.config.DaoConfig;
034import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
035import ca.uhn.fhir.jpa.api.dao.IDao;
036import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
037import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
038import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean;
039import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider;
040import ca.uhn.fhir.jpa.dao.BaseStorageDao;
041import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
042import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser;
043import ca.uhn.fhir.jpa.dao.IResultIterator;
044import ca.uhn.fhir.jpa.dao.ISearchBuilder;
045import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
046import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
047import ca.uhn.fhir.jpa.dao.search.ResourceNotFoundInIndexException;
048import ca.uhn.fhir.jpa.entity.ResourceSearchView;
049import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
050import ca.uhn.fhir.jpa.model.config.PartitionSettings;
051import ca.uhn.fhir.jpa.model.dao.JpaPid;
052import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity;
053import ca.uhn.fhir.jpa.model.entity.ModelConfig;
054import ca.uhn.fhir.jpa.model.entity.ResourceTag;
055import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
056import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
057import ca.uhn.fhir.jpa.search.SearchConstants;
058import ca.uhn.fhir.jpa.search.builder.sql.GeneratedSql;
059import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder;
060import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryExecutor;
061import ca.uhn.fhir.jpa.search.builder.sql.SqlObjectFactory;
062import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc;
063import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
064import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
065import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil;
066import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
067import ca.uhn.fhir.jpa.util.BaseIterator;
068import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
069import ca.uhn.fhir.jpa.util.QueryChunker;
070import ca.uhn.fhir.jpa.util.QueryParameterUtils;
071import ca.uhn.fhir.jpa.util.SqlQueryList;
072import ca.uhn.fhir.model.api.IQueryParameterType;
073import ca.uhn.fhir.model.api.IResource;
074import ca.uhn.fhir.model.api.Include;
075import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
076import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum;
077import ca.uhn.fhir.rest.api.Constants;
078import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
079import ca.uhn.fhir.rest.api.SearchContainedModeEnum;
080import ca.uhn.fhir.rest.api.SortOrderEnum;
081import ca.uhn.fhir.rest.api.SortSpec;
082import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
083import ca.uhn.fhir.rest.api.server.RequestDetails;
084import ca.uhn.fhir.rest.param.DateRangeParam;
085import ca.uhn.fhir.rest.param.ReferenceParam;
086import ca.uhn.fhir.rest.param.StringParam;
087import ca.uhn.fhir.rest.param.TokenParam;
088import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
089import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
090import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
091import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
092import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
093import ca.uhn.fhir.util.StopWatch;
094import ca.uhn.fhir.util.StringUtil;
095import ca.uhn.fhir.util.UrlUtil;
096import com.google.common.collect.Streams;
097import com.healthmarketscience.sqlbuilder.Condition;
098import org.apache.commons.lang3.Validate;
099import org.apache.commons.lang3.math.NumberUtils;
100import org.hl7.fhir.instance.model.api.IAnyResource;
101import org.hl7.fhir.instance.model.api.IBaseResource;
102import org.slf4j.Logger;
103import org.slf4j.LoggerFactory;
104import org.springframework.beans.factory.annotation.Autowired;
105import org.springframework.jdbc.core.JdbcTemplate;
106import org.springframework.jdbc.core.SingleColumnRowMapper;
107import org.springframework.transaction.support.TransactionSynchronizationManager;
108
109import javax.annotation.Nonnull;
110import javax.persistence.EntityManager;
111import javax.persistence.PersistenceContext;
112import javax.persistence.PersistenceContextType;
113import javax.persistence.Query;
114import javax.persistence.Tuple;
115import javax.persistence.TypedQuery;
116import javax.persistence.criteria.CriteriaBuilder;
117import java.util.ArrayList;
118import java.util.Collection;
119import java.util.Collections;
120import java.util.HashMap;
121import java.util.HashSet;
122import java.util.Iterator;
123import java.util.List;
124import java.util.Map;
125import java.util.Objects;
126import java.util.Optional;
127import java.util.Set;
128import java.util.stream.Collectors;
129
130import static org.apache.commons.lang3.StringUtils.defaultString;
131import static org.apache.commons.lang3.StringUtils.isBlank;
132import static org.apache.commons.lang3.StringUtils.isNotBlank;
133
134/**
135 * The SearchBuilder is responsible for actually forming the SQL query that handles
136 * searches for resources
137 */
138public class SearchBuilder implements ISearchBuilder<JpaPid> {
139
140        /**
141         * See loadResourcesByPid
142         * for an explanation of why we use the constant 800
143         */
144        // NB: keep public
145        @Deprecated
146        public static final int MAXIMUM_PAGE_SIZE = SearchConstants.MAX_PAGE_SIZE;
147        public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50;
148        public static final String RESOURCE_ID_ALIAS = "resource_id";
149        public static final String RESOURCE_VERSION_ALIAS = "resource_version";
150        private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class);
151        private static final JpaPid NO_MORE = JpaPid.fromId(-1L);
152        private static final String MY_TARGET_RESOURCE_PID = "myTargetResourcePid";
153        private static final String MY_SOURCE_RESOURCE_PID = "mySourceResourcePid";
154        private static final String MY_TARGET_RESOURCE_TYPE = "myTargetResourceType";
155        private static final String MY_SOURCE_RESOURCE_TYPE = "mySourceResourceType";
156        private static final String MY_TARGET_RESOURCE_VERSION = "myTargetResourceVersion";
157        public static boolean myUseMaxPageSize50ForTest = false;
158        protected final IInterceptorBroadcaster myInterceptorBroadcaster;
159        protected final IResourceTagDao myResourceTagDao;
160        private final String myResourceName;
161        private final Class<? extends IBaseResource> myResourceType;
162        private final HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory;
163        private final SqlObjectFactory mySqlBuilderFactory;
164        private final HibernatePropertiesProvider myDialectProvider;
165        private final ModelConfig myModelConfig;
166        private final ISearchParamRegistry mySearchParamRegistry;
167        private final PartitionSettings myPartitionSettings;
168        private final DaoRegistry myDaoRegistry;
169        private final IResourceSearchViewDao myResourceSearchViewDao;
170        private final FhirContext myContext;
171        private final IIdHelperService<JpaPid> myIdHelperService;
172        private final DaoConfig myDaoConfig;
173        private final IDao myCallingDao;
174        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
175        protected EntityManager myEntityManager;
176        private List<JpaPid> myAlsoIncludePids;
177        private CriteriaBuilder myCriteriaBuilder;
178        private SearchParameterMap myParams;
179        private String mySearchUuid;
180        private int myFetchSize;
181        private Integer myMaxResultsToFetch;
182        private Set<JpaPid> myPidSet;
183        private boolean myHasNextIteratorQuery = false;
184        private RequestPartitionId myRequestPartitionId;
185        @Autowired(required = false)
186        private IFulltextSearchSvc myFulltextSearchSvc;
187        @Autowired(required = false)
188        private IElasticsearchSvc myIElasticsearchSvc;
189        @Autowired
190        private IJpaStorageResourceParser myJpaStorageResourceParser;
191
192        /**
193         * Constructor
194         */
195        public SearchBuilder(
196                IDao theDao,
197                String theResourceName,
198                DaoConfig theDaoConfig,
199                HapiFhirLocalContainerEntityManagerFactoryBean theEntityManagerFactory,
200                SqlObjectFactory theSqlBuilderFactory,
201                HibernatePropertiesProvider theDialectProvider,
202                ModelConfig theModelConfig,
203                ISearchParamRegistry theSearchParamRegistry,
204                PartitionSettings thePartitionSettings,
205                IInterceptorBroadcaster theInterceptorBroadcaster,
206                IResourceTagDao theResourceTagDao,
207                DaoRegistry theDaoRegistry,
208                IResourceSearchViewDao theResourceSearchViewDao,
209                FhirContext theContext,
210                IIdHelperService theIdHelperService,
211                Class<? extends IBaseResource> theResourceType
212        ) {
213                myCallingDao = theDao;
214                myResourceName = theResourceName;
215                myResourceType = theResourceType;
216                myDaoConfig = theDaoConfig;
217
218                myEntityManagerFactory = theEntityManagerFactory;
219                mySqlBuilderFactory = theSqlBuilderFactory;
220                myDialectProvider = theDialectProvider;
221                myModelConfig = theModelConfig;
222                mySearchParamRegistry = theSearchParamRegistry;
223                myPartitionSettings = thePartitionSettings;
224                myInterceptorBroadcaster = theInterceptorBroadcaster;
225                myResourceTagDao = theResourceTagDao;
226                myDaoRegistry = theDaoRegistry;
227                myResourceSearchViewDao = theResourceSearchViewDao;
228                myContext = theContext;
229                myIdHelperService = theIdHelperService;
230        }
231
232        @Override
233        public void setMaxResultsToFetch(Integer theMaxResultsToFetch) {
234                myMaxResultsToFetch = theMaxResultsToFetch;
235        }
236
237        private void searchForIdsWithAndOr(SearchQueryBuilder theSearchSqlBuilder, QueryStack theQueryStack, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
238                myParams = theParams;
239
240                // Remove any empty parameters
241                theParams.clean();
242
243                // For DSTU3, pull out near-distance first so when it comes time to evaluate near, we already know the distance
244                if (myContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) {
245                        Dstu3DistanceHelper.setNearDistance(myResourceType, theParams);
246                }
247
248                // Attempt to lookup via composite unique key.
249                if (isCompositeUniqueSpCandidate()) {
250                        attemptComboUniqueSpProcessing(theQueryStack, theParams, theRequest);
251                }
252
253                SearchContainedModeEnum searchContainedMode = theParams.getSearchContainedMode();
254
255                // Handle _id and _tag last, since they can typically be tacked onto a different parameter
256                List<String> paramNames = myParams.keySet().stream().filter(t -> !t.equals(IAnyResource.SP_RES_ID))
257                        .filter(t -> !t.equals(Constants.PARAM_TAG)).collect(Collectors.toList());
258                if (myParams.containsKey(IAnyResource.SP_RES_ID)) {
259                        paramNames.add(IAnyResource.SP_RES_ID);
260                }
261                if (myParams.containsKey(Constants.PARAM_TAG)) {
262                        paramNames.add(Constants.PARAM_TAG);
263                }
264
265                // Handle each parameter
266                for (String nextParamName : paramNames) {
267                        if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) {
268                                // Skip parameters for Subject, Patient, Code and Category for LastN as these will be filtered by Elasticsearch
269                                continue;
270                        }
271                        List<List<IQueryParameterType>> andOrParams = myParams.get(nextParamName);
272                        Condition predicate = theQueryStack.searchForIdsWithAndOr(null, myResourceName, nextParamName, andOrParams, theRequest, myRequestPartitionId, searchContainedMode);
273                        if (predicate != null) {
274                                theSearchSqlBuilder.addPredicate(predicate);
275                        }
276                }
277        }
278
279        /**
280         * A search is a candidate for Composite Unique SP if unique indexes are enabled, there is no EverythingMode, and the
281         * parameters all have no modifiers.
282         */
283        private boolean isCompositeUniqueSpCandidate() {
284                return myDaoConfig.isUniqueIndexesEnabled() &&
285                        myParams.getEverythingMode() == null &&
286                        myParams.isAllParametersHaveNoModifier();
287        }
288
289        @SuppressWarnings("ConstantConditions")
290        @Override
291        public Long createCountQuery(SearchParameterMap theParams, String theSearchUuid,
292                                                                                  RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) {
293
294                assert theRequestPartitionId != null;
295                assert TransactionSynchronizationManager.isActualTransactionActive();
296
297                init(theParams, theSearchUuid, theRequestPartitionId);
298
299                if (checkUseHibernateSearch()) {
300                        long count = myFulltextSearchSvc.count(myResourceName, theParams.clone());
301                        return count;
302                }
303
304                List<ISearchQueryExecutor> queries = createQuery(theParams.clone(), null, null, null, true, theRequest, null);
305                if (queries.isEmpty()) {
306                        return 0L;
307                } else {
308                        return queries.get(0).next();
309                }
310        }
311
312        /**
313         * @param thePidSet May be null
314         */
315        @Override
316        public void setPreviouslyAddedResourcePids(@Nonnull List<JpaPid> thePidSet) {
317                myPidSet = new HashSet<>(thePidSet);
318        }
319
320        @SuppressWarnings("ConstantConditions")
321        @Override
322        public IResultIterator createQuery(SearchParameterMap theParams, SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) {
323                assert theRequestPartitionId != null;
324                assert TransactionSynchronizationManager.isActualTransactionActive();
325
326                init(theParams, theSearchRuntimeDetails.getSearchUuid(), theRequestPartitionId);
327
328                if (myPidSet == null) {
329                        myPidSet = new HashSet<>();
330                }
331
332                return new QueryIterator(theSearchRuntimeDetails, theRequest);
333        }
334
335
336        private void init(SearchParameterMap theParams, String theSearchUuid, RequestPartitionId theRequestPartitionId) {
337                myCriteriaBuilder = myEntityManager.getCriteriaBuilder();
338                // we mutate the params.  Make a private copy.
339                myParams = theParams.clone();
340                mySearchUuid = theSearchUuid;
341                myRequestPartitionId = theRequestPartitionId;
342        }
343
344        private List<ISearchQueryExecutor> createQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCountOnlyFlag, RequestDetails theRequest,
345                                                                                                                                  SearchRuntimeDetails theSearchRuntimeDetails) {
346
347                ArrayList<ISearchQueryExecutor> queries = new ArrayList<>();
348
349                if (checkUseHibernateSearch()) {
350                        // we're going to run at least part of the search against the Fulltext service.
351
352                        // Ugh - we have two different return types for now
353                        ISearchQueryExecutor fulltextExecutor = null;
354                        List<JpaPid> fulltextMatchIds = null;
355                        int resultCount = 0;
356                        if (myParams.isLastN()) {
357                                fulltextMatchIds = executeLastNAgainstIndex(theMaximumResults);
358                                resultCount = fulltextMatchIds.size();
359                        } else if (myParams.getEverythingMode() != null) {
360                                fulltextMatchIds = queryHibernateSearchForEverythingPids();
361                                resultCount = fulltextMatchIds.size();
362                        } else {
363                                fulltextExecutor = myFulltextSearchSvc.searchNotScrolled(myResourceName, myParams, myMaxResultsToFetch);
364                        }
365
366                        if (fulltextExecutor == null) {
367                                fulltextExecutor = SearchQueryExecutors.from(fulltextMatchIds);
368                        }
369
370                        if (theSearchRuntimeDetails != null) {
371                                theSearchRuntimeDetails.setFoundIndexMatchesCount(resultCount);
372                                HookParams params = new HookParams()
373                                        .add(RequestDetails.class, theRequest)
374                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
375                                        .add(SearchRuntimeDetails.class, theSearchRuntimeDetails);
376                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params);
377                        }
378
379                        // can we skip the database entirely and return the pid list from here?
380                        boolean canSkipDatabase =
381                                // if we processed an AND clause, and it returned nothing, then nothing can match.
382                                !fulltextExecutor.hasNext() ||
383                                        // Our hibernate search query doesn't respect partitions yet
384                                        (!myPartitionSettings.isPartitioningEnabled() &&
385                                                // were there AND terms left?  Then we still need the db.
386                                                theParams.isEmpty() &&
387                                                // not every param is a param. :-(
388                                                theParams.getNearDistanceParam() == null &&
389                                                // todo MB don't we support _lastUpdated and _offset now?
390                                                theParams.getLastUpdated() == null &&
391                                                theParams.getEverythingMode() == null &&
392                                                theParams.getOffset() == null
393                                        );
394
395                        if (canSkipDatabase) {
396                                ourLog.trace("Query finished after HSearch.  Skip db query phase");
397                                if (theMaximumResults != null) {
398                                        fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theMaximumResults);
399                                }
400                                queries.add(fulltextExecutor);
401                        } else {
402                                ourLog.trace("Query needs db after HSearch.  Chunking.");
403                                // Finish the query in the database for the rest of the search parameters, sorting, partitioning, etc.
404                                // We break the pids into chunks that fit in the 1k limit for jdbc bind params.
405                                new QueryChunker<Long>()
406                                        .chunk(Streams.stream(fulltextExecutor).collect(Collectors.toList()), t -> doCreateChunkedQueries(theParams, t, theOffset, sort, theCountOnlyFlag, theRequest, queries));
407                        }
408                } else {
409                        // do everything in the database.
410                        Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, theMaximumResults, theCountOnlyFlag, theRequest, null);
411                        query.ifPresent(queries::add);
412                }
413
414                return queries;
415        }
416
417        /**
418         * Check to see if query should use Hibernate Search, and error if the query can't continue.
419         *
420         * @return true if the query should first be processed by Hibernate Search
421         * @throws InvalidRequestException if fulltext search is not enabled but the query requires it - _content or _text
422         */
423        private boolean checkUseHibernateSearch() {
424                boolean fulltextEnabled = (myFulltextSearchSvc != null) && !myFulltextSearchSvc.isDisabled();
425
426                if (!fulltextEnabled) {
427                        failIfUsed(Constants.PARAM_TEXT);
428                        failIfUsed(Constants.PARAM_CONTENT);
429                }
430
431                // someday we'll want a query planner to figure out if we _should_ or _must_ use the ft index, not just if we can.
432                return fulltextEnabled && myParams != null &&
433                        myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE &&
434                        myFulltextSearchSvc.supportsSomeOf(myParams);
435        }
436
437        private void failIfUsed(String theParamName) {
438                if (myParams.containsKey(theParamName)) {
439                        throw new InvalidRequestException(Msg.code(1192) + "Fulltext search is not enabled on this service, can not process parameter: " + theParamName);
440                }
441        }
442
443        private List<JpaPid> executeLastNAgainstIndex(Integer theMaximumResults) {
444                // Can we use our hibernate search generated index on resource to support lastN?:
445                if (myDaoConfig.isAdvancedHSearchIndexing()) {
446                        if (myFulltextSearchSvc == null) {
447                                throw new InvalidRequestException(Msg.code(2027) + "LastN operation is not enabled on this service, can not process this request");
448                        }
449                        return myFulltextSearchSvc.lastN(myParams, theMaximumResults)
450                                .stream().map(lastNResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, String.valueOf(lastNResourceId)))
451                                .collect(Collectors.toList());
452                } else {
453                        if (myIElasticsearchSvc == null) {
454                                throw new InvalidRequestException(Msg.code(2033) + "LastN operation is not enabled on this service, can not process this request");
455                        }
456                        // use the dedicated observation ES/Lucene index to support lastN query
457                        return myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults).stream()
458                                .map(lastnResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId))
459                                .collect(Collectors.toList());
460                }
461        }
462
463        private List<JpaPid> queryHibernateSearchForEverythingPids() {
464                JpaPid pid = null;
465                if (myParams.get(IAnyResource.SP_RES_ID) != null) {
466                        String idParamValue;
467                        IQueryParameterType idParam = myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
468                        if (idParam instanceof TokenParam) {
469                                TokenParam idParm = (TokenParam) idParam;
470                                idParamValue = idParm.getValue();
471                        } else {
472                                StringParam idParm = (StringParam) idParam;
473                                idParamValue = idParm.getValue();
474                        }
475
476                        pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
477                }
478                List<JpaPid> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid);
479                return pids;
480        }
481
482        private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<ISearchQueryExecutor> theQueries) {
483                if (thePids.size() < getMaximumPageSize()) {
484                        normalizeIdListForLastNInClause(thePids);
485                }
486                Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, thePids.size(), theCount, theRequest, thePids);
487                query.ifPresent(t -> theQueries.add(t));
488        }
489
490        /**
491         * Combs through the params for any _id parameters and extracts the PIDs for them
492         *
493         * @param theTargetPids
494         */
495        private void extractTargetPidsFromIdParams(HashSet<Long> theTargetPids) {
496                // get all the IQueryParameterType objects
497                // for _id -> these should all be StringParam values
498                HashSet<String> ids = new HashSet<>();
499                List<List<IQueryParameterType>> params = myParams.get(IAnyResource.SP_RES_ID);
500                for (List<IQueryParameterType> paramList : params) {
501                        for (IQueryParameterType param : paramList) {
502                                if (param instanceof StringParam) {
503                                        // we expect all _id values to be StringParams
504                                        ids.add(((StringParam) param).getValue());
505                                } else if (param instanceof TokenParam) {
506                                        ids.add(((TokenParam) param).getValue());
507                                } else {
508                                        // we do not expect the _id parameter to be a non-string value
509                                        throw new IllegalArgumentException(Msg.code(1193) + "_id parameter must be a StringParam or TokenParam");
510                                }
511                        }
512                }
513
514                // fetch our target Pids
515                // this will throw if an id is not found
516                Map<String, JpaPid> idToPid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId,
517                        myResourceName,
518                        new ArrayList<>(ids));
519                if (myAlsoIncludePids == null) {
520                        myAlsoIncludePids = new ArrayList<>();
521                }
522
523                // add the pids to targetPids
524                for (JpaPid pid : idToPid.values()) {
525                        myAlsoIncludePids.add(pid);
526                        theTargetPids.add(pid.getId());
527                }
528        }
529
530        private Optional<SearchQueryExecutor> createChunkedQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCountOnlyFlag, RequestDetails theRequest, List<Long> thePidList) {
531                String sqlBuilderResourceName = myParams.getEverythingMode() == null ? myResourceName : null;
532                SearchQueryBuilder sqlBuilder = new SearchQueryBuilder(myContext, myDaoConfig.getModelConfig(), myPartitionSettings, myRequestPartitionId, sqlBuilderResourceName, mySqlBuilderFactory, myDialectProvider, theCountOnlyFlag);
533                QueryStack queryStack3 = new QueryStack(theParams, myDaoConfig, myDaoConfig.getModelConfig(), myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings);
534
535                if (theParams.keySet().size() > 1 || theParams.getSort() != null || theParams.keySet().contains(Constants.PARAM_HAS) || isPotentiallyContainedReferenceParameterExistsAtRoot(theParams)) {
536                        List<RuntimeSearchParam> activeComboParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
537                        if (activeComboParams.isEmpty()) {
538                                sqlBuilder.setNeedResourceTableRoot(true);
539                        }
540                }
541
542                JdbcTemplate jdbcTemplate = new JdbcTemplate(myEntityManagerFactory.getDataSource());
543                jdbcTemplate.setFetchSize(myFetchSize);
544                if (theMaximumResults != null) {
545                        jdbcTemplate.setMaxRows(theMaximumResults);
546                }
547
548                if (myParams.getEverythingMode() != null) {
549                        HashSet<Long> targetPids = new HashSet<>();
550                        if (myParams.get(IAnyResource.SP_RES_ID) != null) {
551                                extractTargetPidsFromIdParams(targetPids);
552                        } else {
553                                // For Everything queries, we make the query root by the ResourceLink table, since this query
554                                // is basically a reverse-include search. For type/Everything (as opposed to instance/Everything)
555                                // the one problem with this approach is that it doesn't catch Patients that have absolutely
556                                // nothing linked to them. So we do one additional query to make sure we catch those too.
557                                SearchQueryBuilder fetchPidsSqlBuilder = new SearchQueryBuilder(myContext, myDaoConfig.getModelConfig(), myPartitionSettings, myRequestPartitionId, myResourceName, mySqlBuilderFactory, myDialectProvider, theCountOnlyFlag);
558                                GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theOffset, myMaxResultsToFetch);
559                                String sql = allTargetsSql.getSql();
560                                Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]);
561                                List<Long> output = jdbcTemplate.query(sql, args, new SingleColumnRowMapper<>(Long.class));
562                                if (myAlsoIncludePids == null) {
563                                        myAlsoIncludePids = new ArrayList<>(output.size());
564                                }
565                                myAlsoIncludePids.addAll(JpaPid.fromLongList(output));
566
567                        }
568
569                        List<String> typeSourceResources = new ArrayList<>();
570                        if (myParams.get(Constants.PARAM_TYPE) != null) {
571                                typeSourceResources.addAll(extractTypeSourceResourcesFromParams());
572                        }
573
574                        queryStack3.addPredicateEverythingOperation(myResourceName, typeSourceResources, targetPids.toArray(new Long[0]));
575                } else {
576                        /*
577                         * If we're doing a filter, always use the resource table as the root - This avoids the possibility of
578                         * specific filters with ORs as their root from working around the natural resource type / deletion
579                         * status / partition IDs built into queries.
580                         */
581                        if (theParams.containsKey(Constants.PARAM_FILTER)) {
582                                Condition partitionIdPredicate = sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId);
583                                if (partitionIdPredicate != null) {
584                                        sqlBuilder.addPredicate(partitionIdPredicate);
585                                }
586                        }
587
588                        // Normal search
589                        searchForIdsWithAndOr(sqlBuilder, queryStack3, myParams, theRequest);
590                }
591
592                // If we haven't added any predicates yet, we're doing a search for all resources. Make sure we add the
593                // partition ID predicate in that case.
594                if (!sqlBuilder.haveAtLeastOnePredicate()) {
595                        Condition partitionIdPredicate = sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId);
596                        if (partitionIdPredicate != null) {
597                                sqlBuilder.addPredicate(partitionIdPredicate);
598                        }
599                }
600
601                // Add PID list predicate for full text search and/or lastn operation
602                if (thePidList != null && thePidList.size() > 0) {
603                        sqlBuilder.addResourceIdsPredicate(thePidList);
604                }
605
606                // Last updated
607                DateRangeParam lu = myParams.getLastUpdated();
608                if (lu != null && !lu.isEmpty()) {
609                        Condition lastUpdatedPredicates = sqlBuilder.addPredicateLastUpdated(lu);
610                        sqlBuilder.addPredicate(lastUpdatedPredicates);
611                }
612
613                /*
614                 * Exclude the pids already in the previous iterator. This is an optimization, as opposed
615                 * to something needed to guarantee correct results.
616                 *
617                 * Why do we need it? Suppose for example, a query like:
618                 *    Observation?category=foo,bar,baz
619                 * And suppose you have many resources that have all 3 of these category codes. In this case
620                 * the SQL query will probably return the same PIDs multiple times, and if this happens enough
621                 * we may exhaust the query results without getting enough distinct results back. When that
622                 * happens we re-run the query with a larger limit. Excluding results we already know about
623                 * tries to ensure that we get new unique results.
624                 *
625                 * The challenge with that though is that lots of DBs have an issue with too many
626                 * parameters in one query. So we only do this optimization if there aren't too
627                 * many results.
628                 */
629                if (myHasNextIteratorQuery) {
630                        if (myPidSet.size() + sqlBuilder.countBindVariables() < 900) {
631                                sqlBuilder.excludeResourceIdsPredicate(myPidSet);
632                        }
633                }
634
635                /*
636                 * If offset is present, we want deduplicate the results by using GROUP BY
637                 */
638                if (theOffset != null) {
639                        queryStack3.addGrouping();
640                        queryStack3.setUseAggregate(true);
641                }
642
643                /*
644                 * Sort
645                 *
646                 * If we have a sort, we wrap the criteria search (the search that actually
647                 * finds the appropriate resources) in an outer search which is then sorted
648                 */
649                if (sort != null) {
650                        assert !theCountOnlyFlag;
651
652                        createSort(queryStack3, sort);
653                }
654
655
656                /*
657                 * Now perform the search
658                 */
659                GeneratedSql generatedSql = sqlBuilder.generate(theOffset, myMaxResultsToFetch);
660                if (generatedSql.isMatchNothing()) {
661                        return Optional.empty();
662                }
663
664                SearchQueryExecutor executor = mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, myMaxResultsToFetch);
665                return Optional.of(executor);
666        }
667
668        private Collection<String> extractTypeSourceResourcesFromParams() {
669
670                List<List<IQueryParameterType>> listOfList = myParams.get(Constants.PARAM_TYPE);
671
672                // first off, let's flatten the list of list
673                List<IQueryParameterType> iQueryParameterTypesList = listOfList.stream().flatMap(List::stream).collect(Collectors.toList());
674
675                // then, extract all elements of each CSV into one big list
676                List<String> resourceTypes = iQueryParameterTypesList
677                        .stream()
678                        .map(param -> ((StringParam) param).getValue())
679                        .map(csvString -> List.of(csvString.split(",")))
680                        .flatMap(List::stream).collect(Collectors.toList());
681
682                Set<String> knownResourceTypes = myContext.getResourceTypes();
683
684                // remove leading/trailing whitespaces if any and remove duplicates
685                Set<String> retVal = new HashSet<>();
686
687                for (String type : resourceTypes) {
688                        String trimmed = type.trim();
689                        if (!knownResourceTypes.contains(trimmed)) {
690                                throw new ResourceNotFoundException(Msg.code(2197) + "Unknown resource type '" + trimmed + "' in _type parameter.");
691                        }
692                        retVal.add(trimmed);
693                }
694
695                return retVal;
696        }
697
698        private boolean isPotentiallyContainedReferenceParameterExistsAtRoot(SearchParameterMap theParams) {
699                return myModelConfig.isIndexOnContainedResources() && theParams.values().stream()
700                        .flatMap(Collection::stream)
701                        .flatMap(Collection::stream)
702                        .anyMatch(t -> t instanceof ReferenceParam);
703        }
704
705        private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
706                /*
707                        The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying
708                        numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info:
709                        https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage.
710
711                        Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of
712                        arguments never exceeds the maximum specified below.
713                 */
714                int listSize = lastnResourceIds.size();
715
716                if (listSize > 1 && listSize < 10) {
717                        padIdListWithPlaceholders(lastnResourceIds, 10);
718                } else if (listSize > 10 && listSize < 50) {
719                        padIdListWithPlaceholders(lastnResourceIds, 50);
720                } else if (listSize > 50 && listSize < 100) {
721                        padIdListWithPlaceholders(lastnResourceIds, 100);
722                } else if (listSize > 100 && listSize < 200) {
723                        padIdListWithPlaceholders(lastnResourceIds, 200);
724                } else if (listSize > 200 && listSize < 500) {
725                        padIdListWithPlaceholders(lastnResourceIds, 500);
726                } else if (listSize > 500 && listSize < 800) {
727                        padIdListWithPlaceholders(lastnResourceIds, 800);
728                }
729
730                return lastnResourceIds;
731        }
732
733        private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
734                while (theIdList.size() < preferredListSize) {
735                        theIdList.add(-1L);
736                }
737        }
738
739        private void createSort(QueryStack theQueryStack, SortSpec theSort) {
740                if (theSort == null || isBlank(theSort.getParamName())) {
741                        return;
742                }
743
744                boolean ascending = (theSort.getOrder() == null) || (theSort.getOrder() == SortOrderEnum.ASC);
745
746                if (IAnyResource.SP_RES_ID.equals(theSort.getParamName())) {
747
748                        theQueryStack.addSortOnResourceId(ascending);
749
750                } else if (Constants.PARAM_LASTUPDATED.equals(theSort.getParamName())) {
751
752                        theQueryStack.addSortOnLastUpdated(ascending);
753
754                } else {
755
756                        RuntimeSearchParam param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName());
757                        if (param == null) {
758                                String msg = myContext.getLocalizer().getMessageSanitized(BaseStorageDao.class, "invalidSortParameter", theSort.getParamName(), getResourceName(), mySearchParamRegistry.getValidSearchParameterNamesIncludingMeta(getResourceName()));
759                                throw new InvalidRequestException(Msg.code(1194) + msg);
760                        }
761
762                        switch (param.getParamType()) {
763                                case STRING:
764                                        theQueryStack.addSortOnString(myResourceName, theSort.getParamName(), ascending);
765                                        break;
766                                case DATE:
767                                        theQueryStack.addSortOnDate(myResourceName, theSort.getParamName(), ascending);
768                                        break;
769                                case REFERENCE:
770                                        theQueryStack.addSortOnResourceLink(myResourceName, theSort.getParamName(), ascending);
771                                        break;
772                                case TOKEN:
773                                        theQueryStack.addSortOnToken(myResourceName, theSort.getParamName(), ascending);
774                                        break;
775                                case NUMBER:
776                                        theQueryStack.addSortOnNumber(myResourceName, theSort.getParamName(), ascending);
777                                        break;
778                                case URI:
779                                        theQueryStack.addSortOnUri(myResourceName, theSort.getParamName(), ascending);
780                                        break;
781                                case QUANTITY:
782                                        theQueryStack.addSortOnQuantity(myResourceName, theSort.getParamName(), ascending);
783                                        break;
784                                case COMPOSITE:
785                                        List<RuntimeSearchParam> compositeList = JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, param);
786                                        if (compositeList == null) {
787                                                throw new InvalidRequestException(Msg.code(1195) + "The composite _sort parameter " + theSort.getParamName() + " is not defined by the resource " + myResourceName);
788                                        }
789                                        if (compositeList.size() != 2) {
790                                                throw new InvalidRequestException(Msg.code(1196) + "The composite _sort parameter " + theSort.getParamName()
791                                                        + " must have 2 composite types declared in parameter annotation, found "
792                                                        + compositeList.size());
793                                        }
794                                        RuntimeSearchParam left = compositeList.get(0);
795                                        RuntimeSearchParam right = compositeList.get(1);
796
797                                        createCompositeSort(theQueryStack, left.getParamType(), left.getName(), ascending);
798                                        createCompositeSort(theQueryStack, right.getParamType(), right.getName(), ascending);
799
800                                        break;
801                                case SPECIAL:
802                                case HAS:
803                                default:
804                                        throw new InvalidRequestException(Msg.code(1197) + "This server does not support _sort specifications of type " + param.getParamType() + " - Can't serve _sort=" + theSort.getParamName());
805                        }
806
807                }
808
809                // Recurse
810                createSort(theQueryStack, theSort.getChain());
811
812        }
813
814        private void createCompositeSort(QueryStack theQueryStack, RestSearchParameterTypeEnum theParamType, String theParamName, boolean theAscending) {
815
816                switch (theParamType) {
817                        case STRING:
818                                theQueryStack.addSortOnString(myResourceName, theParamName, theAscending);
819                                break;
820                        case DATE:
821                                theQueryStack.addSortOnDate(myResourceName, theParamName, theAscending);
822                                break;
823                        case TOKEN:
824                                theQueryStack.addSortOnToken(myResourceName, theParamName, theAscending);
825                                break;
826                        case QUANTITY:
827                                theQueryStack.addSortOnQuantity(myResourceName, theParamName, theAscending);
828                                break;
829                        case NUMBER:
830                        case REFERENCE:
831                        case COMPOSITE:
832                        case URI:
833                        case HAS:
834                        case SPECIAL:
835                        default:
836                                throw new InvalidRequestException(Msg.code(1198) + "Don't know how to handle composite parameter with type of " + theParamType + " on _sort=" + theParamName);
837                }
838
839        }
840
841        private void doLoadPids(Collection<JpaPid> thePids, Collection<JpaPid> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation,
842                                                                        Map<JpaPid, Integer> thePosition) {
843
844                Map<Long, Long> resourcePidToVersion = null;
845                for (JpaPid next : thePids) {
846                        if (next.getVersion() != null && myModelConfig.isRespectVersionsForSearchIncludes()) {
847                                if (resourcePidToVersion == null) {
848                                        resourcePidToVersion = new HashMap<>();
849                                }
850                                resourcePidToVersion.put((next).getId(), next.getVersion());
851                        }
852                }
853
854                List<Long> versionlessPids = JpaPid.toLongList(thePids);
855                if (versionlessPids.size() < getMaximumPageSize()) {
856                        versionlessPids = normalizeIdListForLastNInClause(versionlessPids);
857                }
858
859                // -- get the resource from the searchView
860                Collection<ResourceSearchView> resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(versionlessPids);
861
862                //-- preload all tags with tag definition if any
863                Map<Long, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList);
864
865                for (IBaseResourceEntity next : resourceSearchViewList) {
866                        if (next.getDeleted() != null) {
867                                continue;
868                        }
869
870                        Class<? extends IBaseResource> resourceType = myContext.getResourceDefinition(next.getResourceType()).getImplementingClass();
871
872                        JpaPid resourceId = JpaPid.fromId(next.getResourceId());
873
874                        /*
875                         * If a specific version is requested via an include, we'll replace the current version
876                         * with the specific desired version. This is not the most efficient thing, given that
877                         * we're loading the current version and then turning around and throwing it away again.
878                         * This could be optimized and probably should be, but it's not critical given that
879                         * this only applies to includes, which don't tend to be massive in numbers.
880                         */
881                        if (resourcePidToVersion != null) {
882                                Long version = resourcePidToVersion.get(next.getResourceId());
883                                resourceId.setVersion(version);
884                                if (version != null && !version.equals(next.getVersion())) {
885                                        IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(resourceType);
886                                        next = dao.readEntity(next.getIdDt().withVersion(Long.toString(version)), null);
887                                }
888                        }
889
890                        IBaseResource resource = null;
891                        if (next != null) {
892                                resource = myJpaStorageResourceParser.toResource(resourceType, next, tagMap.get(next.getId()), theForHistoryOperation);
893                        }
894                        if (resource == null) {
895                                ourLog.warn("Unable to find resource {}/{}/_history/{} in database", next.getResourceType(), next.getIdDt().getIdPart(), next.getVersion());
896                                continue;
897                        }
898
899                        Integer index = thePosition.get(resourceId);
900                        if (index == null) {
901                                ourLog.warn("Got back unexpected resource PID {}", resourceId);
902                                continue;
903                        }
904
905                        if (theIncludedPids.contains(resourceId)) {
906                                ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put(resource, BundleEntrySearchModeEnum.INCLUDE);
907                        } else {
908                                ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put(resource, BundleEntrySearchModeEnum.MATCH);
909                        }
910
911                        theResourceListToPopulate.set(index, resource);
912                }
913        }
914
915        private Map<Long, Collection<ResourceTag>> getResourceTagMap(Collection<? extends IBaseResourceEntity> theResourceSearchViewList) {
916
917                List<Long> idList = new ArrayList<>(theResourceSearchViewList.size());
918
919                //-- find all resource has tags
920                for (IBaseResourceEntity resource : theResourceSearchViewList) {
921                        if (resource.isHasTags())
922                                idList.add(resource.getId());
923                }
924
925                return getPidToTagMap(idList);
926        }
927
928        @Nonnull
929        private Map<Long, Collection<ResourceTag>> getPidToTagMap(List<Long> thePidList) {
930                Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>();
931
932                //-- no tags
933                if (thePidList.size() == 0)
934                        return tagMap;
935
936                //-- get all tags for the idList
937                Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(thePidList);
938
939                //-- build the map, key = resourceId, value = list of ResourceTag
940                JpaPid resourceId;
941                Collection<ResourceTag> tagCol;
942                for (ResourceTag tag : tagList) {
943
944                        resourceId = JpaPid.fromId(tag.getResourceId());
945                        tagCol = tagMap.get(resourceId.getId());
946                        if (tagCol == null) {
947                                tagCol = new ArrayList<>();
948                                tagCol.add(tag);
949                                tagMap.put(resourceId.getId(), tagCol);
950                        } else {
951                                tagCol.add(tag);
952                        }
953                }
954
955                return tagMap;
956        }
957
958        @Override
959        public void loadResourcesByPid(Collection<JpaPid> thePids, Collection<JpaPid> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation, RequestDetails theDetails) {
960                if (thePids.isEmpty()) {
961                        ourLog.debug("The include pids are empty");
962                        // return;
963                }
964
965                // Dupes will cause a crash later anyhow, but this is expensive so only do it
966                // when running asserts
967                assert new HashSet<>(thePids).size() == thePids.size() : "PID list contains duplicates: " + thePids;
968
969                Map<JpaPid, Integer> position = new HashMap<>();
970                for (JpaPid next : thePids) {
971                        position.put(next, theResourceListToPopulate.size());
972                        theResourceListToPopulate.add(null);
973                }
974
975                // Can we fast track this loading by checking elastic search?
976                if (isLoadingFromElasticSearchSupported(thePids)) {
977                        try {
978                                theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids));
979                                return;
980
981                        } catch (ResourceNotFoundInIndexException theE) {
982                                // some resources were not found in index, so we will inform this and resort to JPA search
983                                ourLog.warn("Some resources were not found in index. Make sure all resources were indexed. Resorting to database search.");
984                        }
985                }
986
987                // We only chunk because some jdbc drivers can't handle long param lists.
988                new QueryChunker<JpaPid>().chunk(thePids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
989        }
990
991        /**
992         * Check if we can load the resources from Hibernate Search instead of the database.
993         * We assume this is faster.
994         * <p>
995         * Hibernate Search only stores the current version, and only if enabled.
996         *
997         * @param thePids the pids to check for versioned references
998         * @return can we fetch from Hibernate Search?
999         */
1000        private boolean isLoadingFromElasticSearchSupported(Collection<JpaPid> thePids) {
1001                // is storage enabled?
1002                return myDaoConfig.isStoreResourceInHSearchIndex() &&
1003                        myDaoConfig.isAdvancedHSearchIndexing() &&
1004                        // we don't support history
1005                        thePids.stream().noneMatch(p -> p.getVersion() != null) &&
1006                        // skip the complexity for metadata in dstu2
1007                        myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3);
1008        }
1009
1010        private List<IBaseResource> loadResourcesFromElasticSearch(Collection<JpaPid> thePids) {
1011                // Do we use the fulltextsvc via hibernate-search to load resources or be backwards compatible with older ES only impl
1012                // to handle lastN?
1013                if (myDaoConfig.isAdvancedHSearchIndexing() && myDaoConfig.isStoreResourceInHSearchIndex()) {
1014                        List<Long> pidList = thePids.stream().map(pid -> (pid).getId()).collect(Collectors.toList());
1015
1016                        List<IBaseResource> resources = myFulltextSearchSvc.getResources(pidList);
1017                        return resources;
1018                } else if (!Objects.isNull(myParams) && myParams.isLastN()) {
1019                        // legacy LastN implementation
1020                        return myIElasticsearchSvc.getObservationResources(thePids);
1021                } else {
1022                        return Collections.emptyList();
1023                }
1024        }
1025
1026        /**
1027         * THIS SHOULD RETURN HASHSET and not just Set because we add to it later
1028         * so it can't be Collections.emptySet() or some such thing.
1029         * The JpaPid returned will have resource type populated.
1030         */
1031        @Override
1032        public Set<JpaPid> loadIncludes(FhirContext theContext, EntityManager theEntityManager, Collection<JpaPid> theMatches, Collection<Include> theIncludes,
1033                                                                                                                                 boolean theReverseMode, DateRangeParam theLastUpdated, String theSearchIdOrDescription, RequestDetails theRequest, Integer theMaxCount) {
1034                if (theMatches.size() == 0) {
1035                        return new HashSet<>();
1036                }
1037                if (theIncludes == null || theIncludes.isEmpty()) {
1038                        return new HashSet<>();
1039                }
1040                String searchPidFieldName = theReverseMode ? MY_TARGET_RESOURCE_PID : MY_SOURCE_RESOURCE_PID;
1041                String findPidFieldName = theReverseMode ? MY_SOURCE_RESOURCE_PID : MY_TARGET_RESOURCE_PID;
1042                String findResourceTypeFieldName = theReverseMode ? MY_SOURCE_RESOURCE_TYPE : MY_TARGET_RESOURCE_TYPE;
1043                String findVersionFieldName = null;
1044                if (!theReverseMode && myModelConfig.isRespectVersionsForSearchIncludes()) {
1045                        findVersionFieldName = MY_TARGET_RESOURCE_VERSION;
1046                }
1047
1048                List<JpaPid> nextRoundMatches = new ArrayList<>(theMatches);
1049                HashSet<JpaPid> allAdded = new HashSet<>();
1050                HashSet<JpaPid> original = new HashSet<>(theMatches);
1051                ArrayList<Include> includes = new ArrayList<>(theIncludes);
1052
1053                int roundCounts = 0;
1054                StopWatch w = new StopWatch();
1055
1056                boolean addedSomeThisRound;
1057                do {
1058                        roundCounts++;
1059
1060                        HashSet<JpaPid> pidsToInclude = new HashSet<>();
1061
1062                        for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) {
1063                                Include nextInclude = iter.next();
1064                                if (nextInclude.isRecurse() == false) {
1065                                        iter.remove();
1066                                }
1067
1068                                // Account for _include=*
1069                                boolean matchAll = "*".equals(nextInclude.getValue());
1070
1071                                // Account for _include=[resourceType]:*
1072                                String wantResourceType = null;
1073                                if (!matchAll) {
1074                                        if ("*".equals(nextInclude.getParamName())) {
1075                                                wantResourceType = nextInclude.getParamType();
1076                                                matchAll = true;
1077                                        }
1078                                }
1079
1080                                if (matchAll) {
1081                                        StringBuilder sqlBuilder = new StringBuilder();
1082                                        sqlBuilder.append("SELECT r.").append(findPidFieldName);
1083                                        sqlBuilder.append(", r.").append(findResourceTypeFieldName);
1084                                        if (findVersionFieldName != null) {
1085                                                sqlBuilder.append(", r." + findVersionFieldName);
1086                                        }
1087                                        sqlBuilder.append(" FROM ResourceLink r WHERE ");
1088
1089                                        sqlBuilder.append("r.");
1090                                        sqlBuilder.append(searchPidFieldName);
1091                                        sqlBuilder.append(" IN (:target_pids)");
1092
1093                                        // Technically if the request is a qualified star (e.g. _include=Observation:*) we
1094                                        // should always be checking the source resource type on the resource link. We don't
1095                                        // actually index that column though by default, so in order to try and be efficient
1096                                        // we don't actually include it for includes (but we do for revincludes). This is
1097                                        // because for an include it doesn't really make sense to include a different
1098                                        // resource type than the one you are searching on.
1099                                        if (wantResourceType != null && theReverseMode) {
1100                                                sqlBuilder.append(" AND r.mySourceResourceType = :want_resource_type");
1101                                        } else {
1102                                                wantResourceType = null;
1103                                        }
1104
1105                                        String sql = sqlBuilder.toString();
1106                                        List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize());
1107                                        for (Collection<JpaPid> nextPartition : partitions) {
1108                                                TypedQuery<?> q = theEntityManager.createQuery(sql, Object[].class);
1109                                                q.setParameter("target_pids", JpaPid.toLongList(nextPartition));
1110                                                if (wantResourceType != null) {
1111                                                        q.setParameter("want_resource_type", wantResourceType);
1112                                                }
1113                                                if (theMaxCount != null) {
1114                                                        q.setMaxResults(theMaxCount);
1115                                                }
1116                                                List<?> results = q.getResultList();
1117                                                for (Object nextRow : results) {
1118                                                        if (nextRow == null) {
1119                                                                // This can happen if there are outgoing references which are canonical or point to
1120                                                                // other servers
1121                                                                continue;
1122                                                        }
1123
1124                                                        Long version = null;
1125                                                        Long resourceLink = (Long) ((Object[]) nextRow)[0];
1126                                                        String resourceType = (String) ((Object[]) nextRow)[1];
1127                                                        if (findVersionFieldName != null) {
1128                                                                version = (Long) ((Object[]) nextRow)[2];
1129                                                        }
1130
1131                                                        if (resourceLink != null) {
1132                                                                JpaPid pid = JpaPid.fromIdAndVersionAndResourceType(resourceLink, version, resourceType);
1133                                                                pidsToInclude.add(pid);
1134                                                        }
1135                                                }
1136                                        }
1137                                } else {
1138
1139                                        List<String> paths;
1140
1141                                        // Start replace
1142                                        RuntimeSearchParam param;
1143                                        String resType = nextInclude.getParamType();
1144                                        if (isBlank(resType)) {
1145                                                continue;
1146                                        }
1147                                        RuntimeResourceDefinition def = theContext.getResourceDefinition(resType);
1148                                        if (def == null) {
1149                                                ourLog.warn("Unknown resource type in include/revinclude=" + nextInclude.getValue());
1150                                                continue;
1151                                        }
1152
1153                                        String paramName = nextInclude.getParamName();
1154                                        if (isNotBlank(paramName)) {
1155                                                param = mySearchParamRegistry.getActiveSearchParam(resType, paramName);
1156                                        } else {
1157                                                param = null;
1158                                        }
1159                                        if (param == null) {
1160                                                ourLog.warn("Unknown param name in include/revinclude=" + nextInclude.getValue());
1161                                                continue;
1162                                        }
1163
1164                                        paths = param.getPathsSplitForResourceType(resType);
1165                                        // end replace
1166
1167                                        String targetResourceType = defaultString(nextInclude.getParamTargetType(), null);
1168                                        for (String nextPath : paths) {
1169                                                boolean haveTargetTypesDefinedByParam = param.hasTargets();
1170                                                String findPidFieldSqlColumn = findPidFieldName.equals(MY_SOURCE_RESOURCE_PID) ? "src_resource_id" : "target_resource_id";
1171                                                String fieldsToLoad = "r." + findPidFieldSqlColumn + " AS " + RESOURCE_ID_ALIAS;
1172                                                if (findVersionFieldName != null) {
1173                                                        fieldsToLoad += ", r.target_resource_version AS " + RESOURCE_VERSION_ALIAS;
1174                                                }
1175
1176                                                // Query for includes lookup has consider 2 cases
1177                                                // Case 1: Where target_resource_id is available in hfj_res_link table for local references
1178                                                // Case 2: Where target_resource_id is null in hfj_res_link table and referred by a canonical url in target_resource_url
1179
1180                                                // Case 1:
1181                                                String searchPidFieldSqlColumn = searchPidFieldName.equals(MY_TARGET_RESOURCE_PID) ? "target_resource_id" : "src_resource_id";
1182                                                StringBuilder resourceIdBasedQuery = new StringBuilder("SELECT " + fieldsToLoad +
1183                                                        " FROM hfj_res_link r " +
1184                                                        " WHERE r.src_path = :src_path AND " +
1185                                                        " r.target_resource_id IS NOT NULL AND " +
1186                                                        " r." + searchPidFieldSqlColumn + " IN (:target_pids) ");
1187                                                if (targetResourceType != null) {
1188                                                        resourceIdBasedQuery.append(" AND r.target_resource_type = :target_resource_type ");
1189                                                } else if (haveTargetTypesDefinedByParam) {
1190                                                        resourceIdBasedQuery.append(" AND r.target_resource_type in (:target_resource_types) ");
1191                                                }
1192
1193                                                // Case 2:
1194                                                String fieldsToLoadFromSpidxUriTable = "rUri.res_id";
1195                                                // to match the fields loaded in union
1196                                                if (fieldsToLoad.split(",").length > 1) {
1197                                                        for (int i = 0; i < fieldsToLoad.split(",").length - 1; i++) {
1198                                                                fieldsToLoadFromSpidxUriTable += ", NULL";
1199                                                        }
1200                                                }
1201                                                //@formatter:off
1202                                                StringBuilder resourceUrlBasedQuery = new StringBuilder("SELECT " + fieldsToLoadFromSpidxUriTable +
1203                                                        " FROM hfj_res_link r " +
1204                                                        " JOIN hfj_spidx_uri rUri ON ( " +
1205                                                        "   r.target_resource_url = rUri.sp_uri AND " +
1206                                                        "   rUri.sp_name = 'url' ");
1207
1208                                                if (targetResourceType != null) {
1209                                                        resourceUrlBasedQuery.append(" AND rUri.res_type = :target_resource_type ");
1210
1211                                                } else if (haveTargetTypesDefinedByParam) {
1212                                                        resourceUrlBasedQuery.append(" AND rUri.res_type IN (:target_resource_types) ");
1213                                                }
1214
1215                                                resourceUrlBasedQuery.append(" ) ");
1216                                                resourceUrlBasedQuery.append(
1217                                                        " WHERE r.src_path = :src_path AND " +
1218                                                                " r.target_resource_id IS NULL AND " +
1219                                                                " r." + searchPidFieldSqlColumn + " IN (:target_pids) ");
1220                                                //@formatter:on
1221
1222                                                String sql = resourceIdBasedQuery + " UNION " + resourceUrlBasedQuery;
1223
1224                                                List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize());
1225                                                for (Collection<JpaPid> nextPartition : partitions) {
1226                                                        Query q = theEntityManager.createNativeQuery(sql, Tuple.class);
1227                                                        q.setParameter("src_path", nextPath);
1228                                                        q.setParameter("target_pids", JpaPid.toLongList(nextPartition));
1229                                                        if (targetResourceType != null) {
1230                                                                q.setParameter("target_resource_type", targetResourceType);
1231                                                        } else if (haveTargetTypesDefinedByParam) {
1232                                                                q.setParameter("target_resource_types", param.getTargets());
1233                                                        }
1234
1235                                                        if (theMaxCount != null) {
1236                                                                q.setMaxResults(theMaxCount);
1237                                                        }
1238                                                        List<Tuple> results = q.getResultList();
1239                                                        for (Tuple result : results) {
1240                                                                if (result != null) {
1241                                                                        Long resourceId = NumberUtils.createLong(String.valueOf(result.get(RESOURCE_ID_ALIAS)));
1242                                                                        Long resourceVersion = null;
1243                                                                        if (findVersionFieldName != null && result.get(RESOURCE_VERSION_ALIAS) != null) {
1244                                                                                resourceVersion = NumberUtils.createLong(String.valueOf(result.get(RESOURCE_VERSION_ALIAS)));
1245                                                                        }
1246                                                                        pidsToInclude.add(JpaPid.fromIdAndVersion(resourceId, resourceVersion));
1247                                                                }
1248                                                        }
1249                                                }
1250                                        }
1251                                }
1252                        }
1253
1254                        if (theReverseMode) {
1255                                if (theLastUpdated != null && (theLastUpdated.getLowerBoundAsInstant() != null || theLastUpdated.getUpperBoundAsInstant() != null)) {
1256                                        pidsToInclude = new HashSet<>(QueryParameterUtils.filterResourceIdsByLastUpdated(theEntityManager, theLastUpdated, pidsToInclude));
1257                                }
1258                        }
1259
1260                        nextRoundMatches.clear();
1261                        for (JpaPid next : pidsToInclude) {
1262                                if (original.contains(next) == false && allAdded.contains(next) == false) {
1263                                        nextRoundMatches.add(next);
1264                                }
1265                        }
1266
1267                        addedSomeThisRound = allAdded.addAll(pidsToInclude);
1268
1269                        if (theMaxCount != null && allAdded.size() >= theMaxCount) {
1270                                break;
1271                        }
1272
1273                } while (includes.size() > 0 && nextRoundMatches.size() > 0 && addedSomeThisRound);
1274
1275                allAdded.removeAll(original);
1276
1277                ourLog.info("Loaded {} {} in {} rounds and {} ms for search {}", allAdded.size(), theReverseMode ? "_revincludes" : "_includes", roundCounts, w.getMillisAndRestart(), theSearchIdOrDescription);
1278
1279                // Interceptor call: STORAGE_PREACCESS_RESOURCES
1280                // This can be used to remove results from the search result details before
1281                // the user has a chance to know that they were in the results
1282                if (allAdded.size() > 0) {
1283
1284                        if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_PREACCESS_RESOURCES, myInterceptorBroadcaster, theRequest)) {
1285                                List<JpaPid> includedPidList = new ArrayList<>(allAdded);
1286                                JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(includedPidList, () -> this);
1287                                HookParams params = new HookParams()
1288                                        .add(IPreResourceAccessDetails.class, accessDetails)
1289                                        .add(RequestDetails.class, theRequest)
1290                                        .addIfMatchesType(ServletRequestDetails.class, theRequest);
1291                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
1292
1293                                for (int i = includedPidList.size() - 1; i >= 0; i--) {
1294                                        if (accessDetails.isDontReturnResourceAtIndex(i)) {
1295                                                JpaPid value = includedPidList.remove(i);
1296                                                if (value != null) {
1297                                                        allAdded.remove(value);
1298                                                }
1299                                        }
1300                                }
1301                        }
1302                }
1303
1304                return allAdded;
1305        }
1306
1307        private List<Collection<JpaPid>> partition(Collection<JpaPid> theNextRoundMatches, int theMaxLoad) {
1308                if (theNextRoundMatches.size() <= theMaxLoad) {
1309                        return Collections.singletonList(theNextRoundMatches);
1310                } else {
1311
1312                        List<Collection<JpaPid>> retVal = new ArrayList<>();
1313                        Collection<JpaPid> current = null;
1314                        for (JpaPid next : theNextRoundMatches) {
1315                                if (current == null) {
1316                                        current = new ArrayList<>(theMaxLoad);
1317                                        retVal.add(current);
1318                                }
1319
1320                                current.add(next);
1321
1322                                if (current.size() >= theMaxLoad) {
1323                                        current = null;
1324                                }
1325                        }
1326
1327                        return retVal;
1328                }
1329        }
1330
1331        private void attemptComboUniqueSpProcessing(QueryStack theQueryStack3, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
1332                RuntimeSearchParam comboParam = null;
1333                List<String> comboParamNames = null;
1334                List<RuntimeSearchParam> exactMatchParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
1335                if (exactMatchParams.size() > 0) {
1336                        comboParam = exactMatchParams.get(0);
1337                        comboParamNames = new ArrayList<>(theParams.keySet());
1338                }
1339
1340                if (comboParam == null) {
1341                        List<RuntimeSearchParam> candidateComboParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName);
1342                        for (RuntimeSearchParam nextCandidate : candidateComboParams) {
1343                                List<String> nextCandidateParamNames = JpaParamUtil
1344                                        .resolveComponentParameters(mySearchParamRegistry, nextCandidate)
1345                                        .stream()
1346                                        .map(t -> t.getName())
1347                                        .collect(Collectors.toList());
1348                                if (theParams.keySet().containsAll(nextCandidateParamNames)) {
1349                                        comboParam = nextCandidate;
1350                                        comboParamNames = nextCandidateParamNames;
1351                                        break;
1352                                }
1353                        }
1354                }
1355
1356                if (comboParam != null) {
1357                        // Since we're going to remove elements below
1358                        theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList));
1359
1360                        StringBuilder sb = new StringBuilder();
1361                        sb.append(myResourceName);
1362                        sb.append("?");
1363
1364                        boolean first = true;
1365
1366                        Collections.sort(comboParamNames);
1367                        for (String nextParamName : comboParamNames) {
1368                                List<List<IQueryParameterType>> nextValues = theParams.get(nextParamName);
1369
1370                                // TODO Hack to fix weird IOOB on the next stanza until James comes back and makes sense of this.
1371                                if (nextValues.isEmpty()) {
1372                                        ourLog.error("query parameter {} is unexpectedly empty. Encountered while considering {} index for {}", nextParamName, comboParam.getName(), theRequest.getCompleteUrl());
1373                                        sb = null;
1374                                        break;
1375                                }
1376
1377                                if (nextValues.get(0).size() != 1) {
1378                                        sb = null;
1379                                        break;
1380                                }
1381
1382                                // Reference params are only eligible for using a composite index if they
1383                                // are qualified
1384                                RuntimeSearchParam nextParamDef = mySearchParamRegistry.getActiveSearchParam(myResourceName, nextParamName);
1385                                if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
1386                                        ReferenceParam param = (ReferenceParam) nextValues.get(0).get(0);
1387                                        if (isBlank(param.getResourceType())) {
1388                                                sb = null;
1389                                                break;
1390                                        }
1391                                }
1392
1393                                List<? extends IQueryParameterType> nextAnd = nextValues.remove(0);
1394                                IQueryParameterType nextOr = nextAnd.remove(0);
1395                                String nextOrValue = nextOr.getValueAsQueryToken(myContext);
1396
1397                                if (comboParam.getComboSearchParamType() == ComboSearchParamType.NON_UNIQUE) {
1398                                        if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.STRING) {
1399                                                nextOrValue = StringUtil.normalizeStringForSearchIndexing(nextOrValue);
1400                                        }
1401                                }
1402
1403                                if (first) {
1404                                        first = false;
1405                                } else {
1406                                        sb.append('&');
1407                                }
1408
1409                                nextParamName = UrlUtil.escapeUrlParam(nextParamName);
1410                                nextOrValue = UrlUtil.escapeUrlParam(nextOrValue);
1411
1412                                sb.append(nextParamName).append('=').append(nextOrValue);
1413
1414                        }
1415
1416                        if (sb != null) {
1417                                String indexString = sb.toString();
1418                                ourLog.debug("Checking for {} combo index for query: {}", comboParam.getComboSearchParamType(), indexString);
1419
1420                                // Interceptor broadcast: JPA_PERFTRACE_INFO
1421                                StorageProcessingMessage msg = new StorageProcessingMessage()
1422                                        .setMessage("Using " + comboParam.getComboSearchParamType() + " index for query for search: " + indexString);
1423                                HookParams params = new HookParams()
1424                                        .add(RequestDetails.class, theRequest)
1425                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
1426                                        .add(StorageProcessingMessage.class, msg);
1427                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INFO, params);
1428
1429                                switch (comboParam.getComboSearchParamType()) {
1430                                        case UNIQUE:
1431                                                theQueryStack3.addPredicateCompositeUnique(indexString, myRequestPartitionId);
1432                                                break;
1433                                        case NON_UNIQUE:
1434                                                theQueryStack3.addPredicateCompositeNonUnique(indexString, myRequestPartitionId);
1435                                                break;
1436                                }
1437
1438                                // Remove any empty parameters remaining after this
1439                                theParams.clean();
1440                        }
1441                }
1442        }
1443
1444        private <T> void ensureSubListsAreWritable(List<List<T>> theListOfLists) {
1445                for (int i = 0; i < theListOfLists.size(); i++) {
1446                        List<T> oldSubList = theListOfLists.get(i);
1447                        if (!(oldSubList instanceof ArrayList)) {
1448                                List<T> newSubList = new ArrayList<>(oldSubList);
1449                                theListOfLists.set(i, newSubList);
1450                        }
1451                }
1452        }
1453
1454        @Override
1455        public void setFetchSize(int theFetchSize) {
1456                myFetchSize = theFetchSize;
1457        }
1458
1459        public SearchParameterMap getParams() {
1460                return myParams;
1461        }
1462
1463        public CriteriaBuilder getBuilder() {
1464                return myCriteriaBuilder;
1465        }
1466
1467        public Class<? extends IBaseResource> getResourceType() {
1468                return myResourceType;
1469        }
1470
1471        public String getResourceName() {
1472                return myResourceName;
1473        }
1474
1475        public class IncludesIterator extends BaseIterator<JpaPid> implements Iterator<JpaPid> {
1476
1477                private final RequestDetails myRequest;
1478                private final Set<JpaPid> myCurrentPids;
1479                private Iterator<JpaPid> myCurrentIterator;
1480                private JpaPid myNext;
1481
1482                IncludesIterator(Set<JpaPid> thePidSet, RequestDetails theRequest) {
1483                        myCurrentPids = new HashSet<>(thePidSet);
1484                        myCurrentIterator = null;
1485                        myRequest = theRequest;
1486                }
1487
1488                private void fetchNext() {
1489                        while (myNext == null) {
1490
1491                                if (myCurrentIterator == null) {
1492                                        Set<Include> includes = Collections.singleton(new Include("*", true));
1493                                        Set<JpaPid> newPids = loadIncludes(myContext, myEntityManager, myCurrentPids, includes, false, getParams().getLastUpdated(), mySearchUuid, myRequest, null);
1494                                        myCurrentIterator = newPids.iterator();
1495                                }
1496
1497                                if (myCurrentIterator.hasNext()) {
1498                                        myNext = myCurrentIterator.next();
1499                                } else {
1500                                        myNext = NO_MORE;
1501                                }
1502
1503                        }
1504                }
1505
1506                @Override
1507                public boolean hasNext() {
1508                        fetchNext();
1509                        return !NO_MORE.equals(myNext);
1510                }
1511
1512                @Override
1513                public JpaPid next() {
1514                        fetchNext();
1515                        JpaPid retVal = myNext;
1516                        myNext = null;
1517                        return retVal;
1518                }
1519
1520        }
1521
1522        private final class QueryIterator extends BaseIterator<JpaPid> implements IResultIterator<JpaPid> {
1523
1524                private final SearchRuntimeDetails mySearchRuntimeDetails;
1525                private final RequestDetails myRequest;
1526                private final boolean myHaveRawSqlHooks;
1527                private final boolean myHavePerfTraceFoundIdHook;
1528                private final SortSpec mySort;
1529                private final Integer myOffset;
1530                private boolean myFirst = true;
1531                private IncludesIterator myIncludesIterator;
1532                private JpaPid myNext;
1533                private ISearchQueryExecutor myResultsIterator;
1534                private boolean myFetchIncludesForEverythingOperation;
1535                private int mySkipCount = 0;
1536                private int myNonSkipCount = 0;
1537                private List<ISearchQueryExecutor> myQueryList = new ArrayList<>();
1538
1539                private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
1540                        mySearchRuntimeDetails = theSearchRuntimeDetails;
1541                        mySort = myParams.getSort();
1542                        myOffset = myParams.getOffset();
1543                        myRequest = theRequest;
1544
1545                        // Includes are processed inline for $everything query when we don't have a '_type' specified
1546                        if (myParams.getEverythingMode() != null && !myParams.containsKey(Constants.PARAM_TYPE)) {
1547                                myFetchIncludesForEverythingOperation = true;
1548                        }
1549
1550                        myHavePerfTraceFoundIdHook = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, myInterceptorBroadcaster, myRequest);
1551                        myHaveRawSqlHooks = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, myRequest);
1552
1553                }
1554
1555                private void fetchNext() {
1556
1557                        try {
1558                                if (myHaveRawSqlHooks) {
1559                                        CurrentThreadCaptureQueriesListener.startCapturing();
1560                                }
1561
1562                                // If we don't have a query yet, create one
1563                                if (myResultsIterator == null) {
1564                                        if (myMaxResultsToFetch == null) {
1565                                                if (myParams.getLoadSynchronousUpTo() != null) {
1566                                                        myMaxResultsToFetch = myParams.getLoadSynchronousUpTo();
1567                                                } else if (myParams.getOffset() != null && myParams.getCount() != null) {
1568                                                        myMaxResultsToFetch = myParams.getCount();
1569                                                } else {
1570                                                        myMaxResultsToFetch = myDaoConfig.getFetchSizeDefaultMaximum();
1571                                                }
1572                                        }
1573
1574                                        initializeIteratorQuery(myOffset, myMaxResultsToFetch);
1575
1576                                        if (myAlsoIncludePids == null) {
1577                                                myAlsoIncludePids = new ArrayList<>();
1578                                        }
1579                                }
1580
1581                                if (myNext == null) {
1582
1583
1584                                        for (Iterator<JpaPid> myPreResultsIterator = myAlsoIncludePids.iterator(); myPreResultsIterator.hasNext(); ) {
1585                                                JpaPid next = myPreResultsIterator.next();
1586                                                if (next != null)
1587                                                        if (myPidSet.add(next)) {
1588                                                                myNext = next;
1589                                                                break;
1590                                                        }
1591                                        }
1592
1593                                        if (myNext == null) {
1594                                                while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) {
1595                                                        // Update iterator with next chunk if necessary.
1596                                                        if (!myResultsIterator.hasNext()) {
1597                                                                retrieveNextIteratorQuery();
1598                                                        }
1599
1600                                                        Long nextLong = myResultsIterator.next();
1601                                                        if (myHavePerfTraceFoundIdHook) {
1602                                                                HookParams params = new HookParams()
1603                                                                        .add(Integer.class, System.identityHashCode(this))
1604                                                                        .add(Object.class, nextLong);
1605                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, params);
1606                                                        }
1607
1608                                                        if (nextLong != null) {
1609                                                                JpaPid next = JpaPid.fromId(nextLong);
1610                                                                if (myPidSet.add(next)) {
1611                                                                        myNext = next;
1612                                                                        myNonSkipCount++;
1613                                                                        break;
1614                                                                } else {
1615                                                                        mySkipCount++;
1616                                                                }
1617                                                        }
1618
1619                                                        if (!myResultsIterator.hasNext()) {
1620                                                                if (myMaxResultsToFetch != null && (mySkipCount + myNonSkipCount == myMaxResultsToFetch)) {
1621                                                                        if (mySkipCount > 0 && myNonSkipCount == 0) {
1622
1623                                                                                StorageProcessingMessage message = new StorageProcessingMessage();
1624                                                                                String msg = "Pass completed with no matching results seeking rows " + myPidSet.size() + "-" + mySkipCount + ". This indicates an inefficient query! Retrying with new max count of " + myMaxResultsToFetch;
1625                                                                                ourLog.warn(msg);
1626                                                                                message.setMessage(msg);
1627                                                                                HookParams params = new HookParams()
1628                                                                                        .add(RequestDetails.class, myRequest)
1629                                                                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1630                                                                                        .add(StorageProcessingMessage.class, message);
1631                                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_WARNING, params);
1632
1633                                                                                myMaxResultsToFetch += 1000;
1634                                                                                initializeIteratorQuery(myOffset, myMaxResultsToFetch);
1635                                                                        }
1636                                                                }
1637                                                        }
1638                                                }
1639                                        }
1640
1641                                        if (myNext == null) {
1642                                                if (myFetchIncludesForEverythingOperation) {
1643                                                        myIncludesIterator = new IncludesIterator(myPidSet, myRequest);
1644                                                        myFetchIncludesForEverythingOperation = false;
1645                                                }
1646                                                if (myIncludesIterator != null) {
1647                                                        while (myIncludesIterator.hasNext()) {
1648                                                                JpaPid next = myIncludesIterator.next();
1649                                                                if (next != null)
1650                                                                        if (myPidSet.add(next)) {
1651                                                                                myNext = next;
1652                                                                                break;
1653                                                                        }
1654                                                        }
1655                                                        if (myNext == null) {
1656                                                                myNext = NO_MORE;
1657                                                        }
1658                                                } else {
1659                                                        myNext = NO_MORE;
1660                                                }
1661                                        }
1662
1663                                } // if we need to fetch the next result
1664
1665                                mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size());
1666
1667                        } finally {
1668                                if (myHaveRawSqlHooks) {
1669                                        SqlQueryList capturedQueries = CurrentThreadCaptureQueriesListener.getCurrentQueueAndStopCapturing();
1670                                        HookParams params = new HookParams()
1671                                                .add(RequestDetails.class, myRequest)
1672                                                .addIfMatchesType(ServletRequestDetails.class, myRequest)
1673                                                .add(SqlQueryList.class, capturedQueries);
1674                                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_RAW_SQL, params);
1675                                }
1676                        }
1677
1678                        if (myFirst) {
1679                                HookParams params = new HookParams()
1680                                        .add(RequestDetails.class, myRequest)
1681                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1682                                        .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
1683                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FIRST_RESULT_LOADED, params);
1684                                myFirst = false;
1685                        }
1686
1687                        if (NO_MORE.equals(myNext)) {
1688                                HookParams params = new HookParams()
1689                                        .add(RequestDetails.class, myRequest)
1690                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1691                                        .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
1692                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_SELECT_COMPLETE, params);
1693                        }
1694
1695                }
1696
1697                private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToFetch) {
1698                        if (myQueryList.isEmpty()) {
1699                                // Capture times for Lucene/Elasticsearch queries as well
1700                                mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
1701                                myQueryList = createQuery(myParams, mySort, theOffset, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails);
1702                        }
1703
1704                        mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
1705
1706                        retrieveNextIteratorQuery();
1707
1708                        mySkipCount = 0;
1709                        myNonSkipCount = 0;
1710                }
1711
1712                private void retrieveNextIteratorQuery() {
1713                        close();
1714                        if (myQueryList != null && myQueryList.size() > 0) {
1715                                myResultsIterator = myQueryList.remove(0);
1716                                myHasNextIteratorQuery = true;
1717                        } else {
1718                                myResultsIterator = SearchQueryExecutor.emptyExecutor();
1719                                myHasNextIteratorQuery = false;
1720                        }
1721
1722                }
1723
1724                @Override
1725                public boolean hasNext() {
1726                        if (myNext == null) {
1727                                fetchNext();
1728                        }
1729                        return !NO_MORE.equals(myNext);
1730                }
1731
1732                @Override
1733                public JpaPid next() {
1734                        fetchNext();
1735                        JpaPid retVal = myNext;
1736                        myNext = null;
1737                        Validate.isTrue(!NO_MORE.equals(retVal), "No more elements");
1738                        return retVal;
1739                }
1740
1741                @Override
1742                public int getSkippedCount() {
1743                        return mySkipCount;
1744                }
1745
1746                @Override
1747                public int getNonSkippedCount() {
1748                        return myNonSkipCount;
1749                }
1750
1751                @Override
1752                public Collection<JpaPid> getNextResultBatch(long theBatchSize) {
1753                        Collection<JpaPid> batch = new ArrayList<>();
1754                        while (this.hasNext() && batch.size() < theBatchSize) {
1755                                batch.add(this.next());
1756                        }
1757                        return batch;
1758                }
1759
1760                @Override
1761                public void close() {
1762                        if (myResultsIterator != null) {
1763                                myResultsIterator.close();
1764                        }
1765                        myResultsIterator = null;
1766                }
1767
1768        }
1769
1770        public static int getMaximumPageSize() {
1771                if (myUseMaxPageSize50ForTest) {
1772                        return MAXIMUM_PAGE_SIZE_FOR_TESTING;
1773                } else {
1774                        return MAXIMUM_PAGE_SIZE;
1775                }
1776        }
1777
1778        public static void setMaxPageSize50ForTest(boolean theIsTest) {
1779                myUseMaxPageSize50ForTest = theIsTest;
1780        }
1781
1782}