001package ca.uhn.fhir.jpa.search.builder;
002
003/*
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2022 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 * http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.context.ComboSearchParamType;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.context.FhirVersionEnum;
026import ca.uhn.fhir.context.RuntimeResourceDefinition;
027import ca.uhn.fhir.context.RuntimeSearchParam;
028import ca.uhn.fhir.i18n.Msg;
029import ca.uhn.fhir.interceptor.api.HookParams;
030import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
031import ca.uhn.fhir.interceptor.api.Pointcut;
032import ca.uhn.fhir.interceptor.model.RequestPartitionId;
033import ca.uhn.fhir.jpa.api.config.DaoConfig;
034import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
035import ca.uhn.fhir.jpa.api.dao.IDao;
036import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
037import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
038import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean;
039import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider;
040import ca.uhn.fhir.jpa.dao.BaseStorageDao;
041import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
042import ca.uhn.fhir.jpa.dao.IResultIterator;
043import ca.uhn.fhir.jpa.dao.ISearchBuilder;
044import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
045import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
046import ca.uhn.fhir.jpa.entity.ResourceSearchView;
047import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
048import ca.uhn.fhir.jpa.model.config.PartitionSettings;
049import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity;
050import ca.uhn.fhir.jpa.model.entity.ModelConfig;
051import ca.uhn.fhir.jpa.model.entity.ResourceTable;
052import ca.uhn.fhir.jpa.model.entity.ResourceTag;
053import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
054import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
055import ca.uhn.fhir.jpa.search.SearchConstants;
056import ca.uhn.fhir.jpa.search.builder.sql.GeneratedSql;
057import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder;
058import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryExecutor;
059import ca.uhn.fhir.jpa.search.builder.sql.SqlObjectFactory;
060import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc;
061import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
062import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
063import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil;
064import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
065import ca.uhn.fhir.jpa.util.BaseIterator;
066import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
067import ca.uhn.fhir.jpa.util.QueryChunker;
068import ca.uhn.fhir.jpa.util.SqlQueryList;
069import ca.uhn.fhir.model.api.IQueryParameterType;
070import ca.uhn.fhir.model.api.IResource;
071import ca.uhn.fhir.model.api.Include;
072import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
073import ca.uhn.fhir.model.primitive.InstantDt;
074import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum;
075import ca.uhn.fhir.rest.api.Constants;
076import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
077import ca.uhn.fhir.rest.api.SearchContainedModeEnum;
078import ca.uhn.fhir.rest.api.SortOrderEnum;
079import ca.uhn.fhir.rest.api.SortSpec;
080import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
081import ca.uhn.fhir.rest.api.server.RequestDetails;
082import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
083import ca.uhn.fhir.rest.param.DateRangeParam;
084import ca.uhn.fhir.rest.param.ReferenceParam;
085import ca.uhn.fhir.rest.param.StringParam;
086import ca.uhn.fhir.rest.param.TokenParam;
087import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
088import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
089import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
090import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
091import ca.uhn.fhir.util.StopWatch;
092import ca.uhn.fhir.util.StringUtil;
093import ca.uhn.fhir.util.UrlUtil;
094import com.google.common.annotations.VisibleForTesting;
095import com.google.common.collect.Streams;
096import com.healthmarketscience.sqlbuilder.Condition;
097import org.apache.commons.lang3.Validate;
098import org.apache.commons.lang3.math.NumberUtils;
099import org.hl7.fhir.instance.model.api.IAnyResource;
100import org.hl7.fhir.instance.model.api.IBaseResource;
101import org.slf4j.Logger;
102import org.slf4j.LoggerFactory;
103import org.springframework.beans.factory.annotation.Autowired;
104import org.springframework.jdbc.core.JdbcTemplate;
105import org.springframework.jdbc.core.SingleColumnRowMapper;
106import org.springframework.transaction.support.TransactionSynchronizationManager;
107
108import javax.annotation.Nonnull;
109import javax.persistence.EntityManager;
110import javax.persistence.PersistenceContext;
111import javax.persistence.PersistenceContextType;
112import javax.persistence.Query;
113import javax.persistence.Tuple;
114import javax.persistence.TypedQuery;
115import javax.persistence.criteria.CriteriaBuilder;
116import javax.persistence.criteria.CriteriaQuery;
117import javax.persistence.criteria.From;
118import javax.persistence.criteria.Predicate;
119import javax.persistence.criteria.Root;
120import java.util.ArrayList;
121import java.util.Collection;
122import java.util.Collections;
123import java.util.HashMap;
124import java.util.HashSet;
125import java.util.Iterator;
126import java.util.List;
127import java.util.Map;
128import java.util.Objects;
129import java.util.Optional;
130import java.util.Set;
131import java.util.stream.Collectors;
132
133import static org.apache.commons.lang3.StringUtils.defaultString;
134import static org.apache.commons.lang3.StringUtils.isBlank;
135import static org.apache.commons.lang3.StringUtils.isNotBlank;
136
137/**
138 * The SearchBuilder is responsible for actually forming the SQL query that handles
139 * searches for resources
140 */
141public class SearchBuilder implements ISearchBuilder {
142
143        /**
144         * See loadResourcesByPid
145         * for an explanation of why we use the constant 800
146         */
147        // NB: keep public
148        @Deprecated
149        public static final int MAXIMUM_PAGE_SIZE = SearchConstants.MAX_PAGE_SIZE;
150        public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50;
151        private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class);
152        private static final ResourcePersistentId NO_MORE = new ResourcePersistentId(-1L);
153        private static final String MY_TARGET_RESOURCE_PID = "myTargetResourcePid";
154        private static final String MY_SOURCE_RESOURCE_PID = "mySourceResourcePid";
155        private static final String MY_TARGET_RESOURCE_VERSION = "myTargetResourceVersion";
156        public static final String RESOURCE_ID_ALIAS = "resource_id";
157        public static final String RESOURCE_VERSION_ALIAS = "resource_version";
158        public static boolean myUseMaxPageSize50ForTest = false;
159        private final String myResourceName;
160        private final Class<? extends IBaseResource> myResourceType;
161        private final IDao myCallingDao;
162        @Autowired
163        protected IInterceptorBroadcaster myInterceptorBroadcaster;
164        @Autowired
165        protected IResourceTagDao myResourceTagDao;
166        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
167        protected EntityManager myEntityManager;
168        @Autowired
169        private DaoConfig myDaoConfig;
170        @Autowired
171        private DaoRegistry myDaoRegistry;
172        @Autowired
173        private IResourceSearchViewDao myResourceSearchViewDao;
174        @Autowired
175        private FhirContext myContext;
176        @Autowired
177        private IIdHelperService myIdHelperService;
178        @Autowired(required = false)
179        private IFulltextSearchSvc myFulltextSearchSvc;
180        @Autowired(required = false)
181        private IElasticsearchSvc myIElasticsearchSvc;
182        @Autowired
183        private ISearchParamRegistry mySearchParamRegistry;
184        private List<ResourcePersistentId> myAlsoIncludePids;
185        private CriteriaBuilder myCriteriaBuilder;
186        private SearchParameterMap myParams;
187        private String mySearchUuid;
188        private int myFetchSize;
189        private Integer myMaxResultsToFetch;
190        private Set<ResourcePersistentId> myPidSet;
191        private boolean myHasNextIteratorQuery = false;
192        private RequestPartitionId myRequestPartitionId;
193        @Autowired
194        private PartitionSettings myPartitionSettings;
195        @Autowired
196        private HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory;
197        @Autowired
198        private SqlObjectFactory mySqlBuilderFactory;
199        @Autowired
200        private HibernatePropertiesProvider myDialectProvider;
201        @Autowired
202        private ModelConfig myModelConfig;
203
204
205        /**
206         * Constructor
207         */
208        public SearchBuilder(IDao theDao, String theResourceName, Class<? extends IBaseResource> theResourceType) {
209                myCallingDao = theDao;
210                myResourceName = theResourceName;
211                myResourceType = theResourceType;
212        }
213
214        @Override
215        public void setMaxResultsToFetch(Integer theMaxResultsToFetch) {
216                myMaxResultsToFetch = theMaxResultsToFetch;
217        }
218
219        private void searchForIdsWithAndOr(SearchQueryBuilder theSearchSqlBuilder, QueryStack theQueryStack, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
220                myParams = theParams;
221
222                // Remove any empty parameters
223                theParams.clean();
224
225                // For DSTU3, pull out near-distance first so when it comes time to evaluate near, we already know the distance
226                if (myContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) {
227                        Dstu3DistanceHelper.setNearDistance(myResourceType, theParams);
228                }
229
230                // Attempt to lookup via composite unique key.
231                if (isCompositeUniqueSpCandidate()) {
232                        attemptComboUniqueSpProcessing(theQueryStack, theParams, theRequest);
233                }
234
235                SearchContainedModeEnum searchContainedMode = theParams.getSearchContainedMode();
236
237                // Handle _id and _tag last, since they can typically be tacked onto a different parameter
238                List<String> paramNames = myParams.keySet().stream().filter(t -> !t.equals(IAnyResource.SP_RES_ID))
239                        .filter(t -> !t.equals(Constants.PARAM_TAG)).collect(Collectors.toList());
240                if (myParams.containsKey(IAnyResource.SP_RES_ID)) {
241                        paramNames.add(IAnyResource.SP_RES_ID);
242                }
243                if (myParams.containsKey(Constants.PARAM_TAG)) {
244                        paramNames.add(Constants.PARAM_TAG);
245                }
246
247                // Handle each parameter
248                for (String nextParamName : paramNames) {
249                        if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) {
250                                // Skip parameters for Subject, Patient, Code and Category for LastN as these will be filtered by Elasticsearch
251                                continue;
252                        }
253                        List<List<IQueryParameterType>> andOrParams = myParams.get(nextParamName);
254                        Condition predicate = theQueryStack.searchForIdsWithAndOr(null, myResourceName, nextParamName, andOrParams, theRequest, myRequestPartitionId, searchContainedMode);
255                        if (predicate != null) {
256                                theSearchSqlBuilder.addPredicate(predicate);
257                        }
258                }
259        }
260
261        /**
262         * A search is a candidate for Composite Unique SP if unique indexes are enabled, there is no EverythingMode, and the
263         * parameters all have no modifiers.
264         */
265        private boolean isCompositeUniqueSpCandidate() {
266                return myDaoConfig.isUniqueIndexesEnabled() &&
267                        myParams.getEverythingMode() == null &&
268                        myParams.isAllParametersHaveNoModifier();
269        }
270
271        @SuppressWarnings("ConstantConditions")
272        @Override
273        public Long createCountQuery(SearchParameterMap theParams, String theSearchUuid,
274                                RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) {
275
276                assert theRequestPartitionId != null;
277                assert TransactionSynchronizationManager.isActualTransactionActive();
278
279                init(theParams, theSearchUuid, theRequestPartitionId);
280
281                if (checkUseHibernateSearch()) {
282                        long count = myFulltextSearchSvc.count(myResourceName, theParams.clone());
283                        return count;
284                }
285
286                List<ISearchQueryExecutor> queries = createQuery(theParams.clone(), null, null, null, true, theRequest, null);
287                if (queries.isEmpty()) {
288                        return 0L;
289                } else {
290                        return queries.get(0).next();
291                }
292        }
293
294        /**
295         * @param thePidSet May be null
296         */
297        @Override
298        public void setPreviouslyAddedResourcePids(@Nonnull List<ResourcePersistentId> thePidSet) {
299                myPidSet = new HashSet<>(thePidSet);
300        }
301
302        @SuppressWarnings("ConstantConditions")
303        @Override
304        public IResultIterator createQuery(SearchParameterMap theParams, SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) {
305                assert theRequestPartitionId != null;
306                assert TransactionSynchronizationManager.isActualTransactionActive();
307
308                init(theParams, theSearchRuntimeDetails.getSearchUuid(), theRequestPartitionId);
309
310                if (myPidSet == null) {
311                        myPidSet = new HashSet<>();
312                }
313
314                return new QueryIterator(theSearchRuntimeDetails, theRequest);
315        }
316
317
318        private void init(SearchParameterMap theParams, String theSearchUuid, RequestPartitionId theRequestPartitionId) {
319                myCriteriaBuilder = myEntityManager.getCriteriaBuilder();
320                myParams = theParams;
321                mySearchUuid = theSearchUuid;
322                myRequestPartitionId = theRequestPartitionId;
323        }
324
325        private List<ISearchQueryExecutor> createQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCountOnlyFlag, RequestDetails theRequest,
326                                                                                                                                                 SearchRuntimeDetails theSearchRuntimeDetails) {
327
328                ArrayList<ISearchQueryExecutor> queries = new ArrayList<>();
329
330                if (checkUseHibernateSearch()) {
331                        // we're going to run at least part of the search against the Fulltext service.
332
333                        // Ugh - we have two different return types for now
334                        ISearchQueryExecutor fulltextExecutor = null;
335                        List<ResourcePersistentId> fulltextMatchIds = null;
336                        int resultCount = 0;
337                        if (myParams.isLastN()) {
338                                fulltextMatchIds = executeLastNAgainstIndex(theMaximumResults);
339                                resultCount = fulltextMatchIds.size();
340                        } else if (myParams.getEverythingMode() != null) {
341                                fulltextMatchIds = queryHibernateSearchForEverythingPids();
342                                resultCount = fulltextMatchIds.size();
343                        } else {
344                                fulltextExecutor = myFulltextSearchSvc.searchAsync(myResourceName, myParams);
345                        }
346
347                        if (fulltextExecutor == null) {
348                                fulltextExecutor = SearchQueryExecutors.from(fulltextMatchIds);
349                        }
350
351                        if (theSearchRuntimeDetails != null) {
352                                // wipmb we no longer have full size.
353                                theSearchRuntimeDetails.setFoundIndexMatchesCount(resultCount);
354                                HookParams params = new HookParams()
355                                        .add(RequestDetails.class, theRequest)
356                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
357                                        .add(SearchRuntimeDetails.class, theSearchRuntimeDetails);
358                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params);
359                        }
360
361                        // wipmb extract
362                        // can we skip the database entirely and return the pid list from here?
363                        boolean canSkipDatabase =
364                                // if we processed an AND clause, and it returned nothing, then nothing can match.
365                                !fulltextExecutor.hasNext() ||
366                                        // Our hibernate search query doesn't respect partitions yet
367                                        (!myPartitionSettings.isPartitioningEnabled() &&
368                                        // were there AND terms left?  Then we still need the db.
369                                                theParams.isEmpty() &&
370                                                // not every param is a param. :-(
371                                                theParams.getNearDistanceParam() == null &&
372                                                theParams.getLastUpdated() == null &&
373                                                theParams.getEverythingMode() == null &&
374                                                theParams.getOffset() == null
375//                                              &&
376//                                              // or sorting?
377//                                              theParams.getSort() == null
378                                        );
379
380                        if (canSkipDatabase) {
381                                if (theMaximumResults != null) {
382                                        fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theMaximumResults);
383                                }
384                                queries.add(fulltextExecutor);
385                        } else {
386                                // Finish the query in the database for the rest of the search parameters, sorting, partitioning, etc.
387                                // We break the pids into chunks that fit in the 1k limit for jdbc bind params.
388                                // wipmb change chunk to take iterator
389                                new QueryChunker<Long>()
390                                        .chunk(Streams.stream(fulltextExecutor).collect(Collectors.toList()), t -> doCreateChunkedQueries(theParams, t, theOffset, sort, theCountOnlyFlag, theRequest, queries));
391                        }
392                } else {
393                        // do everything in the database.
394                        Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, theMaximumResults, theCountOnlyFlag, theRequest, null);
395                        query.ifPresent(queries::add);
396                }
397
398                return queries;
399        }
400
401        /**
402         * Check to see if query should use Hibernate Search, and error if the query can't continue.
403         *
404         * @return true if the query should first be processed by Hibernate Search
405         * @throws InvalidRequestException if fulltext search is not enabled but the query requires it - _content or _text
406         */
407        private boolean checkUseHibernateSearch() {
408                boolean fulltextEnabled = (myFulltextSearchSvc != null) && !myFulltextSearchSvc.isDisabled();
409
410                if (!fulltextEnabled) {
411                        failIfUsed(Constants.PARAM_TEXT);
412                        failIfUsed(Constants.PARAM_CONTENT);
413                }
414
415                // TODO MB someday we'll want a query planner to figure out if we _should_ or _must_ use the ft index, not just if we can.
416                return fulltextEnabled && myParams != null &&
417                        myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE &&
418                        myFulltextSearchSvc.supportsSomeOf(myParams);
419        }
420
421        private void failIfUsed(String theParamName) {
422                if (myParams.containsKey(theParamName)) {
423                        throw new InvalidRequestException(Msg.code(1192) + "Fulltext search is not enabled on this service, can not process parameter: " + theParamName);
424                }
425        }
426
427        private List<ResourcePersistentId> executeLastNAgainstIndex(Integer theMaximumResults) {
428                // Can we use our hibernate search generated index on resource to support lastN?:
429                if (myDaoConfig.isAdvancedHSearchIndexing()) {
430                        if (myFulltextSearchSvc == null) {
431                                throw new InvalidRequestException(Msg.code(2027) + "LastN operation is not enabled on this service, can not process this request");
432                        }
433                        return myFulltextSearchSvc.lastN(myParams, theMaximumResults)
434                                .stream().map(lastNResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, String.valueOf(lastNResourceId)))
435                                .collect(Collectors.toList());
436                } else {
437                        if (myIElasticsearchSvc == null) {
438                                throw new InvalidRequestException(Msg.code(2033) + "LastN operation is not enabled on this service, can not process this request");
439                        }
440                        // use the dedicated observation ES/Lucene index to support lastN query
441                        return myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults).stream()
442                                .map(lastnResourceId -> myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId))
443                                .collect(Collectors.toList());
444                }
445        }
446
447        private List<ResourcePersistentId> queryHibernateSearchForEverythingPids() {
448                ResourcePersistentId pid = null;
449                if (myParams.get(IAnyResource.SP_RES_ID) != null) {
450                        String idParamValue;
451                        IQueryParameterType idParam = myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
452                        if (idParam instanceof TokenParam) {
453                                TokenParam idParm = (TokenParam) idParam;
454                                idParamValue = idParm.getValue();
455                        } else {
456                                StringParam idParm = (StringParam) idParam;
457                                idParamValue = idParm.getValue();
458                        }
459
460                        pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
461                }
462                List<ResourcePersistentId> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid);
463                return pids;
464        }
465
466        private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<ISearchQueryExecutor> theQueries) {
467                if (thePids.size() < getMaximumPageSize()) {
468                        normalizeIdListForLastNInClause(thePids);
469                }
470                Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, thePids.size(), theCount, theRequest, thePids);
471                query.ifPresent(t -> theQueries.add(t));
472        }
473
474        /**
475         * Combs through the params for any _id parameters and extracts the PIDs for them
476         *
477         * @param theTargetPids
478         */
479        private void extractTargetPidsFromIdParams(HashSet<Long> theTargetPids) {
480                // get all the IQueryParameterType objects
481                // for _id -> these should all be StringParam values
482                HashSet<String> ids = new HashSet<>();
483                List<List<IQueryParameterType>> params = myParams.get(IAnyResource.SP_RES_ID);
484                for (List<IQueryParameterType> paramList : params) {
485                        for (IQueryParameterType param : paramList) {
486                                if (param instanceof StringParam) {
487                                        // we expect all _id values to be StringParams
488                                        ids.add(((StringParam) param).getValue());
489                                } else if (param instanceof TokenParam) {
490                                        ids.add(((TokenParam) param).getValue());
491                                } else {
492                                        // we do not expect the _id parameter to be a non-string value
493                                        throw new IllegalArgumentException(Msg.code(1193) + "_id parameter must be a StringParam or TokenParam");
494                                }
495                        }
496                }
497
498                // fetch our target Pids
499                // this will throw if an id is not found
500                Map<String, ResourcePersistentId> idToPid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId,
501                        myResourceName,
502                        new ArrayList<>(ids));
503                if (myAlsoIncludePids == null) {
504                        myAlsoIncludePids = new ArrayList<>();
505                }
506
507                // add the pids to targetPids
508                for (ResourcePersistentId pid : idToPid.values()) {
509                        myAlsoIncludePids.add(pid);
510                        theTargetPids.add(pid.getIdAsLong());
511                }
512        }
513
514        private Optional<SearchQueryExecutor> createChunkedQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCountOnlyFlag, RequestDetails theRequest, List<Long> thePidList) {
515                String sqlBuilderResourceName = myParams.getEverythingMode() == null ? myResourceName : null;
516                SearchQueryBuilder sqlBuilder = new SearchQueryBuilder(myContext, myDaoConfig.getModelConfig(), myPartitionSettings, myRequestPartitionId, sqlBuilderResourceName, mySqlBuilderFactory, myDialectProvider, theCountOnlyFlag);
517                QueryStack queryStack3 = new QueryStack(theParams, myDaoConfig, myDaoConfig.getModelConfig(), myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings);
518
519                if (theParams.keySet().size() > 1 || theParams.getSort() != null || theParams.keySet().contains(Constants.PARAM_HAS) || isPotentiallyContainedReferenceParameterExistsAtRoot(theParams)) {
520                        List<RuntimeSearchParam> activeComboParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
521                        if (activeComboParams.isEmpty()) {
522                                sqlBuilder.setNeedResourceTableRoot(true);
523                        }
524                }
525
526                JdbcTemplate jdbcTemplate = new JdbcTemplate(myEntityManagerFactory.getDataSource());
527                jdbcTemplate.setFetchSize(myFetchSize);
528                if (theMaximumResults != null) {
529                        jdbcTemplate.setMaxRows(theMaximumResults);
530                }
531
532                if (myParams.getEverythingMode() != null) {
533                        HashSet<Long> targetPids = new HashSet<>();
534                        if (myParams.get(IAnyResource.SP_RES_ID) != null) {
535                                extractTargetPidsFromIdParams(targetPids);
536                        } else {
537                                // For Everything queries, we make the query root by the ResourceLink table, since this query
538                                // is basically a reverse-include search. For type/Everything (as opposed to instance/Everything)
539                                // the one problem with this approach is that it doesn't catch Patients that have absolutely
540                                // nothing linked to them. So we do one additional query to make sure we catch those too.
541                                SearchQueryBuilder fetchPidsSqlBuilder = new SearchQueryBuilder(myContext, myDaoConfig.getModelConfig(), myPartitionSettings, myRequestPartitionId, myResourceName, mySqlBuilderFactory, myDialectProvider, theCountOnlyFlag);
542                                GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theOffset, myMaxResultsToFetch);
543                                String sql = allTargetsSql.getSql();
544                                Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]);
545                                List<Long> output = jdbcTemplate.query(sql, args, new SingleColumnRowMapper<>(Long.class));
546                                if (myAlsoIncludePids == null) {
547                                        myAlsoIncludePids = new ArrayList<>(output.size());
548                                }
549                                myAlsoIncludePids.addAll(ResourcePersistentId.fromLongList(output));
550
551                        }
552
553                        queryStack3.addPredicateEverythingOperation(myResourceName, targetPids.toArray(new Long[0]));
554                } else {
555                        /*
556                         * If we're doing a filter, always use the resource table as the root - This avoids the possibility of
557                         * specific filters with ORs as their root from working around the natural resource type / deletion
558                         * status / partition IDs built into queries.
559                         */
560                        if (theParams.containsKey(Constants.PARAM_FILTER)) {
561                                Condition partitionIdPredicate = sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId);
562                                if (partitionIdPredicate != null) {
563                                        sqlBuilder.addPredicate(partitionIdPredicate);
564                                }
565                        }
566
567                        // Normal search
568                        searchForIdsWithAndOr(sqlBuilder, queryStack3, myParams, theRequest);
569                }
570
571                // If we haven't added any predicates yet, we're doing a search for all resources. Make sure we add the
572                // partition ID predicate in that case.
573                if (!sqlBuilder.haveAtLeastOnePredicate()) {
574                        Condition partitionIdPredicate = sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId);
575                        if (partitionIdPredicate != null) {
576                                sqlBuilder.addPredicate(partitionIdPredicate);
577                        }
578                }
579
580                // Add PID list predicate for full text search and/or lastn operation
581                if (thePidList != null && thePidList.size() > 0) {
582                        sqlBuilder.addResourceIdsPredicate(thePidList);
583                }
584
585                // Last updated
586                DateRangeParam lu = myParams.getLastUpdated();
587                if (lu != null && !lu.isEmpty()) {
588                        Condition lastUpdatedPredicates = sqlBuilder.addPredicateLastUpdated(lu);
589                        sqlBuilder.addPredicate(lastUpdatedPredicates);
590                }
591
592                /*
593                 * Exclude the pids already in the previous iterator. This is an optimization, as opposed
594                 * to something needed to guarantee correct results.
595                 *
596                 * Why do we need it? Suppose for example, a query like:
597                 *    Observation?category=foo,bar,baz
598                 * And suppose you have many resources that have all 3 of these category codes. In this case
599                 * the SQL query will probably return the same PIDs multiple times, and if this happens enough
600                 * we may exhaust the query results without getting enough distinct results back. When that
601                 * happens we re-run the query with a larger limit. Excluding results we already know about
602                 * tries to ensure that we get new unique results.
603                 *
604                 * The challenge with that though is that lots of DBs have an issue with too many
605                 * parameters in one query. So we only do this optimization if there aren't too
606                 * many results.
607                 */
608                if (myHasNextIteratorQuery) {
609                        if (myPidSet.size() + sqlBuilder.countBindVariables() < 900) {
610                                sqlBuilder.excludeResourceIdsPredicate(myPidSet);
611                        }
612                }
613
614                /*
615                 * Sort
616                 *
617                 * If we have a sort, we wrap the criteria search (the search that actually
618                 * finds the appropriate resources) in an outer search which is then sorted
619                 */
620                if (sort != null) {
621                        assert !theCountOnlyFlag;
622
623                        createSort(queryStack3, sort);
624                }
625
626                /*
627                 * Now perform the search
628                 */
629                GeneratedSql generatedSql = sqlBuilder.generate(theOffset, myMaxResultsToFetch);
630                if (generatedSql.isMatchNothing()) {
631                        return Optional.empty();
632                }
633
634                SearchQueryExecutor executor = mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, myMaxResultsToFetch);
635                return Optional.of(executor);
636        }
637
638        private boolean isPotentiallyContainedReferenceParameterExistsAtRoot(SearchParameterMap theParams) {
639                return myModelConfig.isIndexOnContainedResources() && theParams.values().stream()
640                        .flatMap(Collection::stream)
641                        .flatMap(Collection::stream)
642                        .anyMatch(t -> t instanceof ReferenceParam);
643        }
644
645        private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
646                /*
647                        The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying
648                        numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info:
649                        https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage.
650
651                        Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of
652                        arguments never exceeds the maximum specified below.
653                 */
654                int listSize = lastnResourceIds.size();
655
656                if (listSize > 1 && listSize < 10) {
657                        padIdListWithPlaceholders(lastnResourceIds, 10);
658                } else if (listSize > 10 && listSize < 50) {
659                        padIdListWithPlaceholders(lastnResourceIds, 50);
660                } else if (listSize > 50 && listSize < 100) {
661                        padIdListWithPlaceholders(lastnResourceIds, 100);
662                } else if (listSize > 100 && listSize < 200) {
663                        padIdListWithPlaceholders(lastnResourceIds, 200);
664                } else if (listSize > 200 && listSize < 500) {
665                        padIdListWithPlaceholders(lastnResourceIds, 500);
666                } else if (listSize > 500 && listSize < 800) {
667                        padIdListWithPlaceholders(lastnResourceIds, 800);
668                }
669
670                return lastnResourceIds;
671        }
672
673        private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
674                while (theIdList.size() < preferredListSize) {
675                        theIdList.add(-1L);
676                }
677        }
678
679        private void createSort(QueryStack theQueryStack, SortSpec theSort) {
680                if (theSort == null || isBlank(theSort.getParamName())) {
681                        return;
682                }
683
684                boolean ascending = (theSort.getOrder() == null) || (theSort.getOrder() == SortOrderEnum.ASC);
685
686                if (IAnyResource.SP_RES_ID.equals(theSort.getParamName())) {
687
688                        theQueryStack.addSortOnResourceId(ascending);
689
690                } else if (Constants.PARAM_LASTUPDATED.equals(theSort.getParamName())) {
691
692                        theQueryStack.addSortOnLastUpdated(ascending);
693
694                } else {
695
696                        RuntimeSearchParam param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName());
697                        if (param == null) {
698                                String msg = myContext.getLocalizer().getMessageSanitized(BaseStorageDao.class, "invalidSortParameter", theSort.getParamName(), getResourceName(), mySearchParamRegistry.getValidSearchParameterNamesIncludingMeta(getResourceName()));
699                                throw new InvalidRequestException(Msg.code(1194) + msg);
700                        }
701
702                        switch (param.getParamType()) {
703                                case STRING:
704                                        theQueryStack.addSortOnString(myResourceName, theSort.getParamName(), ascending);
705                                        break;
706                                case DATE:
707                                        theQueryStack.addSortOnDate(myResourceName, theSort.getParamName(), ascending);
708                                        break;
709                                case REFERENCE:
710                                        theQueryStack.addSortOnResourceLink(myResourceName, theSort.getParamName(), ascending);
711                                        break;
712                                case TOKEN:
713                                        theQueryStack.addSortOnToken(myResourceName, theSort.getParamName(), ascending);
714                                        break;
715                                case NUMBER:
716                                        theQueryStack.addSortOnNumber(myResourceName, theSort.getParamName(), ascending);
717                                        break;
718                                case URI:
719                                        theQueryStack.addSortOnUri(myResourceName, theSort.getParamName(), ascending);
720                                        break;
721                                case QUANTITY:
722                                        theQueryStack.addSortOnQuantity(myResourceName, theSort.getParamName(), ascending);
723                                        break;
724                                case COMPOSITE:
725                                        List<RuntimeSearchParam> compositeList = JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, param);
726                                        if (compositeList == null) {
727                                                throw new InvalidRequestException(Msg.code(1195) + "The composite _sort parameter " + theSort.getParamName() + " is not defined by the resource " + myResourceName);
728                                        }
729                                        if (compositeList.size() != 2) {
730                                                throw new InvalidRequestException(Msg.code(1196) + "The composite _sort parameter " + theSort.getParamName()
731                                                        + " must have 2 composite types declared in parameter annotation, found "
732                                                        + compositeList.size());
733                                        }
734                                        RuntimeSearchParam left = compositeList.get(0);
735                                        RuntimeSearchParam right = compositeList.get(1);
736
737                                        createCompositeSort(theQueryStack, myResourceName, left.getParamType(), left.getName(), ascending);
738                                        createCompositeSort(theQueryStack, myResourceName, right.getParamType(), right.getName(), ascending);
739
740                                        break;
741                                case SPECIAL:
742                                case HAS:
743                                default:
744                                        throw new InvalidRequestException(Msg.code(1197) + "This server does not support _sort specifications of type " + param.getParamType() + " - Can't serve _sort=" + theSort.getParamName());
745                        }
746
747                }
748
749                // Recurse
750                createSort(theQueryStack, theSort.getChain());
751
752        }
753
754        private void createCompositeSort(QueryStack theQueryStack, String theResourceName, RestSearchParameterTypeEnum theParamType, String theParamName, boolean theAscending) {
755
756                switch (theParamType) {
757                        case STRING:
758                                theQueryStack.addSortOnString(myResourceName, theParamName, theAscending);
759                                break;
760                        case DATE:
761                                theQueryStack.addSortOnDate(myResourceName, theParamName, theAscending);
762                                break;
763                        case TOKEN:
764                                theQueryStack.addSortOnToken(myResourceName, theParamName, theAscending);
765                                break;
766                        case QUANTITY:
767                                theQueryStack.addSortOnQuantity(myResourceName, theParamName, theAscending);
768                                break;
769                        case NUMBER:
770                        case REFERENCE:
771                        case COMPOSITE:
772                        case URI:
773                        case HAS:
774                        case SPECIAL:
775                        default:
776                                throw new InvalidRequestException(Msg.code(1198) + "Don't know how to handle composite parameter with type of " + theParamType + " on _sort=" + theParamName);
777                }
778
779        }
780
781        private void doLoadPids(Collection<ResourcePersistentId> thePids, Collection<ResourcePersistentId> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation,
782                                                                        Map<ResourcePersistentId, Integer> thePosition) {
783
784                Map<Long, Long> resourcePidToVersion = null;
785                for (ResourcePersistentId next : thePids) {
786                        if (next.getVersion() != null && myModelConfig.isRespectVersionsForSearchIncludes()) {
787                                if (resourcePidToVersion == null) {
788                                        resourcePidToVersion = new HashMap<>();
789                                }
790                                resourcePidToVersion.put(next.getIdAsLong(), next.getVersion());
791                        }
792                }
793
794                List<Long> versionlessPids = ResourcePersistentId.toLongList(thePids);
795                if (versionlessPids.size() < getMaximumPageSize()) {
796                        versionlessPids = normalizeIdListForLastNInClause(versionlessPids);
797                }
798
799                // -- get the resource from the searchView
800                Collection<ResourceSearchView> resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(versionlessPids);
801
802                //-- preload all tags with tag definition if any
803                Map<Long, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList);
804
805                for (IBaseResourceEntity next : resourceSearchViewList) {
806                        if (next.getDeleted() != null) {
807                                continue;
808                        }
809
810                        Class<? extends IBaseResource> resourceType = myContext.getResourceDefinition(next.getResourceType()).getImplementingClass();
811
812                        ResourcePersistentId resourceId = new ResourcePersistentId(next.getResourceId());
813
814                        /*
815                         * If a specific version is requested via an include, we'll replace the current version
816                         * with the specific desired version. This is not the most efficient thing, given that
817                         * we're loading the current version and then turning around and throwing it away again.
818                         * This could be optimized and probably should be, but it's not critical given that
819                         * this only applies to includes, which don't tend to be massive in numbers.
820                         */
821                        if (resourcePidToVersion != null) {
822                                Long version = resourcePidToVersion.get(next.getResourceId());
823                                resourceId.setVersion(version);
824                                if (version != null && !version.equals(next.getVersion())) {
825                                        IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(resourceType);
826                                        next = dao.readEntity(next.getIdDt().withVersion(Long.toString(version)), null);
827                                }
828                        }
829
830                        IBaseResource resource = null;
831                        if (next != null) {
832                                resource = myCallingDao.toResource(resourceType, next, tagMap.get(next.getId()), theForHistoryOperation);
833                        }
834                        if (resource == null) {
835                                ourLog.warn("Unable to find resource {}/{}/_history/{} in database", next.getResourceType(), next.getIdDt().getIdPart(), next.getVersion());
836                                continue;
837                        }
838
839                        Integer index = thePosition.get(resourceId);
840                        if (index == null) {
841                                ourLog.warn("Got back unexpected resource PID {}", resourceId);
842                                continue;
843                        }
844
845                        if (resource instanceof IResource) {
846                                if (theIncludedPids.contains(resourceId)) {
847                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IResource) resource, BundleEntrySearchModeEnum.INCLUDE);
848                                } else {
849                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IResource) resource, BundleEntrySearchModeEnum.MATCH);
850                                }
851                        } else {
852                                if (theIncludedPids.contains(resourceId)) {
853                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IAnyResource) resource, BundleEntrySearchModeEnum.INCLUDE.getCode());
854                                } else {
855                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IAnyResource) resource, BundleEntrySearchModeEnum.MATCH.getCode());
856                                }
857                        }
858
859                        theResourceListToPopulate.set(index, resource);
860                }
861        }
862
863        private Map<Long, Collection<ResourceTag>> getResourceTagMap(Collection<? extends IBaseResourceEntity> theResourceSearchViewList) {
864
865                List<Long> idList = new ArrayList<>(theResourceSearchViewList.size());
866
867                //-- find all resource has tags
868                for (IBaseResourceEntity resource : theResourceSearchViewList) {
869                        if (resource.isHasTags())
870                                idList.add(resource.getId());
871                }
872
873                return getPidToTagMap(idList);
874        }
875
876        @Nonnull
877        private Map<Long, Collection<ResourceTag>> getPidToTagMap(List<Long> thePidList) {
878                Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>();
879
880                //-- no tags
881                if (thePidList.size() == 0)
882                        return tagMap;
883
884                //-- get all tags for the idList
885                Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(thePidList);
886
887                //-- build the map, key = resourceId, value = list of ResourceTag
888                ResourcePersistentId resourceId;
889                Collection<ResourceTag> tagCol;
890                for (ResourceTag tag : tagList) {
891
892                        resourceId = new ResourcePersistentId(tag.getResourceId());
893                        tagCol = tagMap.get(resourceId.getIdAsLong());
894                        if (tagCol == null) {
895                                tagCol = new ArrayList<>();
896                                tagCol.add(tag);
897                                tagMap.put(resourceId.getIdAsLong(), tagCol);
898                        } else {
899                                tagCol.add(tag);
900                        }
901                }
902
903                return tagMap;
904        }
905
906        @Override
907        public void loadResourcesByPid(Collection<ResourcePersistentId> thePids, Collection<ResourcePersistentId> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation, RequestDetails theDetails) {
908                if (thePids.isEmpty()) {
909                        ourLog.debug("The include pids are empty");
910                        // return;
911                }
912
913                // Dupes will cause a crash later anyhow, but this is expensive so only do it
914                // when running asserts
915                assert new HashSet<>(thePids).size() == thePids.size() : "PID list contains duplicates: " + thePids;
916
917                Map<ResourcePersistentId, Integer> position = new HashMap<>();
918                for (ResourcePersistentId next : thePids) {
919                        position.put(next, theResourceListToPopulate.size());
920                        theResourceListToPopulate.add(null);
921                }
922
923                // Can we fast track this loading by checking elastic search?
924                if (isLoadingFromElasticSearchSupported(thePids)) {
925                        theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids));
926                } else {
927                        // We only chunk because some jdbc drivers can't handle long param lists.
928                        new QueryChunker<ResourcePersistentId>().chunk(thePids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
929                }
930        }
931
932        /**
933         * Check if we can load the resources from Hibernate Search instead of the database.
934         * We assume this is faster.
935         *
936         * Hibernate Search only stores the current version, and only if enabled.
937         * @param thePids the pids to check for versioned references
938         * @return can we fetch from Hibernate Search?
939         */
940        private boolean isLoadingFromElasticSearchSupported(Collection<ResourcePersistentId> thePids) {
941
942                // is storage enabled?
943                return myDaoConfig.isStoreResourceInHSearchIndex() &&
944                        // we don't support history
945                        thePids.stream().noneMatch(p->p.getVersion()!=null) &&
946                        // skip the complexity for metadata in dstu2
947                        myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3);
948        }
949
950        private List<IBaseResource> loadResourcesFromElasticSearch(Collection<ResourcePersistentId> thePids) {
951                // Do we use the fulltextsvc via hibernate-search to load resources or be backwards compatible with older ES only impl
952                // to handle lastN?
953                if (myDaoConfig.isAdvancedHSearchIndexing() && myDaoConfig.isStoreResourceInHSearchIndex()) {
954                        List<Long> pidList = thePids.stream().map(ResourcePersistentId::getIdAsLong).collect(Collectors.toList());
955
956                        // wipmb standardize on ResourcePersistentId
957                        List<IBaseResource> resources = myFulltextSearchSvc.getResources(pidList);
958                        return resources;
959                } else if (!Objects.isNull(myParams) && myParams.isLastN()) {
960                        // legacy LastN implementation
961                        return myIElasticsearchSvc.getObservationResources(thePids);
962                } else {
963                        return Collections.emptyList();
964                }
965        }
966
967        /**
968         * THIS SHOULD RETURN HASHSET and not just Set because we add to it later
969         * so it can't be Collections.emptySet() or some such thing
970         */
971        @Override
972        public Set<ResourcePersistentId> loadIncludes(FhirContext theContext, EntityManager theEntityManager, Collection<ResourcePersistentId> theMatches, Set<Include> theIncludes,
973                                                                                                                                 boolean theReverseMode, DateRangeParam theLastUpdated, String theSearchIdOrDescription, RequestDetails theRequest, Integer theMaxCount) {
974                if (theMatches.size() == 0) {
975                        return new HashSet<>();
976                }
977                if (theIncludes == null || theIncludes.isEmpty()) {
978                        return new HashSet<>();
979                }
980                String searchPidFieldName = theReverseMode ? MY_TARGET_RESOURCE_PID : MY_SOURCE_RESOURCE_PID;
981                String findPidFieldName = theReverseMode ? MY_SOURCE_RESOURCE_PID : MY_TARGET_RESOURCE_PID;
982                String findVersionFieldName = null;
983                if (!theReverseMode && myModelConfig.isRespectVersionsForSearchIncludes()) {
984                        findVersionFieldName = MY_TARGET_RESOURCE_VERSION;
985                }
986
987                List<ResourcePersistentId> nextRoundMatches = new ArrayList<>(theMatches);
988                HashSet<ResourcePersistentId> allAdded = new HashSet<>();
989                HashSet<ResourcePersistentId> original = new HashSet<>(theMatches);
990                ArrayList<Include> includes = new ArrayList<>(theIncludes);
991
992                int roundCounts = 0;
993                StopWatch w = new StopWatch();
994
995                boolean addedSomeThisRound;
996                do {
997                        roundCounts++;
998
999                        HashSet<ResourcePersistentId> pidsToInclude = new HashSet<>();
1000
1001                        for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) {
1002                                Include nextInclude = iter.next();
1003                                if (nextInclude.isRecurse() == false) {
1004                                        iter.remove();
1005                                }
1006
1007                                // Account for _include=*
1008                                boolean matchAll = "*".equals(nextInclude.getValue());
1009
1010                                // Account for _include=[resourceType]:*
1011                                String wantResourceType = null;
1012                                if (!matchAll) {
1013                                        if ("*".equals(nextInclude.getParamName())) {
1014                                                wantResourceType = nextInclude.getParamType();
1015                                                matchAll = true;
1016                                        }
1017                                }
1018
1019                                if (matchAll) {
1020                                        StringBuilder sqlBuilder = new StringBuilder();
1021                                        sqlBuilder.append("SELECT r.").append(findPidFieldName);
1022                                        if (findVersionFieldName != null) {
1023                                                sqlBuilder.append(", r." + findVersionFieldName);
1024                                        }
1025                                        sqlBuilder.append(" FROM ResourceLink r WHERE ");
1026
1027                                        sqlBuilder.append("r.");
1028                                        sqlBuilder.append(searchPidFieldName);
1029                                        sqlBuilder.append(" IN (:target_pids)");
1030
1031                                        // Technically if the request is a qualified star (e.g. _include=Observation:*) we
1032                                        // should always be checking the source resource type on the resource link. We don't
1033                                        // actually index that column though by default, so in order to try and be efficient
1034                                        // we don't actually include it for includes (but we do for revincludes). This is
1035                                        // because for an include it doesn't really make sense to include a different
1036                                        // resource type than the one you are searching on.
1037                                        if (wantResourceType != null && theReverseMode) {
1038                                                sqlBuilder.append(" AND r.mySourceResourceType = :want_resource_type");
1039                                        } else {
1040                                                wantResourceType = null;
1041                                        }
1042
1043                                        String sql = sqlBuilder.toString();
1044                                        List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
1045                                        for (Collection<ResourcePersistentId> nextPartition : partitions) {
1046                                                TypedQuery<?> q = theEntityManager.createQuery(sql, Object[].class);
1047                                                q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition));
1048                                                if (wantResourceType != null) {
1049                                                        q.setParameter("want_resource_type", wantResourceType);
1050                                                }
1051                                                if (theMaxCount != null) {
1052                                                        q.setMaxResults(theMaxCount);
1053                                                }
1054                                                List<?> results = q.getResultList();
1055                                                for (Object nextRow : results) {
1056                                                        if (nextRow == null) {
1057                                                                // This can happen if there are outgoing references which are canonical or point to
1058                                                                // other servers
1059                                                                continue;
1060                                                        }
1061
1062                                                        Long resourceLink;
1063                                                        Long version = null;
1064                                                        if (findVersionFieldName != null) {
1065                                                                resourceLink = (Long) ((Object[]) nextRow)[0];
1066                                                                version = (Long) ((Object[]) nextRow)[1];
1067                                                        } else {
1068                                                                resourceLink = (Long) nextRow;
1069                                                        }
1070
1071                                                        pidsToInclude.add(new ResourcePersistentId(resourceLink, version));
1072                                                }
1073                                        }
1074                                } else {
1075
1076                                        List<String> paths;
1077                                        RuntimeSearchParam param;
1078                                        String resType = nextInclude.getParamType();
1079                                        if (isBlank(resType)) {
1080                                                continue;
1081                                        }
1082                                        RuntimeResourceDefinition def = theContext.getResourceDefinition(resType);
1083                                        if (def == null) {
1084                                                ourLog.warn("Unknown resource type in include/revinclude=" + nextInclude.getValue());
1085                                                continue;
1086                                        }
1087
1088                                        String paramName = nextInclude.getParamName();
1089                                        if (isNotBlank(paramName)) {
1090                                                param = mySearchParamRegistry.getActiveSearchParam(resType, paramName);
1091                                        } else {
1092                                                param = null;
1093                                        }
1094                                        if (param == null) {
1095                                                ourLog.warn("Unknown param name in include/revinclude=" + nextInclude.getValue());
1096                                                continue;
1097                                        }
1098
1099                                        paths = param.getPathsSplitForResourceType(resType);
1100
1101                                        String targetResourceType = defaultString(nextInclude.getParamTargetType(), null);
1102                                        for (String nextPath : paths) {
1103                                                boolean haveTargetTypesDefinedByParam = param.hasTargets();
1104                                                String findPidFieldSqlColumn = findPidFieldName.equals(MY_SOURCE_RESOURCE_PID) ? "src_resource_id" : "target_resource_id";
1105                                                String fieldsToLoad = "r." + findPidFieldSqlColumn + " AS " + RESOURCE_ID_ALIAS;
1106                                                if (findVersionFieldName != null) {
1107                                                        fieldsToLoad += ", r.target_resource_version AS " + RESOURCE_VERSION_ALIAS;
1108                                                }
1109
1110                                                // Query for includes lookup has consider 2 cases
1111                                                // Case 1: Where target_resource_id is available in hfj_res_link table for local references
1112                                                // Case 2: Where target_resource_id is null in hfj_res_link table and referred by a canonical url in target_resource_url
1113
1114                                                // Case 1:
1115                                                String searchPidFieldSqlColumn = searchPidFieldName.equals(MY_TARGET_RESOURCE_PID) ? "target_resource_id" : "src_resource_id";
1116                                                StringBuilder resourceIdBasedQuery = new StringBuilder("SELECT " + fieldsToLoad +
1117                                                        " FROM hfj_res_link r " +
1118                                                        " WHERE r.src_path = :src_path AND " +
1119                                                        " r.target_resource_id IS NOT NULL AND " +
1120                                                        " r." + searchPidFieldSqlColumn + " IN (:target_pids) ");
1121                                                if(targetResourceType != null) {
1122                                                        resourceIdBasedQuery.append(" AND r.target_resource_type = :target_resource_type ");
1123                                                } else if(haveTargetTypesDefinedByParam) {
1124                                                        resourceIdBasedQuery.append(" AND r.target_resource_type in (:target_resource_types) ");
1125                                                }
1126
1127                                                // Case 2:
1128                                                String fieldsToLoadFromSpidxUriTable = "rUri.res_id";
1129                                                // to match the fields loaded in union
1130                                                if(fieldsToLoad.split(",").length > 1) {
1131                                                        for (int i = 0; i < fieldsToLoad.split(",").length - 1; i++) {
1132                                                                fieldsToLoadFromSpidxUriTable += ", NULL";
1133                                                        }
1134                                                }
1135                                                //@formatter:off
1136                                                String resourceUrlBasedQuery = "SELECT " + fieldsToLoadFromSpidxUriTable +
1137                                                        " FROM hfj_res_link r " +
1138                                                        " JOIN hfj_spidx_uri rUri ON ( " +
1139                                                        "   r.target_resource_url = rUri.sp_uri AND " +
1140                                                        "   rUri.sp_name = 'url' " +
1141                                                            (targetResourceType != null ? " AND rUri.res_type = :target_resource_type " : "") +
1142                                                            (haveTargetTypesDefinedByParam ? " AND rUri.res_type IN (:target_resource_types) " : "") +
1143                                                        " ) " +
1144                                                        " WHERE r.src_path = :src_path AND " +
1145                                                        " r.target_resource_id IS NULL AND " +
1146                                                        " r." + searchPidFieldSqlColumn + " IN (:target_pids) ";
1147                                                //@formatter:on
1148
1149                                                String sql = resourceIdBasedQuery + " UNION " + resourceUrlBasedQuery;
1150
1151                                                List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
1152                                                for (Collection<ResourcePersistentId> nextPartition : partitions) {
1153                                                        Query q = theEntityManager.createNativeQuery(sql, Tuple.class);
1154                                                        q.setParameter("src_path", nextPath);
1155                                                        q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition));
1156                                                        if (targetResourceType != null) {
1157                                                                q.setParameter("target_resource_type", targetResourceType);
1158                                                        } else if (haveTargetTypesDefinedByParam) {
1159                                                                q.setParameter("target_resource_types", param.getTargets());
1160                                                        }
1161                                                        List<Tuple> results = q.getResultList();
1162                                                        if (theMaxCount != null) {
1163                                                                q.setMaxResults(theMaxCount);
1164                                                        }
1165                                                        for (Tuple result : results) {
1166                                                                if (result != null) {
1167                                                                        Long resourceId = NumberUtils.createLong(String.valueOf(result.get(RESOURCE_ID_ALIAS)));
1168                                                                        Long resourceVersion = null;
1169                                                                        if (findVersionFieldName != null && result.get(RESOURCE_VERSION_ALIAS) != null) {
1170                                                                                resourceVersion = NumberUtils.createLong(String.valueOf(result.get(RESOURCE_VERSION_ALIAS)));
1171                                                                        }
1172                                                                        pidsToInclude.add(new ResourcePersistentId(resourceId, resourceVersion));
1173                                                                }
1174                                                        }
1175                                                }
1176                                        }
1177                                }
1178                        }
1179
1180                        if (theReverseMode) {
1181                                if (theLastUpdated != null && (theLastUpdated.getLowerBoundAsInstant() != null || theLastUpdated.getUpperBoundAsInstant() != null)) {
1182                                        pidsToInclude = new HashSet<>(filterResourceIdsByLastUpdated(theEntityManager, theLastUpdated, pidsToInclude));
1183                                }
1184                        }
1185
1186                        nextRoundMatches.clear();
1187                        for (ResourcePersistentId next : pidsToInclude) {
1188                                if (original.contains(next) == false && allAdded.contains(next) == false) {
1189                                        nextRoundMatches.add(next);
1190                                }
1191                        }
1192
1193                        addedSomeThisRound = allAdded.addAll(pidsToInclude);
1194
1195                        if (theMaxCount != null && allAdded.size() >= theMaxCount) {
1196                                break;
1197                        }
1198
1199                } while (includes.size() > 0 && nextRoundMatches.size() > 0 && addedSomeThisRound);
1200
1201                allAdded.removeAll(original);
1202
1203                ourLog.info("Loaded {} {} in {} rounds and {} ms for search {}", allAdded.size(), theReverseMode ? "_revincludes" : "_includes", roundCounts, w.getMillisAndRestart(), theSearchIdOrDescription);
1204
1205                // Interceptor call: STORAGE_PREACCESS_RESOURCES
1206                // This can be used to remove results from the search result details before
1207                // the user has a chance to know that they were in the results
1208                if (allAdded.size() > 0) {
1209
1210                        if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_PREACCESS_RESOURCES, myInterceptorBroadcaster, theRequest)) {
1211                                List<ResourcePersistentId> includedPidList = new ArrayList<>(allAdded);
1212                                JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(includedPidList, () -> this);
1213                                HookParams params = new HookParams()
1214                                        .add(IPreResourceAccessDetails.class, accessDetails)
1215                                        .add(RequestDetails.class, theRequest)
1216                                        .addIfMatchesType(ServletRequestDetails.class, theRequest);
1217                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
1218
1219                                for (int i = includedPidList.size() - 1; i >= 0; i--) {
1220                                        if (accessDetails.isDontReturnResourceAtIndex(i)) {
1221                                                ResourcePersistentId value = includedPidList.remove(i);
1222                                                if (value != null) {
1223                                                        allAdded.remove(value);
1224                                                }
1225                                        }
1226                                }
1227                        }
1228                }
1229
1230                return allAdded;
1231        }
1232
1233        private List<Collection<ResourcePersistentId>> partition(Collection<ResourcePersistentId> theNextRoundMatches, int theMaxLoad) {
1234                if (theNextRoundMatches.size() <= theMaxLoad) {
1235                        return Collections.singletonList(theNextRoundMatches);
1236                } else {
1237
1238                        List<Collection<ResourcePersistentId>> retVal = new ArrayList<>();
1239                        Collection<ResourcePersistentId> current = null;
1240                        for (ResourcePersistentId next : theNextRoundMatches) {
1241                                if (current == null) {
1242                                        current = new ArrayList<>(theMaxLoad);
1243                                        retVal.add(current);
1244                                }
1245
1246                                current.add(next);
1247
1248                                if (current.size() >= theMaxLoad) {
1249                                        current = null;
1250                                }
1251                        }
1252
1253                        return retVal;
1254                }
1255        }
1256
1257        private void attemptComboUniqueSpProcessing(QueryStack theQueryStack3, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
1258                RuntimeSearchParam comboParam = null;
1259                List<String> comboParamNames = null;
1260                List<RuntimeSearchParam> exactMatchParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
1261                if (exactMatchParams.size() > 0) {
1262                        comboParam = exactMatchParams.get(0);
1263                        comboParamNames = new ArrayList<>(theParams.keySet());
1264                }
1265
1266                if (comboParam == null) {
1267                        List<RuntimeSearchParam> candidateComboParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName);
1268                        for (RuntimeSearchParam nextCandidate : candidateComboParams) {
1269                                List<String> nextCandidateParamNames = JpaParamUtil
1270                                        .resolveComponentParameters(mySearchParamRegistry, nextCandidate)
1271                                        .stream()
1272                                        .map(t -> t.getName())
1273                                        .collect(Collectors.toList());
1274                                if (theParams.keySet().containsAll(nextCandidateParamNames)) {
1275                                        comboParam = nextCandidate;
1276                                        comboParamNames = nextCandidateParamNames;
1277                                        break;
1278                                }
1279                        }
1280                }
1281
1282                if (comboParam != null) {
1283                        // Since we're going to remove elements below
1284                        theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList));
1285
1286                        StringBuilder sb = new StringBuilder();
1287                        sb.append(myResourceName);
1288                        sb.append("?");
1289
1290                        boolean first = true;
1291
1292                        Collections.sort(comboParamNames);
1293                        for (String nextParamName : comboParamNames) {
1294                                List<List<IQueryParameterType>> nextValues = theParams.get(nextParamName);
1295
1296                                // TODO Hack to fix weird IOOB on the next stanza until James comes back and makes sense of this.
1297                                if (nextValues.isEmpty()) {
1298                                        ourLog.error("query parameter {} is unexpectedly empty. Encountered while considering {} index for {}", nextParamName, comboParam.getName(), theRequest.getCompleteUrl());
1299                                        sb = null;
1300                                        break;
1301                                }
1302
1303                                if (nextValues.get(0).size() != 1) {
1304                                        sb = null;
1305                                        break;
1306                                }
1307
1308                                // Reference params are only eligible for using a composite index if they
1309                                // are qualified
1310                                RuntimeSearchParam nextParamDef = mySearchParamRegistry.getActiveSearchParam(myResourceName, nextParamName);
1311                                if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
1312                                        ReferenceParam param = (ReferenceParam) nextValues.get(0).get(0);
1313                                        if (isBlank(param.getResourceType())) {
1314                                                sb = null;
1315                                                break;
1316                                        }
1317                                }
1318
1319                                List<? extends IQueryParameterType> nextAnd = nextValues.remove(0);
1320                                IQueryParameterType nextOr = nextAnd.remove(0);
1321                                String nextOrValue = nextOr.getValueAsQueryToken(myContext);
1322
1323                                if (comboParam.getComboSearchParamType() == ComboSearchParamType.NON_UNIQUE) {
1324                                        if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.STRING) {
1325                                                nextOrValue = StringUtil.normalizeStringForSearchIndexing(nextOrValue);
1326                                        }
1327                                }
1328
1329                                if (first) {
1330                                        first = false;
1331                                } else {
1332                                        sb.append('&');
1333                                }
1334
1335                                nextParamName = UrlUtil.escapeUrlParam(nextParamName);
1336                                nextOrValue = UrlUtil.escapeUrlParam(nextOrValue);
1337
1338                                sb.append(nextParamName).append('=').append(nextOrValue);
1339
1340                        }
1341
1342                        if (sb != null) {
1343                                String indexString = sb.toString();
1344                                ourLog.debug("Checking for {} combo index for query: {}", comboParam.getComboSearchParamType(), indexString);
1345
1346                                // Interceptor broadcast: JPA_PERFTRACE_INFO
1347                                StorageProcessingMessage msg = new StorageProcessingMessage()
1348                                        .setMessage("Using " + comboParam.getComboSearchParamType() + " index for query for search: " + indexString);
1349                                HookParams params = new HookParams()
1350                                        .add(RequestDetails.class, theRequest)
1351                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
1352                                        .add(StorageProcessingMessage.class, msg);
1353                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INFO, params);
1354
1355                                switch (comboParam.getComboSearchParamType()) {
1356                                        case UNIQUE:
1357                                                theQueryStack3.addPredicateCompositeUnique(indexString, myRequestPartitionId);
1358                                                break;
1359                                        case NON_UNIQUE:
1360                                                theQueryStack3.addPredicateCompositeNonUnique(indexString, myRequestPartitionId);
1361                                                break;
1362                                }
1363
1364                                // Remove any empty parameters remaining after this
1365                                theParams.clean();
1366                        }
1367                }
1368        }
1369
1370        private <T> void ensureSubListsAreWritable(List<List<T>> theListOfLists) {
1371                for (int i = 0; i < theListOfLists.size(); i++) {
1372                        List<T> oldSubList = theListOfLists.get(i);
1373                        if (!(oldSubList instanceof ArrayList)) {
1374                                List<T> newSubList = new ArrayList<>(oldSubList);
1375                                theListOfLists.set(i, newSubList);
1376                        }
1377                }
1378        }
1379
1380        @Override
1381        public void setFetchSize(int theFetchSize) {
1382                myFetchSize = theFetchSize;
1383        }
1384
1385        public SearchParameterMap getParams() {
1386                return myParams;
1387        }
1388
1389        public CriteriaBuilder getBuilder() {
1390                return myCriteriaBuilder;
1391        }
1392
1393        public Class<? extends IBaseResource> getResourceType() {
1394                return myResourceType;
1395        }
1396
1397        public String getResourceName() {
1398                return myResourceName;
1399        }
1400
1401        @VisibleForTesting
1402        public void setDaoConfigForUnitTest(DaoConfig theDaoConfig) {
1403                myDaoConfig = theDaoConfig;
1404        }
1405
1406        public class IncludesIterator extends BaseIterator<ResourcePersistentId> implements Iterator<ResourcePersistentId> {
1407
1408                private final RequestDetails myRequest;
1409                private final Set<ResourcePersistentId> myCurrentPids;
1410                private Iterator<ResourcePersistentId> myCurrentIterator;
1411                private ResourcePersistentId myNext;
1412
1413                IncludesIterator(Set<ResourcePersistentId> thePidSet, RequestDetails theRequest) {
1414                        myCurrentPids = new HashSet<>(thePidSet);
1415                        myCurrentIterator = null;
1416                        myRequest = theRequest;
1417                }
1418
1419                private void fetchNext() {
1420                        while (myNext == null) {
1421
1422                                if (myCurrentIterator == null) {
1423                                        Set<Include> includes = Collections.singleton(new Include("*", true));
1424                                        Set<ResourcePersistentId> newPids = loadIncludes(myContext, myEntityManager, myCurrentPids, includes, false, getParams().getLastUpdated(), mySearchUuid, myRequest, null);
1425                                        myCurrentIterator = newPids.iterator();
1426                                }
1427
1428                                if (myCurrentIterator.hasNext()) {
1429                                        myNext = myCurrentIterator.next();
1430                                } else {
1431                                        myNext = NO_MORE;
1432                                }
1433
1434                        }
1435                }
1436
1437                @Override
1438                public boolean hasNext() {
1439                        fetchNext();
1440                        return !NO_MORE.equals(myNext);
1441                }
1442
1443                @Override
1444                public ResourcePersistentId next() {
1445                        fetchNext();
1446                        ResourcePersistentId retVal = myNext;
1447                        myNext = null;
1448                        return retVal;
1449                }
1450
1451        }
1452
1453        private final class QueryIterator extends BaseIterator<ResourcePersistentId> implements IResultIterator {
1454
1455                private final SearchRuntimeDetails mySearchRuntimeDetails;
1456                private final RequestDetails myRequest;
1457                private final boolean myHaveRawSqlHooks;
1458                private final boolean myHavePerfTraceFoundIdHook;
1459                private final SortSpec mySort;
1460                private final Integer myOffset;
1461                private boolean myFirst = true;
1462                private IncludesIterator myIncludesIterator;
1463                private ResourcePersistentId myNext;
1464                private ISearchQueryExecutor myResultsIterator;
1465                private boolean myStillNeedToFetchIncludes;
1466                private int mySkipCount = 0;
1467                private int myNonSkipCount = 0;
1468                private List<ISearchQueryExecutor> myQueryList = new ArrayList<>();
1469
1470                private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
1471                        mySearchRuntimeDetails = theSearchRuntimeDetails;
1472                        mySort = myParams.getSort();
1473                        myOffset = myParams.getOffset();
1474                        myRequest = theRequest;
1475
1476                        // Includes are processed inline for $everything query
1477                        if (myParams.getEverythingMode() != null) {
1478                                myStillNeedToFetchIncludes = true;
1479                        }
1480
1481                        myHavePerfTraceFoundIdHook = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, myInterceptorBroadcaster, myRequest);
1482                        myHaveRawSqlHooks = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, myRequest);
1483
1484                }
1485
1486                private void fetchNext() {
1487
1488                        try {
1489                                if (myHaveRawSqlHooks) {
1490                                        CurrentThreadCaptureQueriesListener.startCapturing();
1491                                }
1492
1493                                // If we don't have a query yet, create one
1494                                if (myResultsIterator == null) {
1495                                        if (myMaxResultsToFetch == null) {
1496                                                if (myParams.getLoadSynchronousUpTo() != null) {
1497                                                        myMaxResultsToFetch = myParams.getLoadSynchronousUpTo();
1498                                                } else if (myParams.getOffset() != null && myParams.getCount() != null) {
1499                                                        myMaxResultsToFetch = myParams.getCount();
1500                                                } else {
1501                                                        myMaxResultsToFetch = myDaoConfig.getFetchSizeDefaultMaximum();
1502                                                }
1503                                        }
1504
1505                                        initializeIteratorQuery(myOffset, myMaxResultsToFetch);
1506
1507                                        if (myAlsoIncludePids == null) {
1508                                                myAlsoIncludePids = new ArrayList<>();
1509                                        }
1510                                }
1511
1512                                if (myNext == null) {
1513
1514
1515                                        for (Iterator<ResourcePersistentId> myPreResultsIterator = myAlsoIncludePids.iterator(); myPreResultsIterator.hasNext(); ) {
1516                                                ResourcePersistentId next = myPreResultsIterator.next();
1517                                                if (next != null)
1518                                                        if (myPidSet.add(next)) {
1519                                                                myNext = next;
1520                                                                break;
1521                                                        }
1522                                        }
1523
1524                                        if (myNext == null) {
1525                                                while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) {
1526                                                        // Update iterator with next chunk if necessary.
1527                                                        if (!myResultsIterator.hasNext()) {
1528                                                                retrieveNextIteratorQuery();
1529                                                        }
1530
1531                                                        Long nextLong = myResultsIterator.next();
1532                                                        if (myHavePerfTraceFoundIdHook) {
1533                                                                HookParams params = new HookParams()
1534                                                                        .add(Integer.class, System.identityHashCode(this))
1535                                                                        .add(Object.class, nextLong);
1536                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, params);
1537                                                        }
1538
1539                                                        if (nextLong != null) {
1540                                                                ResourcePersistentId next = new ResourcePersistentId(nextLong);
1541                                                                if (myPidSet.add(next)) {
1542                                                                        myNext = next;
1543                                                                        myNonSkipCount++;
1544                                                                        break;
1545                                                                } else {
1546                                                                        mySkipCount++;
1547                                                                }
1548                                                        }
1549
1550                                                        if (!myResultsIterator.hasNext()) {
1551                                                                if (myMaxResultsToFetch != null && (mySkipCount + myNonSkipCount == myMaxResultsToFetch)) {
1552                                                                        if (mySkipCount > 0 && myNonSkipCount == 0) {
1553
1554                                                                                StorageProcessingMessage message = new StorageProcessingMessage();
1555                                                                                String msg = "Pass completed with no matching results seeking rows " + myPidSet.size() + "-" + mySkipCount + ". This indicates an inefficient query! Retrying with new max count of " + myMaxResultsToFetch;
1556                                                                                ourLog.warn(msg);
1557                                                                                message.setMessage(msg);
1558                                                                                HookParams params = new HookParams()
1559                                                                                        .add(RequestDetails.class, myRequest)
1560                                                                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1561                                                                                        .add(StorageProcessingMessage.class, message);
1562                                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_WARNING, params);
1563
1564                                                                                myMaxResultsToFetch += 1000;
1565                                                                                initializeIteratorQuery(myOffset, myMaxResultsToFetch);
1566                                                                        }
1567                                                                }
1568                                                        }
1569                                                }
1570                                        }
1571
1572                                        if (myNext == null) {
1573                                                if (myStillNeedToFetchIncludes) {
1574                                                        myIncludesIterator = new IncludesIterator(myPidSet, myRequest);
1575                                                        myStillNeedToFetchIncludes = false;
1576                                                }
1577                                                if (myIncludesIterator != null) {
1578                                                        while (myIncludesIterator.hasNext()) {
1579                                                                ResourcePersistentId next = myIncludesIterator.next();
1580                                                                if (next != null)
1581                                                                        if (myPidSet.add(next)) {
1582                                                                                myNext = next;
1583                                                                                break;
1584                                                                        }
1585                                                        }
1586                                                        if (myNext == null) {
1587                                                                myNext = NO_MORE;
1588                                                        }
1589                                                } else {
1590                                                        myNext = NO_MORE;
1591                                                }
1592                                        }
1593
1594                                } // if we need to fetch the next result
1595
1596                                mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size());
1597
1598                        } finally {
1599                                if (myHaveRawSqlHooks) {
1600                                        SqlQueryList capturedQueries = CurrentThreadCaptureQueriesListener.getCurrentQueueAndStopCapturing();
1601                                        HookParams params = new HookParams()
1602                                                .add(RequestDetails.class, myRequest)
1603                                                .addIfMatchesType(ServletRequestDetails.class, myRequest)
1604                                                .add(SqlQueryList.class, capturedQueries);
1605                                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_RAW_SQL, params);
1606                                }
1607                        }
1608
1609                        if (myFirst) {
1610                                HookParams params = new HookParams()
1611                                        .add(RequestDetails.class, myRequest)
1612                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1613                                        .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
1614                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FIRST_RESULT_LOADED, params);
1615                                myFirst = false;
1616                        }
1617
1618                        if (NO_MORE.equals(myNext)) {
1619                                HookParams params = new HookParams()
1620                                        .add(RequestDetails.class, myRequest)
1621                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1622                                        .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
1623                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_SELECT_COMPLETE, params);
1624                        }
1625
1626                }
1627
1628                private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToFetch) {
1629                        if (myQueryList.isEmpty()) {
1630                                // wipmb what is this?
1631                                // Capture times for Lucene/Elasticsearch queries as well
1632                                mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
1633                                myQueryList = createQuery(myParams, mySort, theOffset, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails);
1634                        }
1635
1636                        mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
1637
1638                        retrieveNextIteratorQuery();
1639
1640                        mySkipCount = 0;
1641                        myNonSkipCount = 0;
1642                }
1643
1644                private void retrieveNextIteratorQuery() {
1645                        close();
1646                        if (myQueryList != null && myQueryList.size() > 0) {
1647                                myResultsIterator = myQueryList.remove(0);
1648                                myHasNextIteratorQuery = true;
1649                        } else {
1650                                myResultsIterator = SearchQueryExecutor.emptyExecutor();
1651                                myHasNextIteratorQuery = false;
1652                        }
1653
1654                }
1655
1656                @Override
1657                public boolean hasNext() {
1658                        if (myNext == null) {
1659                                fetchNext();
1660                        }
1661                        return !NO_MORE.equals(myNext);
1662                }
1663
1664                @Override
1665                public ResourcePersistentId next() {
1666                        fetchNext();
1667                        ResourcePersistentId retVal = myNext;
1668                        myNext = null;
1669                        Validate.isTrue(!NO_MORE.equals(retVal), "No more elements");
1670                        return retVal;
1671                }
1672
1673                @Override
1674                public int getSkippedCount() {
1675                        return mySkipCount;
1676                }
1677
1678                @Override
1679                public int getNonSkippedCount() {
1680                        return myNonSkipCount;
1681                }
1682
1683                @Override
1684                public Collection<ResourcePersistentId> getNextResultBatch(long theBatchSize) {
1685                        Collection<ResourcePersistentId> batch = new ArrayList<>();
1686                        while (this.hasNext() && batch.size() < theBatchSize) {
1687                                batch.add(this.next());
1688                        }
1689                        return batch;
1690                }
1691
1692                @Override
1693                public void close() {
1694                        if (myResultsIterator != null) {
1695                                myResultsIterator.close();
1696                        }
1697                        myResultsIterator = null;
1698                }
1699
1700        }
1701
1702        public static int getMaximumPageSize() {
1703                if (myUseMaxPageSize50ForTest) {
1704                        return MAXIMUM_PAGE_SIZE_FOR_TESTING;
1705                } else {
1706                        return MAXIMUM_PAGE_SIZE;
1707                }
1708        }
1709
1710        public static void setMaxPageSize50ForTest(boolean theIsTest) {
1711                myUseMaxPageSize50ForTest = theIsTest;
1712        }
1713
1714        private static List<Predicate> createLastUpdatedPredicates(final DateRangeParam theLastUpdated, CriteriaBuilder builder, From<?, ResourceTable> from) {
1715                List<Predicate> lastUpdatedPredicates = new ArrayList<>();
1716                if (theLastUpdated != null) {
1717                        if (theLastUpdated.getLowerBoundAsInstant() != null) {
1718                                ourLog.debug("LastUpdated lower bound: {}", new InstantDt(theLastUpdated.getLowerBoundAsInstant()));
1719                                Predicate predicateLower = builder.greaterThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getLowerBoundAsInstant());
1720                                lastUpdatedPredicates.add(predicateLower);
1721                        }
1722                        if (theLastUpdated.getUpperBoundAsInstant() != null) {
1723                                Predicate predicateUpper = builder.lessThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getUpperBoundAsInstant());
1724                                lastUpdatedPredicates.add(predicateUpper);
1725                        }
1726                }
1727                return lastUpdatedPredicates;
1728        }
1729
1730        private static List<ResourcePersistentId> filterResourceIdsByLastUpdated(EntityManager theEntityManager, final DateRangeParam theLastUpdated, Collection<ResourcePersistentId> thePids) {
1731                if (thePids.isEmpty()) {
1732                        return Collections.emptyList();
1733                }
1734                CriteriaBuilder builder = theEntityManager.getCriteriaBuilder();
1735                CriteriaQuery<Long> cq = builder.createQuery(Long.class);
1736                Root<ResourceTable> from = cq.from(ResourceTable.class);
1737                cq.select(from.get("myId").as(Long.class));
1738
1739                List<Predicate> lastUpdatedPredicates = createLastUpdatedPredicates(theLastUpdated, builder, from);
1740                lastUpdatedPredicates.add(from.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(thePids)));
1741
1742                cq.where(SearchBuilder.toPredicateArray(lastUpdatedPredicates));
1743                TypedQuery<Long> query = theEntityManager.createQuery(cq);
1744
1745                return ResourcePersistentId.fromLongList(query.getResultList());
1746        }
1747
1748        public static Predicate[] toPredicateArray(List<Predicate> thePredicates) {
1749                return thePredicates.toArray(new Predicate[0]);
1750        }
1751
1752}