Source code

001package ca.uhn.fhir.jpa.dao;
002
003/*
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2022 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 * http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.context.ComboSearchParamType;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.context.FhirVersionEnum;
026import ca.uhn.fhir.context.RuntimeResourceDefinition;
027import ca.uhn.fhir.context.RuntimeSearchParam;
028import ca.uhn.fhir.i18n.Msg;
029import ca.uhn.fhir.interceptor.api.HookParams;
030import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
031import ca.uhn.fhir.interceptor.api.Pointcut;
032import ca.uhn.fhir.interceptor.model.RequestPartitionId;
033import ca.uhn.fhir.jpa.api.config.DaoConfig;
034import ca.uhn.fhir.jpa.api.dao.IDao;
035import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
036import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
037import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
038import ca.uhn.fhir.jpa.dao.predicate.PredicateBuilder;
039import ca.uhn.fhir.jpa.dao.predicate.PredicateBuilderFactory;
040import ca.uhn.fhir.jpa.dao.predicate.SearchBuilderJoinEnum;
041import ca.uhn.fhir.jpa.dao.predicate.SearchBuilderJoinKey;
042import ca.uhn.fhir.jpa.dao.predicate.querystack.QueryStack;
043import ca.uhn.fhir.jpa.entity.ResourceSearchView;
044import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
045import ca.uhn.fhir.jpa.model.config.PartitionSettings;
046import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
047import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique;
048import ca.uhn.fhir.jpa.model.entity.ResourceLink;
049import ca.uhn.fhir.jpa.model.entity.ResourceTable;
050import ca.uhn.fhir.jpa.model.entity.ResourceTag;
051import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
052import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
053import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
054import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc;
055import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
056import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
057import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
058import ca.uhn.fhir.jpa.util.BaseIterator;
059import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
060import ca.uhn.fhir.jpa.util.QueryChunker;
061import ca.uhn.fhir.jpa.util.ScrollableResultsIterator;
062import ca.uhn.fhir.jpa.util.SqlQueryList;
063import ca.uhn.fhir.model.api.IQueryParameterType;
064import ca.uhn.fhir.model.api.IResource;
065import ca.uhn.fhir.model.api.Include;
066import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
067import ca.uhn.fhir.model.primitive.InstantDt;
068import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum;
069import ca.uhn.fhir.rest.api.Constants;
070import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
071import ca.uhn.fhir.rest.api.SortOrderEnum;
072import ca.uhn.fhir.rest.api.SortSpec;
073import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
074import ca.uhn.fhir.rest.api.server.RequestDetails;
075import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
076import ca.uhn.fhir.rest.param.DateRangeParam;
077import ca.uhn.fhir.rest.param.ReferenceParam;
078import ca.uhn.fhir.rest.param.StringParam;
079import ca.uhn.fhir.rest.param.TokenParam;
080import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
081import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
082import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
083import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
084import ca.uhn.fhir.util.StopWatch;
085import ca.uhn.fhir.util.UrlUtil;
086import com.google.common.annotations.VisibleForTesting;
087import org.apache.commons.lang3.Validate;
088import org.hibernate.ScrollMode;
089import org.hibernate.ScrollableResults;
090import org.hibernate.query.Query;
091import org.hl7.fhir.instance.model.api.IAnyResource;
092import org.hl7.fhir.instance.model.api.IBaseResource;
093import org.slf4j.Logger;
094import org.slf4j.LoggerFactory;
095import org.springframework.beans.factory.annotation.Autowired;
096import org.springframework.transaction.support.TransactionSynchronizationManager;
097
098import javax.annotation.Nonnull;
099import javax.persistence.EntityManager;
100import javax.persistence.PersistenceContext;
101import javax.persistence.PersistenceContextType;
102import javax.persistence.TypedQuery;
103import javax.persistence.criteria.CriteriaBuilder;
104import javax.persistence.criteria.CriteriaQuery;
105import javax.persistence.criteria.From;
106import javax.persistence.criteria.Join;
107import javax.persistence.criteria.Order;
108import javax.persistence.criteria.Predicate;
109import javax.persistence.criteria.Root;
110import java.util.ArrayList;
111import java.util.Collection;
112import java.util.Collections;
113import java.util.HashMap;
114import java.util.HashSet;
115import java.util.Iterator;
116import java.util.List;
117import java.util.Map;
118import java.util.Optional;
119import java.util.Set;
120
121import static ca.uhn.fhir.jpa.search.builder.SearchBuilder.getMaximumPageSize;
122import static org.apache.commons.lang3.StringUtils.defaultString;
123import static org.apache.commons.lang3.StringUtils.isBlank;
124import static org.apache.commons.lang3.StringUtils.isNotBlank;
125
126/**
127 * The SearchBuilder is responsible for actually forming the SQL query that handles
128 * searches for resources
129 */
130public class LegacySearchBuilder implements ISearchBuilder {
131
132        private static final List<ResourcePersistentId> EMPTY_LONG_LIST = Collections.unmodifiableList(new ArrayList<>());
133        private static final Logger ourLog = LoggerFactory.getLogger(LegacySearchBuilder.class);
134        private static final ResourcePersistentId NO_MORE = new ResourcePersistentId(-1L);
135        private final String myResourceName;
136        private final Class<? extends IBaseResource> myResourceType;
137        private final IDao myCallingDao;
138        @Autowired
139        protected IInterceptorBroadcaster myInterceptorBroadcaster;
140        @Autowired
141        protected IResourceTagDao myResourceTagDao;
142        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
143        protected EntityManager myEntityManager;
144        private QueryStack myQueryStack;
145        @Autowired
146        private DaoConfig myDaoConfig;
147        @Autowired
148        private IResourceSearchViewDao myResourceSearchViewDao;
149        @Autowired
150        private FhirContext myContext;
151        @Autowired
152        private IIdHelperService myIdHelperService;
153        @Autowired(required = false)
154        private IFulltextSearchSvc myFulltextSearchSvc;
155        @Autowired(required = false)
156        private IElasticsearchSvc myIElasticsearchSvc;
157        @Autowired
158        private ISearchParamRegistry mySearchParamRegistry;
159        @Autowired
160        private PredicateBuilderFactory myPredicateBuilderFactory;
161        private List<ResourcePersistentId> myAlsoIncludePids;
162        private CriteriaBuilder myCriteriaBuilder;
163        private SearchParameterMap myParams;
164        private String mySearchUuid;
165        private int myFetchSize;
166        private Integer myMaxResultsToFetch;
167        private Set<ResourcePersistentId> myPidSet;
168        private PredicateBuilder myPredicateBuilder;
169        private RequestPartitionId myRequestPartitionId;
170        @Autowired
171        private PartitionSettings myPartitionSettings;
172
173        /**
174         * Constructor
175         */
176        public LegacySearchBuilder(IDao theDao, String theResourceName, Class<? extends IBaseResource> theResourceType) {
177                myCallingDao = theDao;
178                myResourceName = theResourceName;
179                myResourceType = theResourceType;
180        }
181
182        @Override
183        public void setMaxResultsToFetch(Integer theMaxResultsToFetch) {
184                myMaxResultsToFetch = theMaxResultsToFetch;
185        }
186
187        private void searchForIdsWithAndOr(String theResourceName, String theNextParamName, List<List<IQueryParameterType>> theAndOrParams, RequestDetails theRequest) {
188                myPredicateBuilder.searchForIdsWithAndOr(theResourceName, theNextParamName, theAndOrParams, theRequest, myRequestPartitionId);
189        }
190
191        private void searchForIdsWithAndOr(@Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
192                myParams = theParams;
193
194                // Remove any empty parameters
195                theParams.clean();
196
197                // For DSTU3, pull out near-distance first so when it comes time to evaluate near, we already know the distance
198                if (myContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) {
199                        Dstu3DistanceHelper.setNearDistance(myResourceType, theParams);
200                }
201
202                // Attempt to lookup via composite unique key.
203                if (isCompositeUniqueSpCandidate()) {
204                        attemptCompositeUniqueSpProcessing(theParams, theRequest);
205                }
206
207                // Handle each parameter
208                for (Map.Entry<String, List<List<IQueryParameterType>>> nextParamEntry : myParams.entrySet()) {
209                        String nextParamName = nextParamEntry.getKey();
210                        if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) {
211                                // Skip parameters for Subject, Patient, Code and Category for LastN as these will be filtered by Elasticsearch
212                                continue;
213                        }
214                        List<List<IQueryParameterType>> andOrParams = nextParamEntry.getValue();
215                        searchForIdsWithAndOr(myResourceName, nextParamName, andOrParams, theRequest);
216                }
217        }
218
219        /**
220         * A search is a candidate for Composite Unique SP if unique indexes are enabled, there is no EverythingMode, and the
221         * parameters all have no modifiers.
222         */
223        private boolean isCompositeUniqueSpCandidate() {
224                return myDaoConfig.isUniqueIndexesEnabled() &&
225                        myParams.getEverythingMode() == null &&
226                        myParams.isAllParametersHaveNoModifier();
227        }
228
229        @Override
230        public Long createCountQuery(SearchParameterMap theParams, String theSearchUuid, RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) {
231                assert theRequestPartitionId != null;
232                assert TransactionSynchronizationManager.isActualTransactionActive();
233
234                init(theParams, theSearchUuid, theRequestPartitionId);
235
236                List<TypedQuery<Long>> queries = createQuery(null, null, null, true, theRequest, null);
237                return new CountQueryIterator(queries.get(0)).next();
238        }
239
240        /**
241         * @param thePidSet May be null
242         */
243        @Override
244        public void setPreviouslyAddedResourcePids(@Nonnull List<ResourcePersistentId> thePidSet) {
245                myPidSet = new HashSet<>(thePidSet);
246        }
247
248        @Override
249        public IResultIterator createQuery(SearchParameterMap theParams, SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) {
250                assert theRequestPartitionId != null;
251                assert TransactionSynchronizationManager.isActualTransactionActive();
252
253                init(theParams, theSearchRuntimeDetails.getSearchUuid(), theRequestPartitionId);
254
255                if (myPidSet == null) {
256                        myPidSet = new HashSet<>();
257                }
258
259                return new QueryIterator(theSearchRuntimeDetails, theRequest);
260        }
261
262        private void init(SearchParameterMap theParams, String theSearchUuid, RequestPartitionId theRequestPartitionId) {
263                myCriteriaBuilder = myEntityManager.getCriteriaBuilder();
264                myQueryStack = new QueryStack(myCriteriaBuilder, myResourceName, theParams, theRequestPartitionId);
265                myParams = theParams;
266                mySearchUuid = theSearchUuid;
267                myPredicateBuilder = new PredicateBuilder(this, myPredicateBuilderFactory);
268                myRequestPartitionId = theRequestPartitionId;
269        }
270
271        private List<TypedQuery<Long>> createQuery(SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCount, RequestDetails theRequest,
272                                                                                                                         SearchRuntimeDetails theSearchRuntimeDetails) {
273
274                List<ResourcePersistentId> pids = new ArrayList<>();
275
276                /*
277                 * Fulltext or lastn search
278                 */
279                if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) {
280                        if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) {
281                                if (myFulltextSearchSvc == null || myFulltextSearchSvc.isDisabled()) {
282                                        if (myParams.containsKey(Constants.PARAM_TEXT)) {
283                                                throw new InvalidRequestException(Msg.code(937) + "Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT);
284                                        } else if (myParams.containsKey(Constants.PARAM_CONTENT)) {
285                                                throw new InvalidRequestException(Msg.code(938) + "Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT);
286                                        }
287                                }
288
289                                if (myParams.getEverythingMode() != null) {
290                                        pids = queryHibernateSearchForEverythingPids();
291                                } else {
292                                        pids = myFulltextSearchSvc.search(myResourceName, myParams);
293                                }
294                        } else if (myParams.isLastN()) {
295                                if (myIElasticsearchSvc == null) {
296                                        if (myParams.isLastN()) {
297                                                throw new InvalidRequestException(Msg.code(939) + "LastN operation is not enabled on this service, can not process this request");
298                                        }
299                                }
300                                List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults);
301                                for (String lastnResourceId : lastnResourceIds) {
302                                        pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId));
303                                }
304                        }
305                        if (theSearchRuntimeDetails != null) {
306                                theSearchRuntimeDetails.setFoundIndexMatchesCount(pids.size());
307                                HookParams params = new HookParams()
308                                        .add(RequestDetails.class, theRequest)
309                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
310                                        .add(SearchRuntimeDetails.class, theSearchRuntimeDetails);
311                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params);
312                        }
313
314                        if (pids.isEmpty()) {
315                                // Will never match
316                                pids = Collections.singletonList(new ResourcePersistentId(-1L));
317                        }
318
319                }
320
321                ArrayList<TypedQuery<Long>> myQueries = new ArrayList<>();
322
323                if (!pids.isEmpty()) {
324                        if (theMaximumResults != null && pids.size() > theMaximumResults) {
325                                pids.subList(0,theMaximumResults-1);
326                        }
327                        new QueryChunker<Long>().chunk(ResourcePersistentId.toLongList(pids), t-> doCreateChunkedQueries(t, sort, theOffset, theCount, theRequest, myQueries));
328                } else {
329                        myQueries.add(createChunkedQuery(sort, theOffset, theMaximumResults, theCount, theRequest, null));
330                }
331
332                return myQueries;
333        }
334
335        private List<ResourcePersistentId> queryHibernateSearchForEverythingPids() {
336                ResourcePersistentId pid = null;
337                if (myParams.get(IAnyResource.SP_RES_ID) != null) {
338                        String idParamValue;
339                        IQueryParameterType idParam = myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
340                        if (idParam instanceof TokenParam) {
341                                TokenParam idParm = (TokenParam) idParam;
342                                idParamValue = idParm.getValue();
343                        } else {
344                                StringParam idParm = (StringParam) idParam;
345                                idParamValue = idParm.getValue();
346                        }
347
348                        pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
349                }
350                List<ResourcePersistentId> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid);
351                return pids;
352        }
353
354        private void doCreateChunkedQueries(List<Long> thePids, SortSpec sort, Integer theOffset, boolean theCount, RequestDetails theRequest, ArrayList<TypedQuery<Long>> theQueries) {
355                if(thePids.size() < getMaximumPageSize()) {
356                        normalizeIdListForLastNInClause(thePids);
357                }
358                theQueries.add(createChunkedQuery(sort, theOffset, thePids.size(), theCount, theRequest, thePids));
359        }
360
361        private TypedQuery<Long> createChunkedQuery(SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List<Long> thePidList) {
362                /*
363                 * Sort
364                 *
365                 * If we have a sort, we wrap the criteria search (the search that actually
366                 * finds the appropriate resources) in an outer search which is then sorted
367                 */
368                if (sort != null) {
369                        assert !theCount;
370
371                        myQueryStack.pushResourceTableQuery();
372
373                        List<Order> orders = createSort(myCriteriaBuilder, myQueryStack, sort);
374                        if (orders.size() > 0) {
375                                myQueryStack.orderBy(orders);
376                        }
377
378                } else {
379
380                        if (theCount) {
381                                myQueryStack.pushResourceTableCountQuery();
382                        } else if (myParams.getEverythingMode() != null && myParams.isLoadSynchronous()) {
383                                myQueryStack.pushResourceTableDistinctQuery();
384                        } else {
385                                myQueryStack.pushResourceTableQuery();
386                        }
387                }
388
389                if (myParams.getEverythingMode() != null) {
390                        From<?, ResourceLink> join = myQueryStack.createJoin(SearchBuilderJoinEnum.REFERENCE, null);
391
392                        if (myParams.get(IAnyResource.SP_RES_ID) != null) {
393                                ResourcePersistentId pid = null;
394                                if (myParams.get(IAnyResource.SP_RES_ID) instanceof StringParam) {
395                                        StringParam idParam = (StringParam) myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
396                                        pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParam.getValue());
397                                } else {
398                                        TokenParam tokenParam = (TokenParam) myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
399                                        pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, tokenParam.getValue());
400                                }
401                                if (myAlsoIncludePids == null) {
402                                        myAlsoIncludePids = new ArrayList<>(1);
403                                }
404                                myAlsoIncludePids.add(pid);
405                                myQueryStack.addPredicate(myCriteriaBuilder.equal(join.get("myTargetResourcePid").as(Long.class), pid.getIdAsLong()));
406                        } else {
407                                Predicate targetTypePredicate = myCriteriaBuilder.equal(join.get("myTargetResourceType").as(String.class), myResourceName);
408                                Predicate sourceTypePredicate = myCriteriaBuilder.equal(myQueryStack.get("myResourceType").as(String.class), myResourceName);
409                                myQueryStack.addPredicate(myCriteriaBuilder.or(sourceTypePredicate, targetTypePredicate));
410                        }
411
412                } else {
413                        // Normal search
414                        searchForIdsWithAndOr(myParams, theRequest);
415                }
416
417                // Add PID list predicate for full text search and/or lastn operation
418                if (thePidList != null && thePidList.size() > 0) {
419                        myQueryStack.addPredicate(myQueryStack.get("myId").as(Long.class).in(thePidList));
420                }
421
422                // Last updated
423                DateRangeParam lu = myParams.getLastUpdated();
424                List<Predicate> lastUpdatedPredicates = createLastUpdatedPredicates(lu, myCriteriaBuilder);
425                myQueryStack.addPredicates(lastUpdatedPredicates);
426
427                /*
428                 * Now perform the search
429                 */
430                CriteriaQuery<Long> outerQuery = (CriteriaQuery<Long>) myQueryStack.pop();
431                final TypedQuery<Long> query = myEntityManager.createQuery(outerQuery);
432                assert myQueryStack.isEmpty();
433                if (!theCount && theOffset != null) {
434                        query.setFirstResult(theOffset);
435                }
436                if (theMaximumResults != null) {
437                        query.setMaxResults(theMaximumResults);
438                }
439
440                return query;
441        }
442
443        private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
444                /*
445                        The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying
446                        numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info:
447                        https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage.
448
449                        Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of
450                        arguments never exceeds the maximum specified below.
451                 */
452                int listSize = lastnResourceIds.size();
453
454                if(listSize > 1 && listSize < 10) {
455                        padIdListWithPlaceholders(lastnResourceIds, 10);
456                } else if (listSize > 10 && listSize < 50) {
457                        padIdListWithPlaceholders(lastnResourceIds, 50);
458                } else if (listSize > 50 && listSize < 100) {
459                        padIdListWithPlaceholders(lastnResourceIds, 100);
460                } else if (listSize > 100 && listSize < 200) {
461                        padIdListWithPlaceholders(lastnResourceIds, 200);
462                } else if (listSize > 200 && listSize < 500) {
463                        padIdListWithPlaceholders(lastnResourceIds, 500);
464                } else if (listSize > 500 && listSize < 800) {
465                        padIdListWithPlaceholders(lastnResourceIds, 800);
466                }
467
468                return lastnResourceIds;
469        }
470
471        private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
472                while(theIdList.size() < preferredListSize) {
473                        theIdList.add(-1L);
474                }
475        }
476
477        /**
478         * @return Returns {@literal true} if any search parameter sorts were found, or false if
479         * no sorts were found, or only non-search parameters ones (e.g. _id, _lastUpdated)
480         */
481        private List<Order> createSort(CriteriaBuilder theBuilder, QueryStack theQueryStack, SortSpec theSort) {
482                if (theSort == null || isBlank(theSort.getParamName())) {
483                        return Collections.emptyList();
484                }
485
486                List<Order> orders = new ArrayList<>(1);
487                if (IAnyResource.SP_RES_ID.equals(theSort.getParamName())) {
488                        From<?, ?> forcedIdJoin = theQueryStack.createJoin(SearchBuilderJoinEnum.FORCED_ID, null);
489                        if (theSort.getOrder() == null || theSort.getOrder() == SortOrderEnum.ASC) {
490                                orders.add(theBuilder.asc(forcedIdJoin.get("myForcedId")));
491                                orders.add(theBuilder.asc(theQueryStack.get("myId")));
492                        } else {
493                                orders.add(theBuilder.desc(forcedIdJoin.get("myForcedId")));
494                                orders.add(theBuilder.desc(theQueryStack.get("myId")));
495                        }
496
497                        orders.addAll(createSort(theBuilder, theQueryStack, theSort.getChain()));
498                        return orders;
499                }
500
501                if (Constants.PARAM_LASTUPDATED.equals(theSort.getParamName())) {
502                        if (theSort.getOrder() == null || theSort.getOrder() == SortOrderEnum.ASC) {
503                                orders.add(theBuilder.asc(theQueryStack.get("myUpdated")));
504                        } else {
505                                orders.add(theBuilder.desc(theQueryStack.get("myUpdated")));
506                        }
507
508                        orders.addAll(createSort(theBuilder, theQueryStack, theSort.getChain()));
509                        return orders;
510                }
511
512                RuntimeSearchParam param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName());
513                if (param == null) {
514                        throw new InvalidRequestException(Msg.code(940) + "Unknown sort parameter '" + theSort.getParamName() + "'");
515                }
516
517                String[] sortAttrName;
518                SearchBuilderJoinEnum joinType;
519
520                switch (param.getParamType()) {
521                        case STRING:
522                                sortAttrName = new String[]{"myValueExact"};
523                                joinType = SearchBuilderJoinEnum.STRING;
524                                break;
525                        case DATE:
526                                sortAttrName = new String[]{"myValueLow"};
527                                joinType = SearchBuilderJoinEnum.DATE;
528                                break;
529                        case REFERENCE:
530                                sortAttrName = new String[]{"myTargetResourcePid"};
531                                joinType = SearchBuilderJoinEnum.REFERENCE;
532                                break;
533                        case TOKEN:
534                                sortAttrName = new String[]{"mySystem", "myValue"};
535                                joinType = SearchBuilderJoinEnum.TOKEN;
536                                break;
537                        case NUMBER:
538                                sortAttrName = new String[]{"myValue"};
539                                joinType = SearchBuilderJoinEnum.NUMBER;
540                                break;
541                        case URI:
542                                sortAttrName = new String[]{"myUri"};
543                                joinType = SearchBuilderJoinEnum.URI;
544                                break;
545                        case QUANTITY:
546                                sortAttrName = new String[]{"myValue"};
547                                joinType = SearchBuilderJoinEnum.QUANTITY;
548                                break;
549                        case SPECIAL:
550                        case COMPOSITE:
551                        case HAS:
552                        default:
553                                throw new InvalidRequestException(Msg.code(941) + "This server does not support _sort specifications of type " + param.getParamType() + " - Can't serve _sort=" + theSort.getParamName());
554                }
555
556                /*
557                 * If we've already got a join for the specific parameter we're
558                 * sorting on, we'll also sort with it. Otherwise we need a new join.
559                 */
560                SearchBuilderJoinKey key = new SearchBuilderJoinKey(theSort.getParamName(), joinType);
561                Optional<Join<?, ?>> joinOpt = theQueryStack.getExistingJoin(key);
562
563                From<?, ?> join;
564                if (!joinOpt.isPresent()) {
565                        join = theQueryStack.createJoin(joinType, theSort.getParamName());
566
567                        if (param.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
568                                theQueryStack.addPredicate(join.get("mySourcePath").as(String.class).in(param.getPathsSplit()));
569                        } else {
570                                if (myDaoConfig.getDisableHashBasedSearches()) {
571                                        Predicate joinParam1 = theBuilder.equal(join.get("myParamName"), theSort.getParamName());
572                                        theQueryStack.addPredicate(joinParam1);
573                                } else {
574                                        Long hashIdentity = BaseResourceIndexedSearchParam.calculateHashIdentity(myPartitionSettings, myRequestPartitionId, myResourceName, theSort.getParamName());
575                                        Predicate joinParam1 = theBuilder.equal(join.get("myHashIdentity"), hashIdentity);
576                                        theQueryStack.addPredicate(joinParam1);
577                                }
578                        }
579                } else {
580                        ourLog.debug("Reusing join for {}", theSort.getParamName());
581                        join = joinOpt.get();
582                }
583
584                for (String next : sortAttrName) {
585                        if (theSort.getOrder() == null || theSort.getOrder() == SortOrderEnum.ASC) {
586                                orders.add(theBuilder.asc(join.get(next)));
587                        } else {
588                                orders.add(theBuilder.desc(join.get(next)));
589                        }
590                }
591
592                orders.addAll(createSort(theBuilder, theQueryStack, theSort.getChain()));
593
594                return orders;
595        }
596
597
598        private void doLoadPids(Collection<ResourcePersistentId> thePids, Collection<ResourcePersistentId> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation,
599                                                                        Map<ResourcePersistentId, Integer> thePosition) {
600
601                List<Long> myLongPersistentIds;
602                if(thePids.size() < getMaximumPageSize()) {
603                        myLongPersistentIds = normalizeIdListForLastNInClause(ResourcePersistentId.toLongList(thePids));
604                } else {
605                        myLongPersistentIds = ResourcePersistentId.toLongList(thePids);
606                }
607
608                // -- get the resource from the searchView
609                Collection<ResourceSearchView> resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(myLongPersistentIds);
610
611                //-- preload all tags with tag definition if any
612                Map<ResourcePersistentId, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList);
613
614                ResourcePersistentId resourceId;
615                for (ResourceSearchView next : resourceSearchViewList) {
616                        if (next.getDeleted() != null) {
617                                continue;
618                        }
619
620                        Class<? extends IBaseResource> resourceType = myContext.getResourceDefinition(next.getResourceType()).getImplementingClass();
621
622                        resourceId = new ResourcePersistentId(next.getId());
623
624                        IBaseResource resource = myCallingDao.toResource(resourceType, next, tagMap.get(resourceId), theForHistoryOperation);
625                        if (resource == null) {
626                                ourLog.warn("Unable to find resource {}/{}/_history/{} in database", next.getResourceType(), next.getIdDt().getIdPart(), next.getVersion());
627                                continue;
628                        }
629                        Integer index = thePosition.get(resourceId);
630                        if (index == null) {
631                                ourLog.warn("Got back unexpected resource PID {}", resourceId);
632                                continue;
633                        }
634
635                        if (resource instanceof IResource) {
636                                if (theIncludedPids.contains(resourceId)) {
637                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IResource) resource, BundleEntrySearchModeEnum.INCLUDE);
638                                } else {
639                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IResource) resource, BundleEntrySearchModeEnum.MATCH);
640                                }
641                        } else {
642                                if (theIncludedPids.contains(resourceId)) {
643                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IAnyResource) resource, BundleEntrySearchModeEnum.INCLUDE.getCode());
644                                } else {
645                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IAnyResource) resource, BundleEntrySearchModeEnum.MATCH.getCode());
646                                }
647                        }
648
649                        theResourceListToPopulate.set(index, resource);
650                }
651        }
652
653        private Map<ResourcePersistentId, Collection<ResourceTag>> getResourceTagMap(Collection<ResourceSearchView> theResourceSearchViewList) {
654
655                List<Long> idList = new ArrayList<>(theResourceSearchViewList.size());
656
657                //-- find all resource has tags
658                for (ResourceSearchView resource : theResourceSearchViewList) {
659                        if (resource.isHasTags())
660                                idList.add(resource.getId());
661                }
662
663                Map<ResourcePersistentId, Collection<ResourceTag>> tagMap = new HashMap<>();
664
665                //-- no tags
666                if (idList.size() == 0)
667                        return tagMap;
668
669                //-- get all tags for the idList
670                Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(idList);
671
672                //-- build the map, key = resourceId, value = list of ResourceTag
673                ResourcePersistentId resourceId;
674                Collection<ResourceTag> tagCol;
675                for (ResourceTag tag : tagList) {
676
677                        resourceId = new ResourcePersistentId(tag.getResourceId());
678                        tagCol = tagMap.get(resourceId);
679                        if (tagCol == null) {
680                                tagCol = new ArrayList<>();
681                                tagCol.add(tag);
682                                tagMap.put(resourceId, tagCol);
683                        } else {
684                                tagCol.add(tag);
685                        }
686                }
687
688                return tagMap;
689        }
690
691        @Override
692        public void loadResourcesByPid(Collection<ResourcePersistentId> thePids, Collection<ResourcePersistentId> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation, RequestDetails theDetails) {
693                if (thePids.isEmpty()) {
694                        ourLog.debug("The include pids are empty");
695                        // return;
696                }
697
698                // Dupes will cause a crash later anyhow, but this is expensive so only do it
699                // when running asserts
700                assert new HashSet<>(thePids).size() == thePids.size() : "PID list contains duplicates: " + thePids;
701
702                Map<ResourcePersistentId, Integer> position = new HashMap<>();
703                for (ResourcePersistentId next : thePids) {
704                        position.put(next, theResourceListToPopulate.size());
705                        theResourceListToPopulate.add(null);
706                }
707
708                List<ResourcePersistentId> pids = new ArrayList<>(thePids);
709                new QueryChunker<ResourcePersistentId>().chunk(pids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
710
711        }
712
713        /**
714         * THIS SHOULD RETURN HASHSET and not just Set because we add to it later
715         * so it can't be Collections.emptySet() or some such thing
716         */
717        @Override
718        public HashSet<ResourcePersistentId> loadIncludes(FhirContext theContext, EntityManager theEntityManager, Collection<ResourcePersistentId> theMatches, Set<Include> theRevIncludes,
719                                                                                                                                          boolean theReverseMode, DateRangeParam theLastUpdated, String theSearchIdOrDescription, RequestDetails theRequest, Integer theMaxCount) {
720                if (theMatches.size() == 0) {
721                        return new HashSet<>();
722                }
723                if (theRevIncludes == null || theRevIncludes.isEmpty()) {
724                        return new HashSet<>();
725                }
726                String searchFieldName = theReverseMode ? "myTargetResourcePid" : "mySourceResourcePid";
727                String findFieldName = theReverseMode ? "mySourceResourcePid" : "myTargetResourcePid";
728
729                Collection<ResourcePersistentId> nextRoundMatches = theMatches;
730                HashSet<ResourcePersistentId> allAdded = new HashSet<>();
731                HashSet<ResourcePersistentId> original = new HashSet<>(theMatches);
732                ArrayList<Include> includes = new ArrayList<>(theRevIncludes);
733
734                int roundCounts = 0;
735                StopWatch w = new StopWatch();
736
737                boolean addedSomeThisRound;
738                do {
739                        roundCounts++;
740
741                        HashSet<ResourcePersistentId> pidsToInclude = new HashSet<>();
742
743                        for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) {
744                                Include nextInclude = iter.next();
745                                if (nextInclude.isRecurse() == false) {
746                                        iter.remove();
747                                }
748
749                                boolean matchAll = "*".equals(nextInclude.getValue());
750                                if (matchAll) {
751                                        String sql;
752                                        sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r." + searchFieldName + " IN (:target_pids) ";
753                                        List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
754                                        for (Collection<ResourcePersistentId> nextPartition : partitions) {
755                                                TypedQuery<Long> q = theEntityManager.createQuery(sql, Long.class);
756                                                q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition));
757                                                List<Long> results = q.getResultList();
758                                                for (Long resourceLink : results) {
759                                                        if (resourceLink == null) {
760                                                                continue;
761                                                        }
762                                                        if (theReverseMode) {
763                                                                pidsToInclude.add(new ResourcePersistentId(resourceLink));
764                                                        } else {
765                                                                pidsToInclude.add(new ResourcePersistentId(resourceLink));
766                                                        }
767                                                }
768                                        }
769                                } else {
770
771                                        List<String> paths;
772                                        RuntimeSearchParam param;
773                                        String resType = nextInclude.getParamType();
774                                        if (isBlank(resType)) {
775                                                continue;
776                                        }
777                                        RuntimeResourceDefinition def = theContext.getResourceDefinition(resType);
778                                        if (def == null) {
779                                                ourLog.warn("Unknown resource type in include/revinclude=" + nextInclude.getValue());
780                                                continue;
781                                        }
782
783                                        String paramName = nextInclude.getParamName();
784                                        if (isNotBlank(paramName)) {
785                                                param = mySearchParamRegistry.getActiveSearchParam(resType, paramName);
786                                        } else {
787                                                param = null;
788                                        }
789                                        if (param == null) {
790                                                ourLog.warn("Unknown param name in include/revinclude=" + nextInclude.getValue());
791                                                continue;
792                                        }
793
794                                        paths = param.getPathsSplit();
795
796                                        String targetResourceType = defaultString(nextInclude.getParamTargetType(), null);
797                                        for (String nextPath : paths) {
798                                                String sql;
799
800                                                boolean haveTargetTypesDefinedByParam = param.hasTargets();
801                                                if (targetResourceType != null) {
802                                                        sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids) AND r.myTargetResourceType = :target_resource_type";
803                                                } else if (haveTargetTypesDefinedByParam) {
804                                                        sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids) AND r.myTargetResourceType in (:target_resource_types)";
805                                                } else {
806                                                        sql = "SELECT r." + findFieldName + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchFieldName + " IN (:target_pids)";
807                                                }
808
809                                                List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
810                                                for (Collection<ResourcePersistentId> nextPartition : partitions) {
811                                                        TypedQuery<Long> q = theEntityManager.createQuery(sql, Long.class);
812                                                        q.setParameter("src_path", nextPath);
813                                                        q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition));
814                                                        if (targetResourceType != null) {
815                                                                q.setParameter("target_resource_type", targetResourceType);
816                                                        } else if (haveTargetTypesDefinedByParam) {
817                                                                q.setParameter("target_resource_types", param.getTargets());
818                                                        }
819                                                        List<Long> results = q.getResultList();
820                                                        for (Long resourceLink : results) {
821                                                                if (resourceLink != null) {
822                                                                        pidsToInclude.add(new ResourcePersistentId(resourceLink));
823                                                                }
824                                                        }
825                                                }
826                                        }
827                                }
828                        }
829
830                        if (theReverseMode) {
831                                if (theLastUpdated != null && (theLastUpdated.getLowerBoundAsInstant() != null || theLastUpdated.getUpperBoundAsInstant() != null)) {
832                                        pidsToInclude = new HashSet<>(filterResourceIdsByLastUpdated(theEntityManager, theLastUpdated, pidsToInclude));
833                                }
834                        }
835
836                        addedSomeThisRound = allAdded.addAll(pidsToInclude);
837                        nextRoundMatches = pidsToInclude;
838                } while (includes.size() > 0 && nextRoundMatches.size() > 0 && addedSomeThisRound);
839
840                allAdded.removeAll(original);
841
842                ourLog.info("Loaded {} {} in {} rounds and {} ms for search {}", allAdded.size(), theReverseMode ? "_revincludes" : "_includes", roundCounts, w.getMillisAndRestart(), theSearchIdOrDescription);
843
844                // Interceptor call: STORAGE_PREACCESS_RESOURCES
845                // This can be used to remove results from the search result details before
846                // the user has a chance to know that they were in the results
847                if (allAdded.size() > 0) {
848                        List<ResourcePersistentId> includedPidList = new ArrayList<>(allAdded);
849                        JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(includedPidList, () -> this);
850                        HookParams params = new HookParams()
851                                .add(IPreResourceAccessDetails.class, accessDetails)
852                                .add(RequestDetails.class, theRequest)
853                                .addIfMatchesType(ServletRequestDetails.class, theRequest);
854                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
855
856                        allAdded = new HashSet<>(includedPidList);
857
858                        for (int i = includedPidList.size() - 1; i >= 0; i--) {
859                                if (accessDetails.isDontReturnResourceAtIndex(i)) {
860                                        ResourcePersistentId value = includedPidList.remove(i);
861                                        if (value != null) {
862                                                allAdded.remove(value);
863                                        }
864                                }
865                        }
866                }
867
868                return allAdded;
869        }
870
871        private List<Collection<ResourcePersistentId>> partition(Collection<ResourcePersistentId> theNextRoundMatches, int theMaxLoad) {
872                if (theNextRoundMatches.size() <= theMaxLoad) {
873                        return Collections.singletonList(theNextRoundMatches);
874                } else {
875
876                        List<Collection<ResourcePersistentId>> retVal = new ArrayList<>();
877                        Collection<ResourcePersistentId> current = null;
878                        for (ResourcePersistentId next : theNextRoundMatches) {
879                                if (current == null) {
880                                        current = new ArrayList<>(theMaxLoad);
881                                        retVal.add(current);
882                                }
883
884                                current.add(next);
885
886                                if (current.size() >= theMaxLoad) {
887                                        current = null;
888                                }
889                        }
890
891                        return retVal;
892                }
893        }
894
895        private void attemptCompositeUniqueSpProcessing(@Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
896                // Since we're going to remove elements below
897                theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList));
898
899                List<RuntimeSearchParam> activeUniqueSearchParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
900                if (activeUniqueSearchParams.size() > 0) {
901
902                        Validate.isTrue(activeUniqueSearchParams.get(0).getComboSearchParamType()== ComboSearchParamType.UNIQUE, "Non unique combo parameters are not supported with the legacy search builder");
903
904                        StringBuilder sb = new StringBuilder();
905                        sb.append(myResourceName);
906                        sb.append("?");
907
908                        boolean first = true;
909
910                        ArrayList<String> keys = new ArrayList<>(theParams.keySet());
911                        Collections.sort(keys);
912                        for (String nextParamName : keys) {
913                                List<List<IQueryParameterType>> nextValues = theParams.get(nextParamName);
914
915                                nextParamName = UrlUtil.escapeUrlParam(nextParamName);
916                                if (nextValues.get(0).size() != 1) {
917                                        sb = null;
918                                        break;
919                                }
920
921                                // Reference params are only eligible for using a composite index if they
922                                // are qualified
923                                RuntimeSearchParam nextParamDef = mySearchParamRegistry.getActiveSearchParam(myResourceName, nextParamName);
924                                if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
925                                        ReferenceParam param = (ReferenceParam) nextValues.get(0).get(0);
926                                        if (isBlank(param.getResourceType())) {
927                                                sb = null;
928                                                break;
929                                        }
930                                }
931
932                                List<? extends IQueryParameterType> nextAnd = nextValues.remove(0);
933                                IQueryParameterType nextOr = nextAnd.remove(0);
934                                String nextOrValue = nextOr.getValueAsQueryToken(myContext);
935                                nextOrValue = UrlUtil.escapeUrlParam(nextOrValue);
936
937                                if (first) {
938                                        first = false;
939                                } else {
940                                        sb.append('&');
941                                }
942
943                                sb.append(nextParamName).append('=').append(nextOrValue);
944
945                        }
946
947                        if (sb != null) {
948                                String indexString = sb.toString();
949                                ourLog.debug("Checking for unique index for query: {}", indexString);
950
951                                // Interceptor broadcast: JPA_PERFTRACE_INFO
952                                StorageProcessingMessage msg = new StorageProcessingMessage()
953                                        .setMessage("Using unique index for query for search: " + indexString);
954                                HookParams params = new HookParams()
955                                        .add(RequestDetails.class, theRequest)
956                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
957                                        .add(StorageProcessingMessage.class, msg);
958                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INFO, params);
959
960                                addPredicateCompositeStringUnique(theParams, indexString, myRequestPartitionId);
961                        }
962                }
963        }
964
965        private <T> void ensureSubListsAreWritable(List<List<T>> theListOfLists) {
966                for (int i = 0; i < theListOfLists.size(); i++) {
967                        List<T> oldSubList = theListOfLists.get(i);
968                        if (!(oldSubList instanceof ArrayList)) {
969                                List<T> newSubList = new ArrayList<>(oldSubList);
970                                theListOfLists.set(i, newSubList);
971                        }
972                }
973        }
974
975        private void addPredicateCompositeStringUnique(@Nonnull SearchParameterMap theParams, String theIndexedString, RequestPartitionId theRequestPartitionId) {
976                From<?, ResourceIndexedComboStringUnique> join = myQueryStack.createJoin(SearchBuilderJoinEnum.COMPOSITE_UNIQUE, null);
977
978                if (!theRequestPartitionId.isAllPartitions()) {
979                        Integer partitionId = theRequestPartitionId.getFirstPartitionIdOrNull();
980                        Predicate predicate = myCriteriaBuilder.equal(join.get("myPartitionIdValue").as(Integer.class), partitionId);
981                        myQueryStack.addPredicate(predicate);
982                }
983
984                Predicate predicate = myCriteriaBuilder.equal(join.get("myIndexString"), theIndexedString);
985                myQueryStack.addPredicateWithImplicitTypeSelection(predicate);
986
987                // Remove any empty parameters remaining after this
988                theParams.clean();
989        }
990
991        @Override
992        public void setFetchSize(int theFetchSize) {
993                myFetchSize = theFetchSize;
994        }
995
996        @VisibleForTesting
997        void setParamsForUnitTest(SearchParameterMap theParams) {
998                myParams = theParams;
999        }
1000
1001        public SearchParameterMap getParams() {
1002                return myParams;
1003        }
1004
1005        @VisibleForTesting
1006        void setEntityManagerForUnitTest(EntityManager theEntityManager) {
1007                myEntityManager = theEntityManager;
1008        }
1009
1010        public CriteriaBuilder getBuilder() {
1011                return myCriteriaBuilder;
1012        }
1013
1014        public QueryStack getQueryStack() {
1015                return myQueryStack;
1016        }
1017
1018        public Class<? extends IBaseResource> getResourceType() {
1019                return myResourceType;
1020        }
1021
1022        public String getResourceName() {
1023                return myResourceName;
1024        }
1025
1026        @VisibleForTesting
1027        public void setDaoConfigForUnitTest(DaoConfig theDaoConfig) {
1028                myDaoConfig = theDaoConfig;
1029        }
1030
1031        private List<Predicate> createLastUpdatedPredicates(final DateRangeParam theLastUpdated, CriteriaBuilder builder) {
1032                List<Predicate> lastUpdatedPredicates = new ArrayList<>();
1033                if (theLastUpdated != null) {
1034                        if (theLastUpdated.getLowerBoundAsInstant() != null) {
1035                                ourLog.debug("LastUpdated lower bound: {}", new InstantDt(theLastUpdated.getLowerBoundAsInstant()));
1036                                Predicate predicateLower = builder.greaterThanOrEqualTo(myQueryStack.getLastUpdatedColumn(), theLastUpdated.getLowerBoundAsInstant());
1037                                lastUpdatedPredicates.add(predicateLower);
1038                        }
1039                        if (theLastUpdated.getUpperBoundAsInstant() != null) {
1040                                ourLog.debug("LastUpdated upper bound: {}", new InstantDt(theLastUpdated.getUpperBoundAsInstant()));
1041                                Predicate predicateUpper = builder.lessThanOrEqualTo(myQueryStack.getLastUpdatedColumn(), theLastUpdated.getUpperBoundAsInstant());
1042                                lastUpdatedPredicates.add(predicateUpper);
1043                        }
1044                }
1045                return lastUpdatedPredicates;
1046        }
1047
1048        public class IncludesIterator extends BaseIterator<ResourcePersistentId> implements Iterator<ResourcePersistentId> {
1049
1050                private final RequestDetails myRequest;
1051                private Iterator<ResourcePersistentId> myCurrentIterator;
1052                private final Set<ResourcePersistentId> myCurrentPids;
1053                private ResourcePersistentId myNext;
1054
1055                IncludesIterator(Set<ResourcePersistentId> thePidSet, RequestDetails theRequest) {
1056                        myCurrentPids = new HashSet<>(thePidSet);
1057                        myCurrentIterator = EMPTY_LONG_LIST.iterator();
1058                        myRequest = theRequest;
1059                }
1060
1061                private void fetchNext() {
1062                        while (myNext == null) {
1063
1064                                if (myCurrentIterator.hasNext()) {
1065                                        myNext = myCurrentIterator.next();
1066                                        break;
1067                                }
1068
1069                                Set<Include> includes = Collections.singleton(new Include("*", true));
1070                                Set<ResourcePersistentId> newPids = loadIncludes(myContext, myEntityManager, myCurrentPids, includes, false, getParams().getLastUpdated(), mySearchUuid, myRequest, null);
1071                                if (newPids.isEmpty()) {
1072                                        myNext = NO_MORE;
1073                                        break;
1074                                }
1075                                myCurrentPids.addAll(newPids);
1076                                myCurrentIterator = newPids.iterator();
1077                        }
1078                }
1079
1080                @Override
1081                public boolean hasNext() {
1082                        fetchNext();
1083                        return !NO_MORE.equals(myNext);
1084                }
1085
1086                @Override
1087                public ResourcePersistentId next() {
1088                        fetchNext();
1089                        ResourcePersistentId retVal = myNext;
1090                        myNext = null;
1091                        return retVal;
1092                }
1093
1094        }
1095
1096        private final class QueryIterator extends BaseIterator<ResourcePersistentId> implements IResultIterator {
1097
1098                private final SearchRuntimeDetails mySearchRuntimeDetails;
1099                private final RequestDetails myRequest;
1100                private final boolean myHaveRawSqlHooks;
1101                private final boolean myHavePerfTraceFoundIdHook;
1102                private boolean myFirst = true;
1103                private IncludesIterator myIncludesIterator;
1104                private ResourcePersistentId myNext;
1105                private ScrollableResultsIterator<Long> myResultsIterator;
1106                private Integer myOffset;
1107                private final SortSpec mySort;
1108                private boolean myStillNeedToFetchIncludes;
1109                private int mySkipCount = 0;
1110                private int myNonSkipCount = 0;
1111
1112                private List<TypedQuery<Long>> myQueryList = new ArrayList<>();
1113
1114                private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
1115                        mySearchRuntimeDetails = theSearchRuntimeDetails;
1116                        mySort = myParams.getSort();
1117                        myOffset = myParams.getOffset();
1118                        myRequest = theRequest;
1119
1120                        // Includes are processed inline for $everything query
1121                        if (myParams.getEverythingMode() != null) {
1122                                myStillNeedToFetchIncludes = true;
1123                        }
1124
1125                        myHavePerfTraceFoundIdHook = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, myInterceptorBroadcaster, myRequest);
1126                        myHaveRawSqlHooks = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, myRequest);
1127
1128                }
1129
1130                private boolean isPagingProviderDatabaseBacked() {
1131                        if (myRequest == null || myRequest.getServer() == null) {
1132                                return false;
1133                        }
1134                        return myRequest.getServer().getPagingProvider() instanceof DatabaseBackedPagingProvider;
1135                }
1136
1137                private void fetchNext() {
1138
1139                        try {
1140                                if (myHaveRawSqlHooks) {
1141                                        CurrentThreadCaptureQueriesListener.startCapturing();
1142                                }
1143
1144                                // If we don't have a query yet, create one
1145                                if (myResultsIterator == null) {
1146                                        if (myMaxResultsToFetch == null) {
1147                                                if (myParams.getLoadSynchronousUpTo() != null) {
1148                                                        myMaxResultsToFetch = myParams.getLoadSynchronousUpTo();
1149                                                } else if (myParams.getCount() != null) {
1150                                                        myMaxResultsToFetch = myParams.getCount();
1151                                                } else {
1152                                                        myMaxResultsToFetch = myDaoConfig.getFetchSizeDefaultMaximum();
1153                                                }
1154                                        }
1155
1156                                        initializeIteratorQuery(myOffset, myMaxResultsToFetch);
1157
1158                                        if (myAlsoIncludePids == null) {
1159                                                myAlsoIncludePids = new ArrayList<>();
1160                                        }
1161                                }
1162
1163                                if (myNext == null) {
1164
1165                                        for (Iterator<ResourcePersistentId> myPreResultsIterator = myAlsoIncludePids.iterator(); myPreResultsIterator.hasNext();) {
1166                                                        ResourcePersistentId next = myPreResultsIterator.next();
1167                                                        if (next != null)
1168                                                                if (myPidSet.add(next)) {
1169                                                                        myNext = next;
1170                                                                        break;
1171                                                                }
1172                                                }
1173
1174                                        if (myNext == null) {
1175                                                while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) {
1176                                                        // Update iterator with next chunk if necessary.
1177                                                        if (!myResultsIterator.hasNext()) {
1178                                                                retrieveNextIteratorQuery();
1179                                                        }
1180
1181                                                        Long nextLong = myResultsIterator.next();
1182                                                        if (myHavePerfTraceFoundIdHook) {
1183                                                                HookParams params = new HookParams()
1184                                                                        .add(Integer.class, System.identityHashCode(this))
1185                                                                        .add(Object.class, nextLong);
1186                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, params);
1187                                                        }
1188
1189                                                        if (nextLong != null) {
1190                                                                ResourcePersistentId next = new ResourcePersistentId(nextLong);
1191                                                                if (myPidSet.add(next)) {
1192                                                                        myNext = next;
1193                                                                        myNonSkipCount++;
1194                                                                        break;
1195                                                                } else {
1196                                                                        mySkipCount++;
1197                                                                }
1198                                                        }
1199
1200                                                        if (!myResultsIterator.hasNext()) {
1201                                                                if (myMaxResultsToFetch != null && (mySkipCount + myNonSkipCount == myMaxResultsToFetch)) {
1202                                                                        if (mySkipCount > 0 && myNonSkipCount == 0) {
1203                                                                                myMaxResultsToFetch += 1000;
1204
1205                                                                                StorageProcessingMessage message = new StorageProcessingMessage();
1206                                                                                String msg = "Pass completed with no matching results. This indicates an inefficient query! Retrying with new max count of " + myMaxResultsToFetch;
1207                                                                                ourLog.warn(msg);
1208                                                                                message.setMessage(msg);
1209                                                                                HookParams params = new HookParams()
1210                                                                                        .add(RequestDetails.class, myRequest)
1211                                                                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1212                                                                                        .add(StorageProcessingMessage.class, message);
1213                                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_WARNING, params);
1214
1215                                                                                initializeIteratorQuery(null, myMaxResultsToFetch);
1216                                                                        }
1217                                                                }
1218                                                        }
1219                                                }
1220                                        }
1221
1222                                        if (myNext == null) {
1223                                                if (myStillNeedToFetchIncludes) {
1224                                                        myIncludesIterator = new IncludesIterator(myPidSet, myRequest);
1225                                                        myStillNeedToFetchIncludes = false;
1226                                                }
1227                                                if (myIncludesIterator != null) {
1228                                                        while (myIncludesIterator.hasNext()) {
1229                                                                ResourcePersistentId next = myIncludesIterator.next();
1230                                                                if (next != null)
1231                                                                        if (myPidSet.add(next)) {
1232                                                                                myNext = next;
1233                                                                                break;
1234                                                                        }
1235                                                        }
1236                                                        if (myNext == null) {
1237                                                                myNext = NO_MORE;
1238                                                        }
1239                                                } else {
1240                                                        myNext = NO_MORE;
1241                                                }
1242                                        }
1243
1244                                } // if we need to fetch the next result
1245
1246                                mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size());
1247
1248                        } finally {
1249                                if (myHaveRawSqlHooks) {
1250                                        SqlQueryList capturedQueries = CurrentThreadCaptureQueriesListener.getCurrentQueueAndStopCapturing();
1251                                        HookParams params = new HookParams()
1252                                                .add(RequestDetails.class, myRequest)
1253                                                .addIfMatchesType(ServletRequestDetails.class, myRequest)
1254                                                .add(SqlQueryList.class, capturedQueries);
1255                                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_RAW_SQL, params);
1256                                }
1257                        }
1258
1259                        if (myFirst) {
1260                                HookParams params = new HookParams()
1261                                        .add(RequestDetails.class, myRequest)
1262                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1263                                        .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
1264                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FIRST_RESULT_LOADED, params);
1265                                myFirst = false;
1266                        }
1267
1268                        if (NO_MORE.equals(myNext)) {
1269                                HookParams params = new HookParams()
1270                                        .add(RequestDetails.class, myRequest)
1271                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1272                                        .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
1273                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_SELECT_COMPLETE, params);
1274                        }
1275
1276                }
1277
1278                private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToFetch) {
1279                        if (myQueryList.isEmpty()) {
1280                                // Capture times for Lucene/Elasticsearch queries as well
1281                                mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
1282                                myQueryList = createQuery(mySort, theOffset, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails);
1283                        }
1284
1285                        mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
1286
1287                        retrieveNextIteratorQuery();
1288
1289                        mySkipCount = 0;
1290                        myNonSkipCount = 0;
1291                }
1292
1293                private void retrieveNextIteratorQuery() {
1294                        if (myQueryList != null && myQueryList.size() > 0) {
1295                                final TypedQuery<Long> query = myQueryList.remove(0);
1296                                Query<Long> hibernateQuery = (Query<Long>) (query);
1297                                hibernateQuery.setFetchSize(myFetchSize);
1298                                ScrollableResults scroll = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
1299                                myResultsIterator = new ScrollableResultsIterator<>(scroll);
1300                        } else {
1301                                myResultsIterator = null;
1302                        }
1303
1304                }
1305
1306                @Override
1307                public boolean hasNext() {
1308                        if (myNext == null) {
1309                                fetchNext();
1310                        }
1311                        return !NO_MORE.equals(myNext);
1312                }
1313
1314                @Override
1315                public ResourcePersistentId next() {
1316                        fetchNext();
1317                        ResourcePersistentId retVal = myNext;
1318                        myNext = null;
1319                        Validate.isTrue(!NO_MORE.equals(retVal), "No more elements");
1320                        return retVal;
1321                }
1322
1323                @Override
1324                public int getSkippedCount() {
1325                        return mySkipCount;
1326                }
1327
1328                @Override
1329                public int getNonSkippedCount() {
1330                        return myNonSkipCount;
1331                }
1332
1333                @Override
1334                public Collection<ResourcePersistentId> getNextResultBatch(long theBatchSize) {
1335                        Collection<ResourcePersistentId> batch = new ArrayList<>();
1336                        while (this.hasNext() && batch.size() < theBatchSize) {
1337                                batch.add(this.next());
1338                        }
1339                        return batch;
1340                }
1341
1342                @Override
1343                public void close() {
1344                        if (myResultsIterator != null) {
1345                                myResultsIterator.close();
1346                        }
1347                }
1348
1349        }
1350
1351        private static class CountQueryIterator implements Iterator<Long> {
1352                private final TypedQuery<Long> myQuery;
1353                private boolean myCountLoaded;
1354                private Long myCount;
1355
1356                CountQueryIterator(TypedQuery<Long> theQuery) {
1357                        myQuery = theQuery;
1358                }
1359
1360                @Override
1361                public boolean hasNext() {
1362                        boolean retVal = myCount != null;
1363                        if (!retVal) {
1364                                if (myCountLoaded == false) {
1365                                        myCount = myQuery.getSingleResult();
1366                                        retVal = true;
1367                                        myCountLoaded = true;
1368                                }
1369                        }
1370                        return retVal;
1371                }
1372
1373                @Override
1374                public Long next() {
1375                        Validate.isTrue(hasNext());
1376                        Validate.isTrue(myCount != null);
1377                        Long retVal = myCount;
1378                        myCount = null;
1379                        return retVal;
1380                }
1381        }
1382
1383        private static List<Predicate> createLastUpdatedPredicates(final DateRangeParam theLastUpdated, CriteriaBuilder builder, From<?, ResourceTable> from) {
1384                List<Predicate> lastUpdatedPredicates = new ArrayList<>();
1385                if (theLastUpdated != null) {
1386                        if (theLastUpdated.getLowerBoundAsInstant() != null) {
1387                                ourLog.debug("LastUpdated lower bound: {}", new InstantDt(theLastUpdated.getLowerBoundAsInstant()));
1388                                Predicate predicateLower = builder.greaterThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getLowerBoundAsInstant());
1389                                lastUpdatedPredicates.add(predicateLower);
1390                        }
1391                        if (theLastUpdated.getUpperBoundAsInstant() != null) {
1392                                Predicate predicateUpper = builder.lessThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getUpperBoundAsInstant());
1393                                lastUpdatedPredicates.add(predicateUpper);
1394                        }
1395                }
1396                return lastUpdatedPredicates;
1397        }
1398
1399        private static List<ResourcePersistentId> filterResourceIdsByLastUpdated(EntityManager theEntityManager, final DateRangeParam theLastUpdated, Collection<ResourcePersistentId> thePids) {
1400                if (thePids.isEmpty()) {
1401                        return Collections.emptyList();
1402                }
1403                CriteriaBuilder builder = theEntityManager.getCriteriaBuilder();
1404                CriteriaQuery<Long> cq = builder.createQuery(Long.class);
1405                Root<ResourceTable> from = cq.from(ResourceTable.class);
1406                cq.select(from.get("myId").as(Long.class));
1407
1408                List<Predicate> lastUpdatedPredicates = createLastUpdatedPredicates(theLastUpdated, builder, from);
1409                lastUpdatedPredicates.add(from.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(thePids)));
1410
1411                cq.where(LegacySearchBuilder.toPredicateArray(lastUpdatedPredicates));
1412                TypedQuery<Long> query = theEntityManager.createQuery(cq);
1413
1414                return ResourcePersistentId.fromLongList(query.getResultList());
1415        }
1416
1417        public static Predicate[] toPredicateArray(List<Predicate> thePredicates) {
1418                return thePredicates.toArray(new Predicate[0]);
1419        }
1420}