Source code

001package ca.uhn.fhir.jpa.search.builder;
002
003/*
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2021 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 * http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.context.ComboSearchParamType;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.context.FhirVersionEnum;
026import ca.uhn.fhir.context.RuntimeResourceDefinition;
027import ca.uhn.fhir.context.RuntimeSearchParam;
028import ca.uhn.fhir.interceptor.api.HookParams;
029import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
030import ca.uhn.fhir.interceptor.api.Pointcut;
031import ca.uhn.fhir.interceptor.model.RequestPartitionId;
032import ca.uhn.fhir.jpa.api.config.DaoConfig;
033import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
034import ca.uhn.fhir.jpa.api.dao.IDao;
035import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
036import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean;
037import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider;
038import ca.uhn.fhir.jpa.dao.BaseStorageDao;
039import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
040import ca.uhn.fhir.jpa.dao.IResultIterator;
041import ca.uhn.fhir.jpa.dao.ISearchBuilder;
042import ca.uhn.fhir.jpa.dao.data.IResourceLinkDao;
043import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
044import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
045import ca.uhn.fhir.jpa.dao.index.IdHelperService;
046import ca.uhn.fhir.jpa.entity.ResourceSearchView;
047import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
048import ca.uhn.fhir.jpa.model.config.PartitionSettings;
049import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity;
050import ca.uhn.fhir.jpa.model.entity.ModelConfig;
051import ca.uhn.fhir.jpa.model.entity.ResourceTable;
052import ca.uhn.fhir.jpa.model.entity.ResourceTag;
053import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
054import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
055import ca.uhn.fhir.jpa.search.SearchConstants;
056import ca.uhn.fhir.jpa.search.builder.sql.GeneratedSql;
057import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder;
058import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryExecutor;
059import ca.uhn.fhir.jpa.search.builder.sql.SqlObjectFactory;
060import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc;
061import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
062import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
063import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil;
064import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
065import ca.uhn.fhir.jpa.util.BaseIterator;
066import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
067import ca.uhn.fhir.jpa.util.QueryChunker;
068import ca.uhn.fhir.jpa.util.SqlQueryList;
069import ca.uhn.fhir.model.api.IQueryParameterType;
070import ca.uhn.fhir.model.api.IResource;
071import ca.uhn.fhir.model.api.Include;
072import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
073import ca.uhn.fhir.model.primitive.InstantDt;
074import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum;
075import ca.uhn.fhir.rest.api.Constants;
076import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
077import ca.uhn.fhir.rest.api.SearchContainedModeEnum;
078import ca.uhn.fhir.rest.api.SortOrderEnum;
079import ca.uhn.fhir.rest.api.SortSpec;
080import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
081import ca.uhn.fhir.rest.api.server.RequestDetails;
082import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
083import ca.uhn.fhir.rest.param.DateRangeParam;
084import ca.uhn.fhir.rest.param.ReferenceParam;
085import ca.uhn.fhir.rest.param.StringParam;
086import ca.uhn.fhir.rest.param.TokenParam;
087import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
088import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
089import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
090import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
091import ca.uhn.fhir.util.StopWatch;
092import ca.uhn.fhir.util.StringUtil;
093import ca.uhn.fhir.util.UrlUtil;
094import com.google.common.annotations.VisibleForTesting;
095import com.google.common.collect.Lists;
096import com.healthmarketscience.sqlbuilder.Condition;
097import org.apache.commons.lang3.Validate;
098import org.hl7.fhir.instance.model.api.IAnyResource;
099import org.hl7.fhir.instance.model.api.IBaseResource;
100import org.slf4j.Logger;
101import org.slf4j.LoggerFactory;
102import org.springframework.beans.factory.annotation.Autowired;
103import org.springframework.jdbc.core.JdbcTemplate;
104import org.springframework.jdbc.core.SingleColumnRowMapper;
105import org.springframework.transaction.support.TransactionSynchronizationManager;
106
107import javax.annotation.Nonnull;
108import javax.persistence.EntityManager;
109import javax.persistence.PersistenceContext;
110import javax.persistence.PersistenceContextType;
111import javax.persistence.TypedQuery;
112import javax.persistence.criteria.CriteriaBuilder;
113import javax.persistence.criteria.CriteriaQuery;
114import javax.persistence.criteria.From;
115import javax.persistence.criteria.Predicate;
116import javax.persistence.criteria.Root;
117import java.util.ArrayList;
118import java.util.Collection;
119import java.util.Collections;
120import java.util.HashMap;
121import java.util.HashSet;
122import java.util.Iterator;
123import java.util.List;
124import java.util.Map;
125import java.util.Optional;
126import java.util.Set;
127import java.util.stream.Collectors;
128
129import static org.apache.commons.lang3.StringUtils.defaultString;
130import static org.apache.commons.lang3.StringUtils.isBlank;
131import static org.apache.commons.lang3.StringUtils.isNotBlank;
132
133/**
134 * The SearchBuilder is responsible for actually forming the SQL query that handles
135 * searches for resources
136 */
137public class SearchBuilder implements ISearchBuilder {
138
139        /**
140         * See loadResourcesByPid
141         * for an explanation of why we use the constant 800
142         */
143        // NB: keep public
144        @Deprecated
145        public static final int MAXIMUM_PAGE_SIZE = SearchConstants.MAX_PAGE_SIZE;
146        public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50;
147        private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class);
148        private static final ResourcePersistentId NO_MORE = new ResourcePersistentId(-1L);
149        public static boolean myUseMaxPageSize50ForTest = false;
150        private final String myResourceName;
151        private final Class<? extends IBaseResource> myResourceType;
152        private final IDao myCallingDao;
153        @Autowired
154        protected IInterceptorBroadcaster myInterceptorBroadcaster;
155        @Autowired
156        protected IResourceTagDao myResourceTagDao;
157        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
158        protected EntityManager myEntityManager;
159        @Autowired
160        private DaoConfig myDaoConfig;
161        @Autowired
162        private DaoRegistry myDaoRegistry;
163        @Autowired
164        private IResourceSearchViewDao myResourceSearchViewDao;
165        @Autowired
166        private FhirContext myContext;
167        @Autowired
168        private IdHelperService myIdHelperService;
169        @Autowired(required = false)
170        private IFulltextSearchSvc myFulltextSearchSvc;
171        @Autowired(required = false)
172        private IElasticsearchSvc myIElasticsearchSvc;
173        @Autowired
174        private ISearchParamRegistry mySearchParamRegistry;
175        private List<ResourcePersistentId> myAlsoIncludePids;
176        private CriteriaBuilder myCriteriaBuilder;
177        private SearchParameterMap myParams;
178        private String mySearchUuid;
179        private int myFetchSize;
180        private Integer myMaxResultsToFetch;
181        private Set<ResourcePersistentId> myPidSet;
182        private RequestPartitionId myRequestPartitionId;
183        @Autowired
184        private PartitionSettings myPartitionSettings;
185        @Autowired
186        private HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory;
187        @Autowired
188        private SqlObjectFactory mySqlBuilderFactory;
189        @Autowired
190        private HibernatePropertiesProvider myDialectProvider;
191        @Autowired
192        private ModelConfig myModelConfig;
193
194        private boolean hasNextIteratorQuery = false;
195
196        /**
197         * Constructor
198         */
199        public SearchBuilder(IDao theDao, String theResourceName, Class<? extends IBaseResource> theResourceType) {
200                myCallingDao = theDao;
201                myResourceName = theResourceName;
202                myResourceType = theResourceType;
203        }
204
205        @Override
206        public void setMaxResultsToFetch(Integer theMaxResultsToFetch) {
207                myMaxResultsToFetch = theMaxResultsToFetch;
208        }
209
210        private void searchForIdsWithAndOr(SearchQueryBuilder theSearchSqlBuilder, QueryStack theQueryStack, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
211                myParams = theParams;
212
213                // Remove any empty parameters
214                theParams.clean();
215
216                // For DSTU3, pull out near-distance first so when it comes time to evaluate near, we already know the distance
217                if (myContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) {
218                        Dstu3DistanceHelper.setNearDistance(myResourceType, theParams);
219                }
220
221                // Attempt to lookup via composite unique key.
222                if (isCompositeUniqueSpCandidate()) {
223                        attemptComboUniqueSpProcessing(theQueryStack, theParams, theRequest);
224                }
225
226                SearchContainedModeEnum searchContainedMode = theParams.getSearchContainedMode();
227
228                // Handle _id and _tag last, since they can typically be tacked onto a different parameter
229                List<String> paramNames = myParams.keySet().stream().filter(t -> !t.equals(IAnyResource.SP_RES_ID))
230                        .filter(t -> !t.equals(Constants.PARAM_TAG)).collect(Collectors.toList());
231                if (myParams.containsKey(IAnyResource.SP_RES_ID)) {
232                        paramNames.add(IAnyResource.SP_RES_ID);
233                }
234                if (myParams.containsKey(Constants.PARAM_TAG)) {
235                        paramNames.add(Constants.PARAM_TAG);
236                }
237
238                // Handle each parameter
239                for (String nextParamName : paramNames) {
240                        if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) {
241                                // Skip parameters for Subject, Patient, Code and Category for LastN as these will be filtered by Elasticsearch
242                                continue;
243                        }
244                        List<List<IQueryParameterType>> andOrParams = myParams.get(nextParamName);
245                        Condition predicate = theQueryStack.searchForIdsWithAndOr(null, myResourceName, nextParamName, andOrParams, theRequest, myRequestPartitionId, searchContainedMode);
246                        if (predicate != null) {
247                                theSearchSqlBuilder.addPredicate(predicate);
248                        }
249                }
250        }
251
252        /**
253         * A search is a candidate for Composite Unique SP if unique indexes are enabled, there is no EverythingMode, and the
254         * parameters all have no modifiers.
255         */
256        private boolean isCompositeUniqueSpCandidate() {
257                return myDaoConfig.isUniqueIndexesEnabled() &&
258                        myParams.getEverythingMode() == null &&
259                        myParams.isAllParametersHaveNoModifier();
260        }
261
262        @SuppressWarnings("ConstantConditions")
263        @Override
264        public Iterator<Long> createCountQuery(SearchParameterMap theParams, String theSearchUuid, RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) {
265                assert theRequestPartitionId != null;
266                assert TransactionSynchronizationManager.isActualTransactionActive();
267
268                init(theParams, theSearchUuid, theRequestPartitionId);
269
270                ArrayList<SearchQueryExecutor> queries = createQuery(myParams, null, null, null, true, theRequest, null);
271                if (queries.isEmpty()) {
272                        return Collections.emptyIterator();
273                }
274                try (SearchQueryExecutor queryExecutor = queries.get(0)) {
275                        return Lists.newArrayList(queryExecutor.next()).iterator();
276                }
277        }
278
279        /**
280         * @param thePidSet May be null
281         */
282        @Override
283        public void setPreviouslyAddedResourcePids(@Nonnull List<ResourcePersistentId> thePidSet) {
284                myPidSet = new HashSet<>(thePidSet);
285        }
286
287        @SuppressWarnings("ConstantConditions")
288        @Override
289        public IResultIterator createQuery(SearchParameterMap theParams, SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest, @Nonnull RequestPartitionId theRequestPartitionId) {
290                assert theRequestPartitionId != null;
291                assert TransactionSynchronizationManager.isActualTransactionActive();
292
293                init(theParams, theSearchRuntimeDetails.getSearchUuid(), theRequestPartitionId);
294
295                if (myPidSet == null) {
296                        myPidSet = new HashSet<>();
297                }
298
299                return new QueryIterator(theSearchRuntimeDetails, theRequest);
300        }
301
302        private void init(SearchParameterMap theParams, String theSearchUuid, RequestPartitionId theRequestPartitionId) {
303                myCriteriaBuilder = myEntityManager.getCriteriaBuilder();
304                myParams = theParams;
305                mySearchUuid = theSearchUuid;
306                myRequestPartitionId = theRequestPartitionId;
307        }
308
309        private ArrayList<SearchQueryExecutor> createQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCount, RequestDetails theRequest,
310                                                                                                                                                SearchRuntimeDetails theSearchRuntimeDetails) {
311
312                List<ResourcePersistentId> pids = new ArrayList<>();
313
314                /*
315                 * Fulltext or lastn search
316                 */
317                if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT) || myParams.isLastN()) {
318                        if (myParams.containsKey(Constants.PARAM_CONTENT) || myParams.containsKey(Constants.PARAM_TEXT)) {
319                                if (myFulltextSearchSvc == null || myFulltextSearchSvc.isDisabled()) {
320                                        if (myParams.containsKey(Constants.PARAM_TEXT)) {
321                                                throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_TEXT);
322                                        } else if (myParams.containsKey(Constants.PARAM_CONTENT)) {
323                                                throw new InvalidRequestException("Fulltext search is not enabled on this service, can not process parameter: " + Constants.PARAM_CONTENT);
324                                        }
325                                }
326
327                                if (myParams.getEverythingMode() != null) {
328                                        pids = myFulltextSearchSvc.everything(myResourceName, myParams, theRequest);
329                                } else {
330                                        pids = myFulltextSearchSvc.search(myResourceName, myParams);
331                                }
332                        } else if (myParams.isLastN()) {
333                                if (myIElasticsearchSvc == null) {
334                                        if (myParams.isLastN()) {
335                                                throw new InvalidRequestException("LastN operation is not enabled on this service, can not process this request");
336                                        }
337                                }
338                                List<String> lastnResourceIds = myIElasticsearchSvc.executeLastN(myParams, myContext, theMaximumResults);
339                                for (String lastnResourceId : lastnResourceIds) {
340                                        pids.add(myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, lastnResourceId));
341                                }
342                        }
343                        if (theSearchRuntimeDetails != null) {
344                                theSearchRuntimeDetails.setFoundIndexMatchesCount(pids.size());
345                                HookParams params = new HookParams()
346                                        .add(RequestDetails.class, theRequest)
347                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
348                                        .add(SearchRuntimeDetails.class, theSearchRuntimeDetails);
349                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE, params);
350                        }
351
352                        if (pids.isEmpty()) {
353                                // Will never match
354                                pids = Collections.singletonList(new ResourcePersistentId(-1L));
355                        }
356
357                }
358
359                ArrayList<SearchQueryExecutor> queries = new ArrayList<>();
360
361                if (!pids.isEmpty()) {
362                        new QueryChunker<Long>().chunk(ResourcePersistentId.toLongList(pids), t -> doCreateChunkedQueries(theParams, t, theOffset, sort, theCount, theRequest, queries));
363                } else {
364                        Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, theMaximumResults, theCount, theRequest, null);
365                        query.ifPresent(t -> queries.add(t));
366                }
367
368                return queries;
369        }
370
371        private void doCreateChunkedQueries(SearchParameterMap theParams, List<Long> thePids, Integer theOffset, SortSpec sort, boolean theCount, RequestDetails theRequest, ArrayList<SearchQueryExecutor> theQueries) {
372                if (thePids.size() < getMaximumPageSize()) {
373                        normalizeIdListForLastNInClause(thePids);
374                }
375                Optional<SearchQueryExecutor> query = createChunkedQuery(theParams, sort, theOffset, thePids.size(), theCount, theRequest, thePids);
376                query.ifPresent(t -> theQueries.add(t));
377        }
378
379        /**
380         * Combs through the params for any _id parameters and extracts the PIDs for them
381         * @param theTargetPids
382         */
383        private void extractTargetPidsFromIdParams(HashSet<Long> theTargetPids) {
384                // get all the IQueryParameterType objects
385                // for _id -> these should all be StringParam values
386                HashSet<String> ids = new HashSet<>();
387                List<List<IQueryParameterType>> params = myParams.get(IAnyResource.SP_RES_ID);
388                for (List<IQueryParameterType> paramList : params) {
389                        for (IQueryParameterType param : paramList) {
390                                if (param instanceof StringParam) {
391                                        // we expect all _id values to be StringParams
392                                        ids.add(((StringParam) param).getValue());
393                                } else if (param instanceof TokenParam) {
394                                        ids.add(((TokenParam)param).getValue());
395                                } else {
396                                        // we do not expect the _id parameter to be a non-string value
397                                        throw new IllegalArgumentException("_id parameter must be a StringParam or TokenParam");
398                                }
399                        }
400                }
401
402                // fetch our target Pids
403                // this will throw if an id is not found
404                Map<String, ResourcePersistentId> idToPid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId,
405                        myResourceName,
406                        new ArrayList<>(ids));
407                if (myAlsoIncludePids == null) {
408                        myAlsoIncludePids = new ArrayList<>();
409                }
410
411                // add the pids to targetPids
412                for (ResourcePersistentId pid : idToPid.values()) {
413                        myAlsoIncludePids.add(pid);
414                        theTargetPids.add(pid.getIdAsLong());
415                }
416        }
417
418        private Optional<SearchQueryExecutor> createChunkedQuery(SearchParameterMap theParams, SortSpec sort, Integer theOffset, Integer theMaximumResults, boolean theCount, RequestDetails theRequest, List<Long> thePidList) {
419                String sqlBuilderResourceName = myParams.getEverythingMode() == null ? myResourceName : null;
420                SearchQueryBuilder sqlBuilder = new SearchQueryBuilder(myContext, myDaoConfig.getModelConfig(), myPartitionSettings, myRequestPartitionId, sqlBuilderResourceName, mySqlBuilderFactory, myDialectProvider, theCount);
421                QueryStack queryStack3 = new QueryStack(theParams, myDaoConfig, myDaoConfig.getModelConfig(), myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings);
422
423                if (theParams.keySet().size() > 1 || theParams.getSort() != null || theParams.keySet().contains(Constants.PARAM_HAS) || isPotentiallyContainedReferenceParameterExistsAtRoot(theParams)) {
424                        List<RuntimeSearchParam> activeComboParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
425                        if (activeComboParams.isEmpty()) {
426                                sqlBuilder.setNeedResourceTableRoot(true);
427                        }
428                }
429
430                JdbcTemplate jdbcTemplate = new JdbcTemplate(myEntityManagerFactory.getDataSource());
431                jdbcTemplate.setFetchSize(myFetchSize);
432                if (theMaximumResults != null) {
433                        jdbcTemplate.setMaxRows(theMaximumResults);
434                }
435
436                if (myParams.getEverythingMode() != null) {
437                        HashSet<Long> targetPids = new HashSet<>();
438                        if (myParams.get(IAnyResource.SP_RES_ID) != null) {
439                                extractTargetPidsFromIdParams(targetPids);
440                        } else {
441                                // For Everything queries, we make the query root by the ResourceLink table, since this query
442                                // is basically a reverse-include search. For type/Everything (as opposed to instance/Everything)
443                                // the one problem with this approach is that it doesn't catch Patients that have absolutely
444                                // nothing linked to them. So we do one additional query to make sure we catch those too.
445                                SearchQueryBuilder fetchPidsSqlBuilder = new SearchQueryBuilder(myContext, myDaoConfig.getModelConfig(), myPartitionSettings, myRequestPartitionId, myResourceName, mySqlBuilderFactory, myDialectProvider, theCount);
446                                GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theOffset, myMaxResultsToFetch);
447                                String sql = allTargetsSql.getSql();
448                                Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]);
449                                List<Long> output = jdbcTemplate.query(sql, args, new SingleColumnRowMapper<>(Long.class));
450                                if (myAlsoIncludePids == null) {
451                                        myAlsoIncludePids = new ArrayList<>(output.size());
452                                }
453                                myAlsoIncludePids.addAll(ResourcePersistentId.fromLongList(output));
454
455                        }
456
457                        queryStack3.addPredicateEverythingOperation(myResourceName, targetPids.toArray(new Long[0]));
458                } else {
459                        /*
460                         * If we're doing a filter, always use the resource table as the root - This avoids the possibility of
461                         * specific filters with ORs as their root from working around the natural resource type / deletion
462                         * status / partition IDs built into queries.
463                         */
464                        if (theParams.containsKey(Constants.PARAM_FILTER)) {
465                                Condition partitionIdPredicate = sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId);
466                                if (partitionIdPredicate != null) {
467                                        sqlBuilder.addPredicate(partitionIdPredicate);
468                                }
469                        }
470
471                        // Normal search
472                        searchForIdsWithAndOr(sqlBuilder, queryStack3, myParams, theRequest);
473                }
474
475                // If we haven't added any predicates yet, we're doing a search for all resources. Make sure we add the
476                // partition ID predicate in that case.
477                if (!sqlBuilder.haveAtLeastOnePredicate()) {
478                        Condition partitionIdPredicate = sqlBuilder.getOrCreateResourceTablePredicateBuilder().createPartitionIdPredicate(myRequestPartitionId);
479                        if (partitionIdPredicate != null) {
480                                sqlBuilder.addPredicate(partitionIdPredicate);
481                        }
482                }
483
484                // Add PID list predicate for full text search and/or lastn operation
485                if (thePidList != null && thePidList.size() > 0) {
486                        sqlBuilder.addResourceIdsPredicate(thePidList);
487                }
488
489                // Last updated
490                DateRangeParam lu = myParams.getLastUpdated();
491                if (lu != null && !lu.isEmpty()) {
492                        Condition lastUpdatedPredicates = sqlBuilder.addPredicateLastUpdated(lu);
493                        sqlBuilder.addPredicate(lastUpdatedPredicates);
494                }
495
496                //-- exclude the pids already in the previous iterator
497                if (hasNextIteratorQuery) {
498                        sqlBuilder.excludeResourceIdsPredicate(myPidSet);
499                }
500
501                /*
502                 * Sort
503                 *
504                 * If we have a sort, we wrap the criteria search (the search that actually
505                 * finds the appropriate resources) in an outer search which is then sorted
506                 */
507                if (sort != null) {
508                        assert !theCount;
509
510                        createSort(queryStack3, sort);
511                }
512
513                /*
514                 * Now perform the search
515                 */
516                GeneratedSql generatedSql = sqlBuilder.generate(theOffset, myMaxResultsToFetch);
517                if (generatedSql.isMatchNothing()) {
518                        return Optional.empty();
519                }
520
521                SearchQueryExecutor executor = mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, myMaxResultsToFetch);
522                return Optional.of(executor);
523        }
524
525        private boolean isPotentiallyContainedReferenceParameterExistsAtRoot(SearchParameterMap theParams) {
526                return myModelConfig.isIndexOnContainedResources() && theParams.values().stream()
527                        .flatMap(Collection::stream)
528                        .flatMap(Collection::stream)
529                        .anyMatch(t -> t instanceof ReferenceParam);
530        }
531
532        private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
533                /*
534                        The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying
535                        numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info:
536                        https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage.
537
538                        Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of
539                        arguments never exceeds the maximum specified below.
540                 */
541                int listSize = lastnResourceIds.size();
542
543                if (listSize > 1 && listSize < 10) {
544                        padIdListWithPlaceholders(lastnResourceIds, 10);
545                } else if (listSize > 10 && listSize < 50) {
546                        padIdListWithPlaceholders(lastnResourceIds, 50);
547                } else if (listSize > 50 && listSize < 100) {
548                        padIdListWithPlaceholders(lastnResourceIds, 100);
549                } else if (listSize > 100 && listSize < 200) {
550                        padIdListWithPlaceholders(lastnResourceIds, 200);
551                } else if (listSize > 200 && listSize < 500) {
552                        padIdListWithPlaceholders(lastnResourceIds, 500);
553                } else if (listSize > 500 && listSize < 800) {
554                        padIdListWithPlaceholders(lastnResourceIds, 800);
555                }
556
557                return lastnResourceIds;
558        }
559
560        private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
561                while (theIdList.size() < preferredListSize) {
562                        theIdList.add(-1L);
563                }
564        }
565
566        private void createSort(QueryStack theQueryStack, SortSpec theSort) {
567                if (theSort == null || isBlank(theSort.getParamName())) {
568                        return;
569                }
570
571                boolean ascending = (theSort.getOrder() == null) || (theSort.getOrder() == SortOrderEnum.ASC);
572
573                if (IAnyResource.SP_RES_ID.equals(theSort.getParamName())) {
574
575                        theQueryStack.addSortOnResourceId(ascending);
576
577                } else if (Constants.PARAM_LASTUPDATED.equals(theSort.getParamName())) {
578
579                        theQueryStack.addSortOnLastUpdated(ascending);
580
581                } else {
582
583                        RuntimeSearchParam param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName());
584                        if (param == null) {
585                                String msg = myContext.getLocalizer().getMessageSanitized(BaseStorageDao.class, "invalidSortParameter", theSort.getParamName(), getResourceName(), mySearchParamRegistry.getValidSearchParameterNamesIncludingMeta(getResourceName()));
586                                throw new InvalidRequestException(msg);
587                        }
588
589                        switch (param.getParamType()) {
590                                case STRING:
591                                        theQueryStack.addSortOnString(myResourceName, theSort.getParamName(), ascending);
592                                        break;
593                                case DATE:
594                                        theQueryStack.addSortOnDate(myResourceName, theSort.getParamName(), ascending);
595                                        break;
596                                case REFERENCE:
597                                        theQueryStack.addSortOnResourceLink(myResourceName, theSort.getParamName(), ascending);
598                                        break;
599                                case TOKEN:
600                                        theQueryStack.addSortOnToken(myResourceName, theSort.getParamName(), ascending);
601                                        break;
602                                case NUMBER:
603                                        theQueryStack.addSortOnNumber(myResourceName, theSort.getParamName(), ascending);
604                                        break;
605                                case URI:
606                                        theQueryStack.addSortOnUri(myResourceName, theSort.getParamName(), ascending);
607                                        break;
608                                case QUANTITY:
609                                        theQueryStack.addSortOnQuantity(myResourceName, theSort.getParamName(), ascending);
610                                        break;
611                                case COMPOSITE:
612                                        List<RuntimeSearchParam> compositeList = JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, param);
613                                        if (compositeList == null) {
614                                                throw new InvalidRequestException("The composite _sort parameter " + theSort.getParamName() + " is not defined by the resource " + myResourceName);
615                                        }
616                                        if (compositeList.size() != 2) {
617                                                throw new InvalidRequestException("The composite _sort parameter " + theSort.getParamName()
618                                                        + " must have 2 composite types declared in parameter annotation, found "
619                                                        + compositeList.size());
620                                        }
621                                        RuntimeSearchParam left = compositeList.get(0);
622                                        RuntimeSearchParam right = compositeList.get(1);
623
624                                        createCompositeSort(theQueryStack, myResourceName, left.getParamType(), left.getName(), ascending);
625                                        createCompositeSort(theQueryStack, myResourceName, right.getParamType(), right.getName(), ascending);
626
627                                        break;
628                                case SPECIAL:
629                                case HAS:
630                                default:
631                                        throw new InvalidRequestException("This server does not support _sort specifications of type " + param.getParamType() + " - Can't serve _sort=" + theSort.getParamName());
632                        }
633
634                }
635
636                // Recurse
637                createSort(theQueryStack, theSort.getChain());
638
639        }
640
641        private void createCompositeSort(QueryStack theQueryStack, String theResourceName, RestSearchParameterTypeEnum theParamType, String theParamName, boolean theAscending) {
642
643                switch (theParamType) {
644                        case STRING:
645                                theQueryStack.addSortOnString(myResourceName, theParamName, theAscending);
646                                break;
647                        case DATE:
648                                theQueryStack.addSortOnDate(myResourceName, theParamName, theAscending);
649                                break;
650                        case TOKEN:
651                                theQueryStack.addSortOnToken(myResourceName, theParamName, theAscending);
652                                break;
653                        case QUANTITY:
654                                theQueryStack.addSortOnQuantity(myResourceName, theParamName, theAscending);
655                                break;
656                        case NUMBER:
657                        case REFERENCE:
658                        case COMPOSITE:
659                        case URI:
660                        case HAS:
661                        case SPECIAL:
662                        default:
663                                throw new InvalidRequestException("Don't know how to handle composite parameter with type of " + theParamType + " on _sort=" + theParamName);
664                }
665
666        }
667
668        private void doLoadPids(Collection<ResourcePersistentId> thePids, Collection<ResourcePersistentId> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation,
669                                                                        Map<ResourcePersistentId, Integer> thePosition) {
670
671                Map<Long, Long> resourcePidToVersion = null;
672                for (ResourcePersistentId next : thePids) {
673                        if (next.getVersion() != null && myModelConfig.isRespectVersionsForSearchIncludes()) {
674                                if (resourcePidToVersion == null) {
675                                        resourcePidToVersion = new HashMap<>();
676                                }
677                                resourcePidToVersion.put(next.getIdAsLong(), next.getVersion());
678                        }
679                }
680
681                List<Long> versionlessPids = ResourcePersistentId.toLongList(thePids);
682                if (versionlessPids.size() < getMaximumPageSize()) {
683                        versionlessPids = normalizeIdListForLastNInClause(versionlessPids);
684                }
685
686                // -- get the resource from the searchView
687                Collection<ResourceSearchView> resourceSearchViewList = myResourceSearchViewDao.findByResourceIds(versionlessPids);
688
689                //-- preload all tags with tag definition if any
690                Map<Long, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList);
691
692                for (IBaseResourceEntity next : resourceSearchViewList) {
693                        if (next.getDeleted() != null) {
694                                continue;
695                        }
696
697                        Class<? extends IBaseResource> resourceType = myContext.getResourceDefinition(next.getResourceType()).getImplementingClass();
698
699                        ResourcePersistentId resourceId = new ResourcePersistentId(next.getResourceId());
700
701                        /*
702                         * If a specific version is requested via an include, we'll replace the current version
703                         * with the specific desired version. This is not the most efficient thing, given that
704                         * we're loading the current version and then turning around and throwing it away again.
705                         * This could be optimized and probably should be, but it's not critical given that
706                         * this only applies to includes, which don't tend to be massive in numbers.
707                         */
708                        if (resourcePidToVersion != null) {
709                                Long version = resourcePidToVersion.get(next.getResourceId());
710                                resourceId.setVersion(version);
711                                if (version != null && !version.equals(next.getVersion())) {
712                                        IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(resourceType);
713                                        next = dao.readEntity(next.getIdDt().withVersion(Long.toString(version)), null);
714                                }
715                        }
716
717                        IBaseResource resource = null;
718                        if (next != null) {
719                                resource = myCallingDao.toResource(resourceType, next, tagMap.get(next.getId()), theForHistoryOperation);
720                        }
721                        if (resource == null) {
722                                ourLog.warn("Unable to find resource {}/{}/_history/{} in database", next.getResourceType(), next.getIdDt().getIdPart(), next.getVersion());
723                                continue;
724                        }
725
726                        Integer index = thePosition.get(resourceId);
727                        if (index == null) {
728                                ourLog.warn("Got back unexpected resource PID {}", resourceId);
729                                continue;
730                        }
731
732                        if (resource instanceof IResource) {
733                                if (theIncludedPids.contains(resourceId)) {
734                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IResource) resource, BundleEntrySearchModeEnum.INCLUDE);
735                                } else {
736                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IResource) resource, BundleEntrySearchModeEnum.MATCH);
737                                }
738                        } else {
739                                if (theIncludedPids.contains(resourceId)) {
740                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IAnyResource) resource, BundleEntrySearchModeEnum.INCLUDE.getCode());
741                                } else {
742                                        ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put((IAnyResource) resource, BundleEntrySearchModeEnum.MATCH.getCode());
743                                }
744                        }
745
746                        theResourceListToPopulate.set(index, resource);
747                }
748        }
749
750        private Map<Long, Collection<ResourceTag>> getResourceTagMap(Collection<? extends IBaseResourceEntity> theResourceSearchViewList) {
751
752                List<Long> idList = new ArrayList<>(theResourceSearchViewList.size());
753
754                //-- find all resource has tags
755                for (IBaseResourceEntity resource : theResourceSearchViewList) {
756                        if (resource.isHasTags())
757                                idList.add(resource.getId());
758                }
759
760                Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>();
761
762                //-- no tags
763                if (idList.size() == 0)
764                        return tagMap;
765
766                //-- get all tags for the idList
767                Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(idList);
768
769                //-- build the map, key = resourceId, value = list of ResourceTag
770                ResourcePersistentId resourceId;
771                Collection<ResourceTag> tagCol;
772                for (ResourceTag tag : tagList) {
773
774                        resourceId = new ResourcePersistentId(tag.getResourceId());
775                        tagCol = tagMap.get(resourceId.getIdAsLong());
776                        if (tagCol == null) {
777                                tagCol = new ArrayList<>();
778                                tagCol.add(tag);
779                                tagMap.put(resourceId.getIdAsLong(), tagCol);
780                        } else {
781                                tagCol.add(tag);
782                        }
783                }
784
785                return tagMap;
786        }
787
788        @Override
789        public void loadResourcesByPid(Collection<ResourcePersistentId> thePids, Collection<ResourcePersistentId> theIncludedPids, List<IBaseResource> theResourceListToPopulate, boolean theForHistoryOperation, RequestDetails theDetails) {
790                if (thePids.isEmpty()) {
791                        ourLog.debug("The include pids are empty");
792                        // return;
793                }
794
795                // Dupes will cause a crash later anyhow, but this is expensive so only do it
796                // when running asserts
797                assert new HashSet<>(thePids).size() == thePids.size() : "PID list contains duplicates: " + thePids;
798
799                Map<ResourcePersistentId, Integer> position = new HashMap<>();
800                for (ResourcePersistentId next : thePids) {
801                        position.put(next, theResourceListToPopulate.size());
802                        theResourceListToPopulate.add(null);
803                }
804
805                List<ResourcePersistentId> pids = new ArrayList<>(thePids);
806                new QueryChunker<ResourcePersistentId>().chunk(pids, t -> doLoadPids(t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
807
808        }
809
810        /**
811         * THIS SHOULD RETURN HASHSET and not just Set because we add to it later
812         * so it can't be Collections.emptySet() or some such thing
813         */
814        @Override
815        public Set<ResourcePersistentId> loadIncludes(FhirContext theContext, EntityManager theEntityManager, Collection<ResourcePersistentId> theMatches, Set<Include> theIncludes,
816                                                                                                                                 boolean theReverseMode, DateRangeParam theLastUpdated, String theSearchIdOrDescription, RequestDetails theRequest, Integer theMaxCount) {
817                if (theMatches.size() == 0) {
818                        return new HashSet<>();
819                }
820                if (theIncludes == null || theIncludes.isEmpty()) {
821                        return new HashSet<>();
822                }
823                String searchPidFieldName = theReverseMode ? "myTargetResourcePid" : "mySourceResourcePid";
824                String findPidFieldName = theReverseMode ? "mySourceResourcePid" : "myTargetResourcePid";
825                String findVersionFieldName = null;
826                if (!theReverseMode && myModelConfig.isRespectVersionsForSearchIncludes()) {
827                        findVersionFieldName = "myTargetResourceVersion";
828                }
829
830                List<ResourcePersistentId> nextRoundMatches = new ArrayList<>(theMatches);
831                HashSet<ResourcePersistentId> allAdded = new HashSet<>();
832                HashSet<ResourcePersistentId> original = new HashSet<>(theMatches);
833                ArrayList<Include> includes = new ArrayList<>(theIncludes);
834
835                int roundCounts = 0;
836                StopWatch w = new StopWatch();
837
838                boolean addedSomeThisRound;
839                do {
840                        roundCounts++;
841
842                        HashSet<ResourcePersistentId> pidsToInclude = new HashSet<>();
843
844                        for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) {
845                                Include nextInclude = iter.next();
846                                if (nextInclude.isRecurse() == false) {
847                                        iter.remove();
848                                }
849
850                                // Account for _include=*
851                                boolean matchAll = "*".equals(nextInclude.getValue());
852
853                                // Account for _include=[resourceType]:*
854                                String wantResourceType = null;
855                                if (!matchAll) {
856                                        if ("*".equals(nextInclude.getParamName())) {
857                                                wantResourceType = nextInclude.getParamType();
858                                                matchAll = true;
859                                        }
860                                }
861
862                                if (matchAll) {
863                                        StringBuilder sqlBuilder = new StringBuilder();
864                                        sqlBuilder.append("SELECT r.").append(findPidFieldName);
865                                        if (findVersionFieldName != null) {
866                                                sqlBuilder.append(", r." + findVersionFieldName);
867                                        }
868                                        sqlBuilder.append(" FROM ResourceLink r WHERE ");
869
870                                        sqlBuilder.append("r.");
871                                        sqlBuilder.append(searchPidFieldName);
872                                        sqlBuilder.append(" IN (:target_pids)");
873
874                                        // Technically if the request is a qualified star (e.g. _include=Observation:*) we
875                                        // should always be checking the source resource type on the resource link. We don't
876                                        // actually index that column though by default, so in order to try and be efficient
877                                        // we don't actually include it for includes (but we do for revincludes). This is
878                                        // because for an include it doesn't really make sense to include a different
879                                        // resource type than the one you are searching on.
880                                        if (wantResourceType != null && theReverseMode) {
881                                                sqlBuilder.append(" AND r.mySourceResourceType = :want_resource_type");
882                                        } else {
883                                                wantResourceType = null;
884                                        }
885
886                                        String sql = sqlBuilder.toString();
887                                        List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
888                                        for (Collection<ResourcePersistentId> nextPartition : partitions) {
889                                                TypedQuery<?> q = theEntityManager.createQuery(sql, Object[].class);
890                                                q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition));
891                                                if (wantResourceType != null) {
892                                                        q.setParameter("want_resource_type", wantResourceType);
893                                                }
894                                                if (theMaxCount != null) {
895                                                        q.setMaxResults(theMaxCount);
896                                                }
897                                                List<?> results = q.getResultList();
898                                                for (Object nextRow : results) {
899                                                        if (nextRow == null) {
900                                                                // This can happen if there are outgoing references which are canonical or point to
901                                                                // other servers
902                                                                continue;
903                                                        }
904
905                                                        Long resourceLink;
906                                                        Long version = null;
907                                                        if (findVersionFieldName != null) {
908                                                                resourceLink = (Long) ((Object[]) nextRow)[0];
909                                                                version = (Long) ((Object[]) nextRow)[1];
910                                                        } else {
911                                                                resourceLink = (Long) nextRow;
912                                                        }
913
914                                                        pidsToInclude.add(new ResourcePersistentId(resourceLink, version));
915                                                }
916                                        }
917                                } else {
918
919                                        List<String> paths;
920                                        RuntimeSearchParam param;
921                                        String resType = nextInclude.getParamType();
922                                        if (isBlank(resType)) {
923                                                continue;
924                                        }
925                                        RuntimeResourceDefinition def = theContext.getResourceDefinition(resType);
926                                        if (def == null) {
927                                                ourLog.warn("Unknown resource type in include/revinclude=" + nextInclude.getValue());
928                                                continue;
929                                        }
930
931                                        String paramName = nextInclude.getParamName();
932                                        if (isNotBlank(paramName)) {
933                                                param = mySearchParamRegistry.getActiveSearchParam(resType, paramName);
934                                        } else {
935                                                param = null;
936                                        }
937                                        if (param == null) {
938                                                ourLog.warn("Unknown param name in include/revinclude=" + nextInclude.getValue());
939                                                continue;
940                                        }
941
942                                        paths = param.getPathsSplit();
943
944                                        String targetResourceType = defaultString(nextInclude.getParamTargetType(), null);
945                                        for (String nextPath : paths) {
946                                                String sql;
947
948                                                boolean haveTargetTypesDefinedByParam = param.hasTargets();
949                                                String fieldsToLoad = "r." + findPidFieldName;
950                                                if (findVersionFieldName != null) {
951                                                        fieldsToLoad += ", r." + findVersionFieldName;
952                                                }
953
954                                                if (targetResourceType != null) {
955                                                        sql = "SELECT " + fieldsToLoad + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchPidFieldName + " IN (:target_pids) AND r.myTargetResourceType = :target_resource_type";
956                                                } else if (haveTargetTypesDefinedByParam) {
957                                                        sql = "SELECT " + fieldsToLoad + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchPidFieldName + " IN (:target_pids) AND r.myTargetResourceType in (:target_resource_types)";
958                                                } else {
959                                                        sql = "SELECT " + fieldsToLoad + " FROM ResourceLink r WHERE r.mySourcePath = :src_path AND r." + searchPidFieldName + " IN (:target_pids)";
960                                                }
961
962                                                List<Collection<ResourcePersistentId>> partitions = partition(nextRoundMatches, getMaximumPageSize());
963                                                for (Collection<ResourcePersistentId> nextPartition : partitions) {
964                                                        TypedQuery<?> q = theEntityManager.createQuery(sql, Object[].class);
965                                                        q.setParameter("src_path", nextPath);
966                                                        q.setParameter("target_pids", ResourcePersistentId.toLongList(nextPartition));
967                                                        if (targetResourceType != null) {
968                                                                q.setParameter("target_resource_type", targetResourceType);
969                                                        } else if (haveTargetTypesDefinedByParam) {
970                                                                q.setParameter("target_resource_types", param.getTargets());
971                                                        }
972                                                        List<?> results = q.getResultList();
973                                                        if (theMaxCount != null) {
974                                                                q.setMaxResults(theMaxCount);
975                                                        }
976                                                        for (Object resourceLink : results) {
977                                                                if (resourceLink != null) {
978                                                                        ResourcePersistentId persistentId;
979                                                                        if (findVersionFieldName != null) {
980                                                                                persistentId = new ResourcePersistentId(((Object[]) resourceLink)[0]);
981                                                                                persistentId.setVersion((Long) ((Object[]) resourceLink)[1]);
982                                                                        } else {
983                                                                                persistentId = new ResourcePersistentId(resourceLink);
984                                                                        }
985                                                                        assert persistentId.getId() instanceof Long;
986                                                                        pidsToInclude.add(persistentId);
987                                                                }
988                                                        }
989                                                }
990                                        }
991                                }
992                        }
993
994                        if (theReverseMode) {
995                                if (theLastUpdated != null && (theLastUpdated.getLowerBoundAsInstant() != null || theLastUpdated.getUpperBoundAsInstant() != null)) {
996                                        pidsToInclude = new HashSet<>(filterResourceIdsByLastUpdated(theEntityManager, theLastUpdated, pidsToInclude));
997                                }
998                        }
999
1000                        nextRoundMatches.clear();
1001                        for (ResourcePersistentId next : pidsToInclude) {
1002                                if (original.contains(next) == false && allAdded.contains(next) == false) {
1003                                        nextRoundMatches.add(next);
1004                                }
1005                        }
1006
1007                        addedSomeThisRound = allAdded.addAll(pidsToInclude);
1008
1009                        if (theMaxCount != null && allAdded.size() >= theMaxCount) {
1010                                break;
1011                        }
1012
1013                } while (includes.size() > 0 && nextRoundMatches.size() > 0 && addedSomeThisRound);
1014
1015                allAdded.removeAll(original);
1016
1017                ourLog.info("Loaded {} {} in {} rounds and {} ms for search {}", allAdded.size(), theReverseMode ? "_revincludes" : "_includes", roundCounts, w.getMillisAndRestart(), theSearchIdOrDescription);
1018
1019                // Interceptor call: STORAGE_PREACCESS_RESOURCES
1020                // This can be used to remove results from the search result details before
1021                // the user has a chance to know that they were in the results
1022                if (allAdded.size() > 0) {
1023
1024                        if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_PREACCESS_RESOURCES, myInterceptorBroadcaster, theRequest)) {
1025                                List<ResourcePersistentId> includedPidList = new ArrayList<>(allAdded);
1026                                JpaPreResourceAccessDetails accessDetails = new JpaPreResourceAccessDetails(includedPidList, () -> this);
1027                                HookParams params = new HookParams()
1028                                        .add(IPreResourceAccessDetails.class, accessDetails)
1029                                        .add(RequestDetails.class, theRequest)
1030                                        .addIfMatchesType(ServletRequestDetails.class, theRequest);
1031                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
1032
1033                                for (int i = includedPidList.size() - 1; i >= 0; i--) {
1034                                        if (accessDetails.isDontReturnResourceAtIndex(i)) {
1035                                                ResourcePersistentId value = includedPidList.remove(i);
1036                                                if (value != null) {
1037                                                        allAdded.remove(value);
1038                                                }
1039                                        }
1040                                }
1041                        }
1042                }
1043
1044                return allAdded;
1045        }
1046
1047        private List<Collection<ResourcePersistentId>> partition(Collection<ResourcePersistentId> theNextRoundMatches, int theMaxLoad) {
1048                if (theNextRoundMatches.size() <= theMaxLoad) {
1049                        return Collections.singletonList(theNextRoundMatches);
1050                } else {
1051
1052                        List<Collection<ResourcePersistentId>> retVal = new ArrayList<>();
1053                        Collection<ResourcePersistentId> current = null;
1054                        for (ResourcePersistentId next : theNextRoundMatches) {
1055                                if (current == null) {
1056                                        current = new ArrayList<>(theMaxLoad);
1057                                        retVal.add(current);
1058                                }
1059
1060                                current.add(next);
1061
1062                                if (current.size() >= theMaxLoad) {
1063                                        current = null;
1064                                }
1065                        }
1066
1067                        return retVal;
1068                }
1069        }
1070
1071        private void attemptComboUniqueSpProcessing(QueryStack theQueryStack3, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
1072                RuntimeSearchParam comboParam = null;
1073                List<String> comboParamNames = null;
1074                List<RuntimeSearchParam> exactMatchParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
1075                if (exactMatchParams.size() > 0) {
1076                        comboParam = exactMatchParams.get(0);
1077                        comboParamNames = new ArrayList<>(theParams.keySet());
1078                }
1079
1080                if (comboParam == null) {
1081                        List<RuntimeSearchParam> candidateComboParams = mySearchParamRegistry.getActiveComboSearchParams(myResourceName);
1082                        for (RuntimeSearchParam nextCandidate : candidateComboParams) {
1083                                List<String> nextCandidateParamNames = JpaParamUtil
1084                                        .resolveComponentParameters(mySearchParamRegistry, nextCandidate)
1085                                        .stream()
1086                                        .map(t -> t.getName())
1087                                        .collect(Collectors.toList());
1088                                if (theParams.keySet().containsAll(nextCandidateParamNames)) {
1089                                        comboParam = nextCandidate;
1090                                        comboParamNames = nextCandidateParamNames;
1091                                        break;
1092                                }
1093                        }
1094                }
1095
1096                if (comboParam != null) {
1097                        // Since we're going to remove elements below
1098                        theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList));
1099
1100                        StringBuilder sb = new StringBuilder();
1101                        sb.append(myResourceName);
1102                        sb.append("?");
1103
1104                        boolean first = true;
1105
1106                        Collections.sort(comboParamNames);
1107                        for (String nextParamName : comboParamNames) {
1108                                List<List<IQueryParameterType>> nextValues = theParams.get(nextParamName);
1109
1110                                // TODO Hack to fix weird IOOB on the next stanza until James comes back and makes sense of this.
1111                                if (nextValues.isEmpty()) {
1112                                        ourLog.error("query parameter {} is unexpectedly empty. Encountered while considering {} index for {}", nextParamName, comboParam.getName(), theRequest.getCompleteUrl());
1113                                        sb = null;
1114                                        break;
1115                                }
1116
1117                                if (nextValues.get(0).size() != 1) {
1118                                        sb = null;
1119                                        break;
1120                                }
1121
1122                                // Reference params are only eligible for using a composite index if they
1123                                // are qualified
1124                                RuntimeSearchParam nextParamDef = mySearchParamRegistry.getActiveSearchParam(myResourceName, nextParamName);
1125                                if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
1126                                        ReferenceParam param = (ReferenceParam) nextValues.get(0).get(0);
1127                                        if (isBlank(param.getResourceType())) {
1128                                                sb = null;
1129                                                break;
1130                                        }
1131                                }
1132
1133                                List<? extends IQueryParameterType> nextAnd = nextValues.remove(0);
1134                                IQueryParameterType nextOr = nextAnd.remove(0);
1135                                String nextOrValue = nextOr.getValueAsQueryToken(myContext);
1136
1137                                if (comboParam.getComboSearchParamType() == ComboSearchParamType.NON_UNIQUE) {
1138                                        if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.STRING) {
1139                                                nextOrValue = StringUtil.normalizeStringForSearchIndexing(nextOrValue);
1140                                        }
1141                                }
1142
1143                                if (first) {
1144                                        first = false;
1145                                } else {
1146                                        sb.append('&');
1147                                }
1148
1149                                nextParamName = UrlUtil.escapeUrlParam(nextParamName);
1150                                nextOrValue = UrlUtil.escapeUrlParam(nextOrValue);
1151
1152                                sb.append(nextParamName).append('=').append(nextOrValue);
1153
1154                        }
1155
1156                        if (sb != null) {
1157                                String indexString = sb.toString();
1158                                ourLog.debug("Checking for {} combo index for query: {}", comboParam.getComboSearchParamType(), indexString);
1159
1160                                // Interceptor broadcast: JPA_PERFTRACE_INFO
1161                                StorageProcessingMessage msg = new StorageProcessingMessage()
1162                                        .setMessage("Using " + comboParam.getComboSearchParamType() + " index for query for search: " + indexString);
1163                                HookParams params = new HookParams()
1164                                        .add(RequestDetails.class, theRequest)
1165                                        .addIfMatchesType(ServletRequestDetails.class, theRequest)
1166                                        .add(StorageProcessingMessage.class, msg);
1167                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INFO, params);
1168
1169                                switch (comboParam.getComboSearchParamType()) {
1170                                        case UNIQUE:
1171                                                theQueryStack3.addPredicateCompositeUnique(indexString, myRequestPartitionId);
1172                                                break;
1173                                        case NON_UNIQUE:
1174                                                theQueryStack3.addPredicateCompositeNonUnique(indexString, myRequestPartitionId);
1175                                                break;
1176                                }
1177
1178                                // Remove any empty parameters remaining after this
1179                                theParams.clean();
1180                        }
1181                }
1182        }
1183
1184        private <T> void ensureSubListsAreWritable(List<List<T>> theListOfLists) {
1185                for (int i = 0; i < theListOfLists.size(); i++) {
1186                        List<T> oldSubList = theListOfLists.get(i);
1187                        if (!(oldSubList instanceof ArrayList)) {
1188                                List<T> newSubList = new ArrayList<>(oldSubList);
1189                                theListOfLists.set(i, newSubList);
1190                        }
1191                }
1192        }
1193
1194        @Override
1195        public void setFetchSize(int theFetchSize) {
1196                myFetchSize = theFetchSize;
1197        }
1198
1199        public SearchParameterMap getParams() {
1200                return myParams;
1201        }
1202
1203        public CriteriaBuilder getBuilder() {
1204                return myCriteriaBuilder;
1205        }
1206
1207        public Class<? extends IBaseResource> getResourceType() {
1208                return myResourceType;
1209        }
1210
1211        public String getResourceName() {
1212                return myResourceName;
1213        }
1214
1215        @VisibleForTesting
1216        public void setDaoConfigForUnitTest(DaoConfig theDaoConfig) {
1217                myDaoConfig = theDaoConfig;
1218        }
1219
1220        public class IncludesIterator extends BaseIterator<ResourcePersistentId> implements Iterator<ResourcePersistentId> {
1221
1222                private final RequestDetails myRequest;
1223                private final Set<ResourcePersistentId> myCurrentPids;
1224                private Iterator<ResourcePersistentId> myCurrentIterator;
1225                private ResourcePersistentId myNext;
1226
1227                IncludesIterator(Set<ResourcePersistentId> thePidSet, RequestDetails theRequest) {
1228                        myCurrentPids = new HashSet<>(thePidSet);
1229                        myCurrentIterator = null;
1230                        myRequest = theRequest;
1231                }
1232
1233                private void fetchNext() {
1234                        while (myNext == null) {
1235
1236                                if (myCurrentIterator == null) {
1237                                        Set<Include> includes = Collections.singleton(new Include("*", true));
1238                                        Set<ResourcePersistentId> newPids = loadIncludes(myContext, myEntityManager, myCurrentPids, includes, false, getParams().getLastUpdated(), mySearchUuid, myRequest, null);
1239                                        myCurrentIterator = newPids.iterator();
1240                                }
1241
1242                                if (myCurrentIterator.hasNext()) {
1243                                        myNext = myCurrentIterator.next();
1244                                } else {
1245                                        myNext = NO_MORE;
1246                                }
1247
1248                        }
1249                }
1250
1251                @Override
1252                public boolean hasNext() {
1253                        fetchNext();
1254                        return !NO_MORE.equals(myNext);
1255                }
1256
1257                @Override
1258                public ResourcePersistentId next() {
1259                        fetchNext();
1260                        ResourcePersistentId retVal = myNext;
1261                        myNext = null;
1262                        return retVal;
1263                }
1264
1265        }
1266
1267        private final class QueryIterator extends BaseIterator<ResourcePersistentId> implements IResultIterator {
1268
1269                private final SearchRuntimeDetails mySearchRuntimeDetails;
1270                private final RequestDetails myRequest;
1271                private final boolean myHaveRawSqlHooks;
1272                private final boolean myHavePerfTraceFoundIdHook;
1273                private final SortSpec mySort;
1274                private final Integer myOffset;
1275                private boolean myFirst = true;
1276                private IncludesIterator myIncludesIterator;
1277                private ResourcePersistentId myNext;
1278                private Iterator<ResourcePersistentId> myPreResultsIterator;
1279                private SearchQueryExecutor myResultsIterator;
1280                private boolean myStillNeedToFetchIncludes;
1281                private int mySkipCount = 0;
1282                private int myNonSkipCount = 0;
1283                private ArrayList<SearchQueryExecutor> myQueryList = new ArrayList<>();
1284
1285                private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
1286                        mySearchRuntimeDetails = theSearchRuntimeDetails;
1287                        mySort = myParams.getSort();
1288                        myOffset = myParams.getOffset();
1289                        myRequest = theRequest;
1290
1291                        // Includes are processed inline for $everything query
1292                        if (myParams.getEverythingMode() != null) {
1293                                myStillNeedToFetchIncludes = true;
1294                        }
1295
1296                        myHavePerfTraceFoundIdHook = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, myInterceptorBroadcaster, myRequest);
1297                        myHaveRawSqlHooks = CompositeInterceptorBroadcaster.hasHooks(Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, myRequest);
1298
1299                }
1300
1301                private void fetchNext() {
1302
1303                        try {
1304                                if (myHaveRawSqlHooks) {
1305                                        CurrentThreadCaptureQueriesListener.startCapturing();
1306                                }
1307
1308                                // If we don't have a query yet, create one
1309                                if (myResultsIterator == null) {
1310                                        if (myMaxResultsToFetch == null) {
1311                                                if (myParams.getLoadSynchronousUpTo() != null) {
1312                                                        myMaxResultsToFetch = myParams.getLoadSynchronousUpTo();
1313                                                } else if (myParams.getOffset() != null && myParams.getCount() != null) {
1314                                                        myMaxResultsToFetch = myParams.getCount();
1315                                                } else {
1316                                                        myMaxResultsToFetch = myDaoConfig.getFetchSizeDefaultMaximum();
1317                                                }
1318                                        }
1319
1320                                        initializeIteratorQuery(myOffset, myMaxResultsToFetch);
1321
1322                                        // If the query resulted in extra results being requested
1323                                        if (myAlsoIncludePids != null) {
1324                                                myPreResultsIterator = myAlsoIncludePids.iterator();
1325                                        }
1326                                }
1327
1328                                if (myNext == null) {
1329
1330                                        if (myPreResultsIterator != null && myPreResultsIterator.hasNext()) {
1331                                                while (myPreResultsIterator.hasNext()) {
1332                                                        ResourcePersistentId next = myPreResultsIterator.next();
1333                                                        if (next != null)
1334                                                                if (myPidSet.add(next)) {
1335                                                                        myNext = next;
1336                                                                        break;
1337                                                                }
1338                                                }
1339                                        }
1340
1341                                        if (myNext == null) {
1342                                                while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) {
1343                                                        // Update iterator with next chunk if necessary.
1344                                                        if (!myResultsIterator.hasNext()) {
1345                                                                retrieveNextIteratorQuery();
1346                                                        }
1347
1348                                                        Long nextLong = myResultsIterator.next();
1349                                                        if (myHavePerfTraceFoundIdHook) {
1350                                                                HookParams params = new HookParams()
1351                                                                        .add(Integer.class, System.identityHashCode(this))
1352                                                                        .add(Object.class, nextLong);
1353                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, params);
1354                                                        }
1355
1356                                                        if (nextLong != null) {
1357                                                                ResourcePersistentId next = new ResourcePersistentId(nextLong);
1358                                                                if (myPidSet.add(next)) {
1359                                                                        myNext = next;
1360                                                                        myNonSkipCount++;
1361                                                                        break;
1362                                                                } else {
1363                                                                        mySkipCount++;
1364                                                                }
1365                                                        }
1366
1367                                                        if (!myResultsIterator.hasNext()) {
1368                                                                if (myMaxResultsToFetch != null && (mySkipCount + myNonSkipCount == myMaxResultsToFetch)) {
1369                                                                        if (mySkipCount > 0 && myNonSkipCount == 0) {
1370                                                                                myMaxResultsToFetch += 1000;
1371
1372                                                                                StorageProcessingMessage message = new StorageProcessingMessage();
1373                                                                                String msg = "Pass completed with no matching results. This indicates an inefficient query! Retrying with new max count of " + myMaxResultsToFetch;
1374                                                                                ourLog.warn(msg);
1375                                                                                message.setMessage(msg);
1376                                                                                HookParams params = new HookParams()
1377                                                                                        .add(RequestDetails.class, myRequest)
1378                                                                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1379                                                                                        .add(StorageProcessingMessage.class, message);
1380                                                                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_WARNING, params);
1381
1382                                                                                initializeIteratorQuery(myOffset, myMaxResultsToFetch);
1383                                                                        }
1384                                                                }
1385                                                        }
1386                                                }
1387                                        }
1388
1389                                        if (myNext == null) {
1390                                                if (myStillNeedToFetchIncludes) {
1391                                                        myIncludesIterator = new IncludesIterator(myPidSet, myRequest);
1392                                                        myStillNeedToFetchIncludes = false;
1393                                                }
1394                                                if (myIncludesIterator != null) {
1395                                                        while (myIncludesIterator.hasNext()) {
1396                                                                ResourcePersistentId next = myIncludesIterator.next();
1397                                                                if (next != null)
1398                                                                        if (myPidSet.add(next)) {
1399                                                                                myNext = next;
1400                                                                                break;
1401                                                                        }
1402                                                        }
1403                                                        if (myNext == null) {
1404                                                                myNext = NO_MORE;
1405                                                        }
1406                                                } else {
1407                                                        myNext = NO_MORE;
1408                                                }
1409                                        }
1410
1411                                } // if we need to fetch the next result
1412
1413                                mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size());
1414
1415                        } finally {
1416                                if (myHaveRawSqlHooks) {
1417                                        SqlQueryList capturedQueries = CurrentThreadCaptureQueriesListener.getCurrentQueueAndStopCapturing();
1418                                        HookParams params = new HookParams()
1419                                                .add(RequestDetails.class, myRequest)
1420                                                .addIfMatchesType(ServletRequestDetails.class, myRequest)
1421                                                .add(SqlQueryList.class, capturedQueries);
1422                                        CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_RAW_SQL, params);
1423                                }
1424                        }
1425
1426                        if (myFirst) {
1427                                HookParams params = new HookParams()
1428                                        .add(RequestDetails.class, myRequest)
1429                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1430                                        .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
1431                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FIRST_RESULT_LOADED, params);
1432                                myFirst = false;
1433                        }
1434
1435                        if (NO_MORE.equals(myNext)) {
1436                                HookParams params = new HookParams()
1437                                        .add(RequestDetails.class, myRequest)
1438                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
1439                                        .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
1440                                CompositeInterceptorBroadcaster.doCallHooks(myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_SELECT_COMPLETE, params);
1441                        }
1442
1443                }
1444
1445                private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToFetch) {
1446                        if (myQueryList.isEmpty()) {
1447                                // Capture times for Lucene/Elasticsearch queries as well
1448                                mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
1449                                myQueryList = createQuery(myParams, mySort, theOffset, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails);
1450                        }
1451
1452                        mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
1453
1454                        retrieveNextIteratorQuery();
1455
1456                        mySkipCount = 0;
1457                        myNonSkipCount = 0;
1458                }
1459
1460                private void retrieveNextIteratorQuery() {
1461                        close();
1462                        if (myQueryList != null && myQueryList.size() > 0) {
1463                                myResultsIterator = myQueryList.remove(0);
1464                                hasNextIteratorQuery = true;
1465                        } else {
1466                                myResultsIterator = SearchQueryExecutor.emptyExecutor();
1467                                hasNextIteratorQuery = false;
1468                        }
1469
1470                }
1471
1472                @Override
1473                public boolean hasNext() {
1474                        if (myNext == null) {
1475                                fetchNext();
1476                        }
1477                        return !NO_MORE.equals(myNext);
1478                }
1479
1480                @Override
1481                public ResourcePersistentId next() {
1482                        fetchNext();
1483                        ResourcePersistentId retVal = myNext;
1484                        myNext = null;
1485                        Validate.isTrue(!NO_MORE.equals(retVal), "No more elements");
1486                        return retVal;
1487                }
1488
1489                @Override
1490                public int getSkippedCount() {
1491                        return mySkipCount;
1492                }
1493
1494                @Override
1495                public int getNonSkippedCount() {
1496                        return myNonSkipCount;
1497                }
1498
1499                @Override
1500                public Collection<ResourcePersistentId> getNextResultBatch(long theBatchSize) {
1501                        Collection<ResourcePersistentId> batch = new ArrayList<>();
1502                        while (this.hasNext() && batch.size() < theBatchSize) {
1503                                batch.add(this.next());
1504                        }
1505                        return batch;
1506                }
1507
1508                @Override
1509                public void close() {
1510                        if (myResultsIterator != null) {
1511                                myResultsIterator.close();
1512                        }
1513                        myResultsIterator = null;
1514                }
1515
1516        }
1517
1518        public static int getMaximumPageSize() {
1519                if (myUseMaxPageSize50ForTest) {
1520                        return MAXIMUM_PAGE_SIZE_FOR_TESTING;
1521                } else {
1522                        return MAXIMUM_PAGE_SIZE;
1523                }
1524        }
1525
1526        public static void setMaxPageSize50ForTest(boolean theIsTest) {
1527                myUseMaxPageSize50ForTest = theIsTest;
1528        }
1529
1530        private static List<Predicate> createLastUpdatedPredicates(final DateRangeParam theLastUpdated, CriteriaBuilder builder, From<?, ResourceTable> from) {
1531                List<Predicate> lastUpdatedPredicates = new ArrayList<>();
1532                if (theLastUpdated != null) {
1533                        if (theLastUpdated.getLowerBoundAsInstant() != null) {
1534                                ourLog.debug("LastUpdated lower bound: {}", new InstantDt(theLastUpdated.getLowerBoundAsInstant()));
1535                                Predicate predicateLower = builder.greaterThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getLowerBoundAsInstant());
1536                                lastUpdatedPredicates.add(predicateLower);
1537                        }
1538                        if (theLastUpdated.getUpperBoundAsInstant() != null) {
1539                                Predicate predicateUpper = builder.lessThanOrEqualTo(from.get("myUpdated"), theLastUpdated.getUpperBoundAsInstant());
1540                                lastUpdatedPredicates.add(predicateUpper);
1541                        }
1542                }
1543                return lastUpdatedPredicates;
1544        }
1545
1546        private static List<ResourcePersistentId> filterResourceIdsByLastUpdated(EntityManager theEntityManager, final DateRangeParam theLastUpdated, Collection<ResourcePersistentId> thePids) {
1547                if (thePids.isEmpty()) {
1548                        return Collections.emptyList();
1549                }
1550                CriteriaBuilder builder = theEntityManager.getCriteriaBuilder();
1551                CriteriaQuery<Long> cq = builder.createQuery(Long.class);
1552                Root<ResourceTable> from = cq.from(ResourceTable.class);
1553                cq.select(from.get("myId").as(Long.class));
1554
1555                List<Predicate> lastUpdatedPredicates = createLastUpdatedPredicates(theLastUpdated, builder, from);
1556                lastUpdatedPredicates.add(from.get("myId").as(Long.class).in(ResourcePersistentId.toLongList(thePids)));
1557
1558                cq.where(SearchBuilder.toPredicateArray(lastUpdatedPredicates));
1559                TypedQuery<Long> query = theEntityManager.createQuery(cq);
1560
1561                return ResourcePersistentId.fromLongList(query.getResultList());
1562        }
1563
1564        public static Predicate[] toPredicateArray(List<Predicate> thePredicates) {
1565                return thePredicates.toArray(new Predicate[0]);
1566        }
1567
1568}